diff --git a/.circleci/config.yml b/.circleci/config.yml
index 94fe9b2bfd8..6e46d3fe33c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -51,9 +51,36 @@ jobs:
command: |
python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
+ mypy_linting:
+ docker:
+ - image: cimg/python:3.12
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+ resource_class: medium
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip uninstall fastuuid -y
+ pip install "mypy==1.18.2"
+ - run:
+ name: MyPy Type Checking
+ command: |
+ cd litellm
+ # Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults
+ python -m mypy .
+ cd ..
+ no_output_timeout: 10m
local_testing:
docker:
- - image: cimg/python:3.11
+ - image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
@@ -79,23 +106,23 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
- pip install "mypy==1.15.0"
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
- pip install "langfuse==2.45.0"
+ pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.81.0
+ pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -118,6 +145,8 @@ jobs:
pip install "jsonschema==4.22.0"
pip install "pytest-xdist==3.6.1"
pip install "websockets==13.1.0"
+ pip install semantic_router --no-deps
+ pip install aurelio_sdk --no-deps
pip uninstall posthog -y
- setup_litellm_enterprise_pip
- save_cache:
@@ -138,19 +167,6 @@ jobs:
python -m pip install black
python -m black .
cd ..
- - run:
- name: Linting Testing
- command: |
- cd litellm
- pip install "cryptography<40.0.0"
- python -m pip install types-requests types-setuptools types-redis types-PyYAML
- if ! python -m mypy . \
- --config-file mypy.ini \
- --ignore-missing-imports; then
- echo "mypy detected errors"
- exit 1
- fi
- cd ..
# Run pytest and generate JUnit XML report
- run:
@@ -158,7 +174,7 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
+ python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -202,23 +218,23 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
- pip install "langfuse==2.45.0"
+ pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.81.0
+ pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -309,23 +325,23 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
- pip install "langfuse==2.45.0"
+ pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.81.0
+ pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -437,6 +453,7 @@ jobs:
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
+
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@@ -458,6 +475,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
+ pip install semantic_router --no-deps
+ pip install aurelio_sdk --no-deps
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
@@ -465,7 +484,7 @@ jobs:
command: |
pwd
ls
- python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
+ python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -481,13 +500,59 @@ jobs:
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
- litellm_proxy_security_tests:
+
+ litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "respx==0.22.0"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-asyncio==0.21.1"
+ pip install semantic_router --no-deps
+ pip install aurelio_sdk --no-deps
+ pip install "pytest-xdist==3.6.1"
+ # Run pytest and generate JUnit XML report
+ - setup_litellm_enterprise_pip
+ - run:
+ name: Run tests
+ command: |
+ pwd
+ ls
+ python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml litellm_router_coverage.xml
+ mv .coverage litellm_router_coverage
+ # Store test results
+ - store_test_results:
+ path: test-results
+
+ - persist_to_workspace:
+ root: .
+ paths:
+ - litellm_router_coverage.xml
+ - litellm_router_coverage
+ litellm_security_tests:
+ machine:
+ image: ubuntu-2204:2023.10.1
+ resource_class: xlarge
+ working_directory: ~/project
steps:
- checkout
- setup_google_dns
@@ -495,15 +560,85 @@ jobs:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
+ - run:
+ name: Install Docker CLI (In case it's not already installed)
+ command: |
+ sudo apt-get update
+ sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+ - run:
+ name: Install Python 3.13
+ command: |
+ curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+ bash miniconda.sh -b -p $HOME/miniconda
+ export PATH="$HOME/miniconda/bin:$PATH"
+ conda init bash
+ source ~/.bashrc
+ conda create -n myenv python=3.13 -y
+ conda activate myenv
+ python --version
- run:
name: Install Dependencies
command: |
+ pip install "pytest==7.3.1"
+ pip install "pytest-asyncio==0.21.1"
+ pip install aiohttp
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
+ pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
+ pip install "mypy==1.18.2"
+ pip install "google-generativeai==0.3.2"
+ pip install "google-cloud-aiplatform==1.43.0"
+ pip install pyarrow
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
+ pip install langchain
+ pip install "langfuse>=2.0.0"
+ pip install "logfire==0.29.0"
+ pip install numpydoc
+ pip install prisma
+ pip install fastapi
+ pip install jsonschema
+ pip install "httpx==0.24.1"
+ pip install "gunicorn==21.2.0"
+ pip install "anyio==3.7.1"
+ pip install "aiodynamo==23.10.1"
+ pip install "asyncio==3.4.3"
+ pip install "PyGithub==1.59.1"
+ pip install "openai==1.100.1"
pip install "pytest-cov==5.0.0"
+ pip install "apscheduler"
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
+ - run:
+ name: Set DATABASE_URL environment variable
+ command: |
+ echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV
+ source $BASH_ENV
+ - run:
+ name: Run Security Scans
+ command: |
+ chmod +x ci_cd/security_scans.sh
+ ./ci_cd/security_scans.sh
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
@@ -522,16 +657,16 @@ jobs:
- run:
name: Rename the coverage files
command: |
- mv coverage.xml litellm_proxy_security_tests_coverage.xml
- mv .coverage litellm_proxy_security_tests_coverage
+ mv coverage.xml litellm_security_tests_coverage.xml
+ mv .coverage litellm_security_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- - litellm_proxy_security_tests_coverage.xml
- - litellm_proxy_security_tests_coverage
+ - litellm_security_tests_coverage.xml
+ - litellm_security_tests_coverage
litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
docker:
- image: cimg/python:3.11
@@ -565,23 +700,24 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
+ pip install "google-genai==1.22.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
- pip install "langfuse==2.45.0"
+ pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
- pip install openai==1.81.0
+ pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@@ -604,6 +740,7 @@ jobs:
pip install "jsonschema==4.22.0"
pip install "pytest-postgresql==7.0.1"
pip install "fakeredis==2.28.1"
+ pip install "pytest-xdist==3.6.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
@@ -622,7 +759,7 @@ jobs:
command: |
pwd
ls
- python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
+ python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -683,43 +820,6 @@ jobs:
paths:
- litellm_assistants_api_coverage.xml
- litellm_assistants_api_coverage
- load_testing:
- docker:
- - image: cimg/python:3.11
- auth:
- username: ${DOCKERHUB_USERNAME}
- password: ${DOCKERHUB_PASSWORD}
- working_directory: ~/project
-
- steps:
- - checkout
- - setup_google_dns
- - run:
- name: Install Dependencies
- command: |
- python -m pip install --upgrade pip
- python -m pip install -r requirements.txt
- pip install "pytest==7.3.1"
- pip install "pytest-retry==1.6.3"
- pip install "pytest-cov==5.0.0"
- pip install "pytest-asyncio==0.21.1"
- pip install "respx==0.22.0"
- - run:
- name: Show current pydantic version
- command: |
- python -m pip show pydantic
- # Run pytest and generate JUnit XML report
- - run:
- name: Run tests
- command: |
- pwd
- ls
- python -m pytest -vv tests/load_tests -x -s -v --junitxml=test-results/junit.xml --durations=5
- no_output_timeout: 120m
-
- # Store test results
- - store_test_results:
- path: test-results
llm_translation_testing:
docker:
- image: cimg/python:3.11
@@ -741,13 +841,14 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
+ pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
- python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
+ python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=5 -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -785,7 +886,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
- pip install "mcp==1.5.0"
+ pip install "mcp==1.10.1"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -830,7 +931,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
- pip install "boto3==1.34.34"
+ pip install "boto3==1.36.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -853,6 +954,52 @@ jobs:
paths:
- guardrails_coverage.xml
- guardrails_coverage
+
+ google_generate_content_endpoint_testing:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "respx==0.22.0"
+ pip install "pydantic==2.10.2"
+ # Run pytest and generate JUnit XML report
+ - run:
+ name: Run tests
+ command: |
+ pwd
+ ls
+ python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+ no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml google_generate_content_endpoint_coverage.xml
+ mv .coverage google_generate_content_endpoint_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - google_generate_content_endpoint_coverage.xml
+ - google_generate_content_endpoint_coverage
+
llm_responses_api_testing:
docker:
- image: cimg/python:3.11
@@ -920,10 +1067,12 @@ jobs:
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
- pip install "mcp==1.5.0"
+ pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
+ pip install "semantic_router==0.1.10"
+ pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
# Run pytest and generate JUnit XML report
- run:
@@ -931,14 +1080,60 @@ jobs:
command: |
pwd
ls
- python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 4
+ python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
+ - run:
+ name: Rename the coverage files
+ command: |
+ mv coverage.xml litellm_mapped_tests_coverage.xml
+ mv .coverage litellm_mapped_tests_coverage
+
+ # Store test results
+ - store_test_results:
+ path: test-results
+ - persist_to_workspace:
+ root: .
+ paths:
+ - litellm_mapped_tests_coverage.xml
+ - litellm_mapped_tests_coverage
+ litellm_mapped_enterprise_tests:
+ docker:
+ - image: cimg/python:3.11
+ auth:
+ username: ${DOCKERHUB_USERNAME}
+ password: ${DOCKERHUB_PASSWORD}
+ working_directory: ~/project
+
+ steps:
+ - checkout
+ - setup_google_dns
+ - run:
+ name: Install Dependencies
+ command: |
+ python -m pip install --upgrade pip
+ python -m pip install -r requirements.txt
+ pip install "pytest-mock==3.12.0"
+ pip install "pytest==7.3.1"
+ pip install "pytest-retry==1.6.3"
+ pip install "pytest-cov==5.0.0"
+ pip install "pytest-asyncio==0.21.1"
+ pip install "respx==0.22.0"
+ pip install "hypercorn==0.17.3"
+ pip install "pydantic==2.10.2"
+ pip install "mcp==1.10.1"
+ pip install "requests-mock>=1.12.1"
+ pip install "responses==0.25.7"
+ pip install "pytest-xdist==3.6.1"
+ pip install "semantic_router==0.1.10"
+ pip install "fastapi-offline==1.7.3"
+ - setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
pwd
ls
- python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
+ prisma generate
+ python -m pytest -vv tests/enterprise --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-enterprise.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
@@ -1023,6 +1218,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
+ pip install pytest-mock
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@@ -1203,10 +1399,11 @@ jobs:
pip install aiohttp
pip install openai
pip install click
- pip install "boto3==1.34.34"
+ pip install "boto3==1.36.0"
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "uvloop==0.21.0"
+ pip install "fastuuid==0.12.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@@ -1238,6 +1435,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
+ pip install "mcp==1.10.1"
- run:
name: Run tests
command: |
@@ -1338,11 +1536,15 @@ jobs:
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
- run: python ./tests/code_coverage_tests/check_licenses.py
- run: python ./tests/code_coverage_tests/router_code_coverage.py
+ - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
+ - run: python ./tests/code_coverage_tests/info_log_check.py
+ - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
+ - run: python ./tests/code_coverage_tests/code_qa_check_tests.py
+ - run: python ./tests/code_coverage_tests/test_proxy_types_import.py
- run: python ./tests/code_coverage_tests/callback_manager_test.py
- run: python ./tests/code_coverage_tests/recursive_detector.py
- run: python ./tests/code_coverage_tests/test_router_strategy_async.py
- run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
- - run: python ./tests/code_coverage_tests/bedrock_pricing.py
- run: python ./tests/documentation_tests/test_env_keys.py
- run: python ./tests/documentation_tests/test_router_settings.py
- run: python ./tests/documentation_tests/test_api_docs.py
@@ -1351,6 +1553,8 @@ jobs:
- run: python ./tests/documentation_tests/test_circular_imports.py
- run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
+ - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+ - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@@ -1388,6 +1592,7 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DISABLE_SCHEMA_UPDATE="True" \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
@@ -1464,12 +1669,12 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -1483,23 +1688,26 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
- pip install "openai==1.81.0"
+ pip install "openai==1.100.1"
- run:
- name: Install Grype
+ name: Install dockerize
command: |
- curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
- name: Build and Scan Docker Images
+ name: Start PostgreSQL Database
command: |
- # Build and scan Dockerfile.database
- echo "Building and scanning Dockerfile.database..."
- docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
- grype litellm-database:latest --fail-on high
-
- # Build and scan main Dockerfile
- echo "Building and scanning main Dockerfile..."
- docker build -t litellm:latest .
- grype litellm:latest --fail-on high
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1508,7 +1716,8 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
+ -e USE_PRISMA_MIGRATE=True \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@@ -1532,6 +1741,7 @@ jobs:
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
my-app:latest \
@@ -1539,13 +1749,10 @@ jobs:
--port 4000 \
--detailed_debug \
- run:
- name: Install curl and dockerize
+ name: Install curl
command: |
sudo apt-get update
sudo apt-get install -y curl
- sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
@@ -1600,13 +1807,13 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "jsonlines==4.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@@ -1621,8 +1828,27 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
- pip install "openai==1.81.0"
+ pip install "openai==1.100.1"
# Run pytest and generate JUnit XML report
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1631,9 +1857,9 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
- -e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \
- -e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
+ -e AZURE_API_KEY=$AZURE_API_KEY \
+ -e AZURE_API_BASE=$AZURE_API_BASE \
-e AZURE_API_VERSION="2024-05-01-preview" \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@@ -1657,6 +1883,7 @@ jobs:
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
my-app:latest \
@@ -1664,13 +1891,10 @@ jobs:
--port 4000 \
--detailed_debug \
- run:
- name: Install curl and dockerize
+ name: Install curl
command: |
sudo apt-get update
sudo apt-get install -y curl
- sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
@@ -1725,12 +1949,12 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
- pip install "boto3==1.34.34"
- pip install "aioboto3==12.3.0"
+ pip install "boto3==1.36.0"
+ pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@@ -1744,7 +1968,26 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
- pip install "openai==1.81.0"
+ pip install "openai==1.100.1"
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1755,7 +1998,7 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
@@ -1768,6 +2011,7 @@ jobs:
-e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
@@ -1775,6 +2019,7 @@ jobs:
-e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
-e COHERE_API_KEY=$COHERE_API_KEY \
-e GCS_FLUSH_INTERVAL="1" \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
@@ -1819,13 +2064,14 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE="bad-license" \
+ --add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
my-app:latest \
@@ -1883,6 +2129,25 @@ jobs:
pip install aiohttp
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1893,7 +2158,7 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
@@ -1906,6 +2171,7 @@ jobs:
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
my-app:latest \
@@ -1913,13 +2179,10 @@ jobs:
--port 4000 \
--detailed_debug \
- run:
- name: Install curl and dockerize
+ name: Install curl
command: |
sudo apt-get update
sudo apt-get install -y curl
- sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
@@ -1978,6 +2241,25 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1988,7 +2270,7 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
@@ -1997,6 +2279,7 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
@@ -2008,7 +2291,7 @@ jobs:
command: |
docker run -d \
-p 4001:4001 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
@@ -2017,6 +2300,7 @@ jobs:
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
+ --add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
@@ -2168,18 +2452,16 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install mypy
+ pip install "mypy==1.18.2"
- run:
name: Build Docker image
command: |
- cd docker/build_from_pip
- docker build -t my-app:latest -f Dockerfile.build_from_pip .
+ docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
- run:
name: Run Docker container
# intentionally give bad redis credentials here
# the OTEL test - should get this as a trace
command: |
- cd docker/build_from_pip
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
@@ -2203,7 +2485,7 @@ jobs:
-e DD_SITE=$DD_SITE \
-e GCS_FLUSH_INTERVAL="1" \
--name my-app \
- -v $(pwd)/litellm_config.yaml:/app/config.yaml \
+ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
@@ -2267,15 +2549,15 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
- pip install "openai==1.81.0"
+ pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install "boto3==1.34.34"
- pip install mypy
+ pip install "boto3==1.36.0"
+ pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
@@ -2290,6 +2572,25 @@ jobs:
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
# Run pytest and generate JUnit XML report
- run:
name: Build Docker image
@@ -2299,16 +2600,19 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
- -e DATABASE_URL=$PROXY_DATABASE_URL \
+ -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e GEMINI_API_KEY=$GEMINI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
+ -e AZURE_API_KEY_PASSHROUGH=$AZURE_API_KEY_PASSHROUGH \
+ -e AZURE_API_BASE_PASSHROUGH=$AZURE_API_BASE_PASSHROUGH \
-e USE_DDTRACE=True \
-e DD_API_KEY=$DD_API_KEY \
-e DD_SITE=$DD_SITE \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
+ --add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
@@ -2316,14 +2620,6 @@ jobs:
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
- - run:
- name: Install curl and dockerize
- command: |
- sudo apt-get update
- sudo apt-get install -y curl
- sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
- sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
- sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
@@ -2392,6 +2688,7 @@ jobs:
ls
python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
+
# Store test results
- store_test_results:
path: test-results
@@ -2417,7 +2714,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
- coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_proxy_security_tests_coverage guardrails_coverage
+ coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@@ -2438,16 +2735,6 @@ jobs:
command: |
cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- - run:
- name: Check if litellm dir, tests dir, or pyproject.toml was modified
- command: |
- if [ -n "$(git diff --name-only $CIRCLE_SHA1^..$CIRCLE_SHA1 | grep -E 'pyproject\.toml|litellm/|tests/')" ]; then
- echo "litellm, tests, or pyproject.toml updated"
- else
- echo "No changes to litellm, tests, or pyproject.toml. Skipping PyPI publish."
- circleci step halt
- fi
-
- run:
name: Checkout code
command: git checkout $CIRCLE_SHA1
@@ -2620,8 +2907,8 @@ jobs:
source "$NVM_DIR/bash_completion"
# Install and use Node version
- nvm install v18.17.0
- nvm use v18.17.0
+ nvm install v20
+ nvm use v20
cd ui/litellm-dashboard
@@ -2655,13 +2942,13 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
- pip install "openai==1.81.0"
+ pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
- pip install mypy
+ pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
@@ -2674,7 +2961,26 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
+ - run:
+ name: Run UI unit tests (Vitest)
+ command: |
+ # Use Node 20 (several deps require >=20)
+ export NVM_DIR="/opt/circleci/.nvm"
+ source "$NVM_DIR/nvm.sh"
+ nvm install 20
+ nvm use 20
+ cd ui/litellm-dashboard
+ npm ci || npm install
+
+ # CI run, with both LCOV (Codecov) and HTML (artifact you can click)
+ CI=true npm run test -- --run --coverage \
+ --coverage.provider=v8 \
+ --coverage.reporter=lcov \
+ --coverage.reporter=html \
+ --coverage.reportsDirectory=coverage/html
+
+
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -2749,6 +3055,25 @@ jobs:
steps:
- checkout
- setup_google_dns
+ - run:
+ name: Install dockerize
+ command: |
+ wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+ sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+ rm dockerize-linux-amd64-v0.6.1.tar.gz
+ - run:
+ name: Start PostgreSQL Database
+ command: |
+ docker run -d \
+ --name postgres-db \
+ -e POSTGRES_USER=postgres \
+ -e POSTGRES_PASSWORD=postgres \
+ -e POSTGRES_DB=circle_test \
+ -p 5432:5432 \
+ postgres:14
+ - run:
+ name: Wait for PostgreSQL to be ready
+ command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Build Docker image
command: |
@@ -2758,6 +3083,7 @@ jobs:
command: |
docker run --name my-app \
-p 4000:4000 \
+ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
myapp:latest \
--port 4000 > docker_output.log 2>&1 || true
@@ -2768,7 +3094,6 @@ jobs:
name: Check for expected error
command: |
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
- grep -q "httpx.ConnectError: All connection attempts failed" docker_output.log && \
grep -q "ERROR: Application startup failed. Exiting." docker_output.log; then
echo "Expected error found. Test passed."
else
@@ -2787,6 +3112,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - mypy_linting:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- local_testing:
filters:
branches:
@@ -2811,7 +3142,7 @@ workflows:
only:
- main
- /litellm_.*/
- - litellm_proxy_security_tests:
+ - litellm_security_tests:
filters:
branches:
only:
@@ -2829,6 +3160,12 @@ workflows:
only:
- main
- /litellm_.*/
+ - litellm_router_unit_testing:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- check_code_and_doc_quality:
filters:
branches:
@@ -2913,12 +3250,24 @@ workflows:
only:
- main
- /litellm_.*/
+ - google_generate_content_endpoint_testing:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- llm_responses_api_testing:
filters:
branches:
only:
- main
- /litellm_.*/
+ - litellm_mapped_enterprise_tests:
+ filters:
+ branches:
+ only:
+ - main
+ - /litellm_.*/
- litellm_mapped_tests:
filters:
branches:
@@ -2959,18 +3308,21 @@ workflows:
requires:
- llm_translation_testing
- mcp_testing
+ - google_generate_content_endpoint_testing
- guardrails_testing
- llm_responses_api_testing
- litellm_mapped_tests
+ - litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
- image_gen_testing
- logging_testing
- litellm_router_testing
+ - litellm_router_unit_testing
- caching_unit_tests
- litellm_proxy_unit_testing
- - litellm_proxy_security_tests
+ - litellm_security_tests
- langfuse_logging_unit_tests
- local_testing
- litellm_assistants_api_testing
@@ -2999,12 +3351,6 @@ workflows:
only:
- main
- /litellm_.*/
- - load_testing:
- filters:
- branches:
- only:
- - main
- - /litellm_.*/
- test_bad_database_url:
filters:
branches:
@@ -3018,21 +3364,24 @@ workflows:
- main
- publish_to_pypi:
requires:
+ - mypy_linting
- local_testing
- build_and_test
- e2e_openai_endpoints
- - load_testing
- test_bad_database_url
- llm_translation_testing
- mcp_testing
+ - google_generate_content_endpoint_testing
- llm_responses_api_testing
- litellm_mapped_tests
+ - litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
- image_gen_testing
- logging_testing
- litellm_router_testing
+ - litellm_router_unit_testing
- caching_unit_tests
- langfuse_logging_unit_tests
- litellm_assistants_api_testing
@@ -3040,7 +3389,7 @@ workflows:
- db_migration_disable_update_check
- e2e_ui_testing
- litellm_proxy_unit_testing
- - litellm_proxy_security_tests
+ - litellm_security_tests
- installing_litellm_on_python
- installing_litellm_on_python_3_13
- proxy_logging_guardrails_model_info_tests
diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt
index b720d15a7fd..8e0f1dfe7e9 100644
--- a/.circleci/requirements.txt
+++ b/.circleci/requirements.txt
@@ -1,5 +1,5 @@
# used by CI/CD testing
-openai==1.81.0
+openai==1.100.1
python-dotenv
tiktoken
importlib_metadata
@@ -10,6 +10,9 @@ anthropic
orjson==3.10.12 # fast /embedding responses
pydantic==2.10.2
google-cloud-aiplatform==1.43.0
+google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
-mcp==1.5.0 # for MCP server
+mcp==1.10.1 # for MCP server
+semantic_router==0.1.10 # for auto-routing with litellm
+fastuuid==0.12.0
\ No newline at end of file
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index b3acd2e346d..50253186c01 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -11,7 +11,12 @@
// },
// Features to add to the dev container. More info: https://containers.dev/features.
- // "features": {},
+ "features": {
+ "ghcr.io/devcontainers/features/node:1": {
+ "version": "lts"
+ },
+ "ghcr.io/devcontainers/features/docker-in-docker:2": {}
+ },
// Configure tool-specific properties.
"customizations": {
@@ -30,7 +35,7 @@
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [4000],
-
+
"containerEnv": {
"LITELLM_LOG": "DEBUG"
},
@@ -48,5 +53,5 @@
// "remoteUser": "litellm",
// Use 'postCreateCommand' to run commands after the container is created.
- "postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+ "postCreateCommand": "bash ./.devcontainer/post-create.sh"
}
\ No newline at end of file
diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh
new file mode 100644
index 00000000000..bd72e91a20f
--- /dev/null
+++ b/.devcontainer/post-create.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "[post-create] Installing poetry via pip"
+python -m pip install --upgrade pip
+python -m pip install poetry
+
+echo "[post-create] Installing Python dependencies (poetry)"
+poetry install --with dev --extras proxy
+
+echo "[post-create] Generating Prisma client"
+poetry run prisma generate
+
+echo "[post-create] Installing npm dependencies"
+cd ui/litellm-dashboard && npm install --no-audit --no-fund
+
+echo "[post-create] Done"
\ No newline at end of file
diff --git a/.github/scripts/scan_keywords.py b/.github/scripts/scan_keywords.py
new file mode 100644
index 00000000000..98d32b61afe
--- /dev/null
+++ b/.github/scripts/scan_keywords.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+
+
+def read_event_payload() -> dict:
+ event_path = os.environ.get("GITHUB_EVENT_PATH")
+ if not event_path or not os.path.exists(event_path):
+ return {}
+ with open(event_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+
+
+def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
+ issue = event.get("issue") or {}
+ title = (issue.get("title") or "").strip()
+ body = (issue.get("body") or "").strip()
+ number = issue.get("number") or 0
+ html_url = issue.get("html_url") or ""
+ author = ((issue.get("user") or {}).get("login") or "").strip()
+ return title, body, number, html_url, author
+
+
+def detect_keywords(text: str, keywords: list[str]) -> list[str]:
+ lowered = text.lower()
+ matches = []
+ for keyword in keywords:
+ k = keyword.strip().lower()
+ if not k:
+ continue
+ if k in lowered:
+ matches.append(keyword.strip())
+ # Deduplicate while preserving order
+ seen = set()
+ unique_matches = []
+ for m in matches:
+ if m not in seen:
+ unique_matches.append(m)
+ seen.add(m)
+ return unique_matches
+
+
+def send_webhook(webhook_url: str, payload: dict) -> None:
+ if not webhook_url:
+ return
+ data = json.dumps(payload).encode("utf-8")
+ req = urllib.request.Request(
+ webhook_url,
+ data=data,
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=10) as resp:
+ resp.read()
+ except urllib.error.HTTPError as e:
+ print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
+ except urllib.error.URLError as e:
+ print(f"Webhook URL error: {e.reason}", file=sys.stderr)
+ except Exception as e:
+ print(f"Webhook unexpected error: {e}", file=sys.stderr)
+
+
+def _excerpt(text: str, max_len: int = 400) -> str:
+ if not text:
+ return ""
+
+ # Keep original formatting
+ if len(text) <= max_len:
+ return text
+ return text[: max_len - 1] + "…"
+
+
+
+def main() -> int:
+ event = read_event_payload()
+ if not event:
+ print("::warning::No event payload found; exiting without labeling.")
+ return 0
+
+ # Read issue details
+ title, body, number, html_url, author = get_issue_text(event)
+ combined_text = f"{title}\n\n{body}".strip()
+
+ # Keywords from env or defaults
+ keywords_env = os.environ.get("KEYWORDS", "")
+ default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
+ keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+
+ matches = detect_keywords(combined_text, keywords)
+ found = bool(matches)
+
+ # Emit outputs
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with open(github_output, "a", encoding="utf-8") as fh:
+ fh.write(f"found={'true' if found else 'false'}\n")
+ fh.write(f"matches={','.join(matches)}\n")
+
+ # Optional webhook notification
+ webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
+ if found and webhook_url:
+ repo_full = (event.get("repository") or {}).get("full_name", "")
+ title_part = f"*{title}*" if title else "New issue"
+ author_part = f" by @{author}" if author else ""
+ body_preview = _excerpt(body)
+ preview_block = f"\n{body_preview}" if body_preview else ""
+ payload = {
+ "text": (
+ f"New issue 🚨\n"
+ f"{title_part}\n\n{preview_block}\n"
+ f"<{html_url}|View issue>\n"
+ f"Author: {author}"
+ )
+ }
+ send_webhook(webhook_url, payload)
+
+ # Print a short log line for Actions UI
+ if found:
+ print(f"Detected provider keywords: {', '.join(matches)}")
+ else:
+ print("No provider keywords detected.")
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
+
+
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
new file mode 100644
index 00000000000..b4e777969d9
--- /dev/null
+++ b/.github/workflows/README.md
@@ -0,0 +1,35 @@
+# Simple PyPI Publishing
+
+A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
+
+## How to Use
+
+1. Go to the **Actions** tab in the GitHub repository
+2. Select **Simple PyPI Publish** from the workflow list
+3. Click **Run workflow**
+4. Enter the version to publish (e.g., `1.74.10`)
+
+## What the Workflow Does
+
+1. **Updates** the version in `pyproject.toml`
+2. **Copies** the model prices backup file
+3. **Builds** the Python package
+4. **Publishes** to PyPI
+
+## Prerequisites
+
+Make sure the following secret is configured in the repository:
+- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
+
+## Example Usage
+
+- Version: `1.74.11` → Publishes as v1.74.11
+- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
+
+## Features
+
+- ✅ Manual trigger with version input
+- ✅ Automatic version updates in `pyproject.toml`
+- ✅ Repository safety check (only runs on official repo)
+- ✅ Clean package building and publishing
+- ✅ Success confirmation with PyPI package link
\ No newline at end of file
diff --git a/.github/workflows/auto_update_price_and_context_window_file.py b/.github/workflows/auto_update_price_and_context_window_file.py
index 3e0731b94bd..461d8d347d9 100644
--- a/.github/workflows/auto_update_price_and_context_window_file.py
+++ b/.github/workflows/auto_update_price_and_context_window_file.py
@@ -43,8 +43,8 @@ def write_to_file(file_path, data):
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
-# Update the existing models and add the missing models
-def transform_remote_data(data):
+# Update the existing models and add the missing models for OpenRouter
+def transform_openrouter_data(data):
transformed = {}
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
@@ -81,6 +81,34 @@ def transform_remote_data(data):
return transformed
+# Update the existing models and add the missing models for Vercel AI Gateway
+def transform_vercel_ai_gateway_data(data):
+ transformed = {}
+ for row in data:
+ obj = {
+ "max_tokens": row["context_window"],
+ "input_cost_per_token": float(row["pricing"]["input"]),
+ "output_cost_per_token": float(row["pricing"]["output"]),
+ 'max_output_tokens': row['max_tokens'],
+ 'max_input_tokens': row["context_window"],
+ }
+
+ # Handle cache pricing if available
+ if "pricing" in row:
+ if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
+ obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
+
+ if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
+ obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
+
+ mode = "embedding" if "embedding" in row["id"].lower() else "chat"
+
+ obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
+
+ transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
+
+ return transformed
+
# Load local data from a specified file
def load_local_data(file_path):
@@ -100,22 +128,32 @@ def load_local_data(file_path):
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
- url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data
+ openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
+ vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
- # Fetch remote data asynchronously
- remote_data = asyncio.run(fetch_data(url))
- # Transform the fetched remote data
- remote_data = transform_remote_data(remote_data)
+
+ # Fetch OpenRouter data
+ openrouter_data = asyncio.run(fetch_data(openrouter_url))
+ # Transform the fetched OpenRouter data
+ openrouter_data = transform_openrouter_data(openrouter_data)
+
+ # Fetch Vercel AI Gateway data
+ vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
+ # Transform the fetched Vercel AI Gateway data
+ vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+
+ # Combine both datasets
+ all_remote_data = {**openrouter_data, **vercel_data}
- # If both local and remote data are available, synchronize and save
- if local_data and remote_data:
- sync_local_data_with_remote(local_data, remote_data)
+ # If both local and openrouter data are available, synchronize and save
+ if local_data and all_remote_data:
+ sync_local_data_with_remote(local_data, all_remote_data)
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml
index e7023138871..cc40d1ac0c0 100644
--- a/.github/workflows/ghcr_deploy.yml
+++ b/.github/workflows/ghcr_deploy.yml
@@ -6,7 +6,7 @@ on:
tag:
description: "The tag version you want to build"
release_type:
- description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
+ description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
default: "latest"
commit_hash:
@@ -73,7 +73,14 @@ jobs:
push: true
file: ./litellm-js/spend-logs/Dockerfile
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-
+ -
+ name: Build and push litellm-non_root image
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ push: true
+ file: ./docker/Dockerfile.non_root
+ tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
@@ -114,9 +121,9 @@ jobs:
tags: |
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
labels: ${{ steps.meta.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -158,7 +165,7 @@ jobs:
tags: |
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-ee.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -201,7 +208,7 @@ jobs:
tags: |
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-database.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -244,7 +251,7 @@ jobs:
tags: |
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-non_root.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -287,7 +294,7 @@ jobs:
tags: |
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
- ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+ ${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml
new file mode 100644
index 00000000000..60c18e3b9af
--- /dev/null
+++ b/.github/workflows/issue-keyword-labeler.yml
@@ -0,0 +1,64 @@
+name: Issue Keyword Labeler
+
+on:
+ issues:
+ types:
+ - opened
+
+jobs:
+ scan-and-label:
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ contents: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Scan for provider keywords
+ id: scan
+ env:
+ PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
+ KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+ run: python3 .github/scripts/scan_keywords.py
+
+ - name: Ensure label exists
+ if: steps.scan.outputs.found == 'true'
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const labelName = 'llm translation';
+ try {
+ await github.rest.issues.getLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: labelName
+ });
+ } catch (error) {
+ if (error.status === 404) {
+ await github.rest.issues.createLabel({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ name: labelName,
+ color: 'c1ff72',
+ description: 'Issues related to LLM provider translation/mapping'
+ });
+ } else {
+ throw error;
+ }
+ }
+
+ - name: Add label to the issue
+ if: steps.scan.outputs.found == 'true'
+ uses: actions/github-script@v7
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ await github.rest.issues.addLabels({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ labels: ['llm translation']
+ });
+
diff --git a/.github/workflows/llm-translation-testing.yml b/.github/workflows/llm-translation-testing.yml
new file mode 100644
index 00000000000..7fda37a66dc
--- /dev/null
+++ b/.github/workflows/llm-translation-testing.yml
@@ -0,0 +1,89 @@
+name: LLM Translation Tests
+
+on:
+ workflow_dispatch:
+ inputs:
+ release_candidate_tag:
+ description: 'Release candidate tag/version'
+ required: true
+ type: string
+ push:
+ tags:
+ - 'v*-rc*' # Triggers on release candidate tags like v1.0.0-rc1
+
+jobs:
+ run-llm-translation-tests:
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
+
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+ with:
+ version: latest
+ virtualenvs-create: true
+ virtualenvs-in-project: true
+
+ - name: Cache Poetry dependencies
+ uses: actions/cache@v3
+ with:
+ path: |
+ ~/.cache/pypoetry
+ .venv
+ key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-poetry-
+
+ - name: Install dependencies
+ run: |
+ poetry install --with dev
+ poetry run pip install pytest-xdist pytest-timeout
+
+ - name: Create test results directory
+ run: mkdir -p test-results
+
+ - name: Run LLM Translation Tests
+ env:
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+ AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+ AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+ AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
+ # Add other API keys as needed
+ run: |
+ python .github/workflows/run_llm_translation_tests.py \
+ --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
+ --commit "${{ github.sha }}" \
+ || true # Continue even if tests fail
+
+ - name: Display test summary
+ if: always()
+ run: |
+ if [ -f "test-results/llm_translation_report.md" ]; then
+ echo "Test report generated successfully!"
+ echo "Artifact will contain:"
+ echo "- test-results/junit.xml (JUnit XML results)"
+ echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
+ else
+ echo "Warning: Test report was not generated"
+ fi
+
+ - name: Upload test artifacts
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+ path: test-results/
+ retention-days: 30
diff --git a/.github/workflows/run_llm_translation_tests.py b/.github/workflows/run_llm_translation_tests.py
new file mode 100755
index 00000000000..5b3a4817ecb
--- /dev/null
+++ b/.github/workflows/run_llm_translation_tests.py
@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""
+Run LLM Translation Tests and Generate Beautiful Markdown Report
+
+This script runs the LLM translation tests and generates a comprehensive
+markdown report with provider-specific breakdowns and test statistics.
+"""
+
+import os
+import sys
+import subprocess
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+import json
+from typing import Dict, List, Tuple, Optional
+
+# ANSI color codes for terminal output
+class Colors:
+ GREEN = '\033[92m'
+ RED = '\033[91m'
+ YELLOW = '\033[93m'
+ BLUE = '\033[94m'
+ PURPLE = '\033[95m'
+ CYAN = '\033[96m'
+ RESET = '\033[0m'
+ BOLD = '\033[1m'
+
+def print_colored(message: str, color: str = Colors.RESET):
+ """Print colored message to terminal"""
+ print(f"{color}{message}{Colors.RESET}")
+
+def get_provider_from_test_file(test_file: str) -> str:
+ """Map test file names to provider names"""
+ provider_mapping = {
+ 'test_anthropic': 'Anthropic',
+ 'test_azure': 'Azure',
+ 'test_bedrock': 'AWS Bedrock',
+ 'test_openai': 'OpenAI',
+ 'test_vertex': 'Google Vertex AI',
+ 'test_gemini': 'Google Vertex AI',
+ 'test_cohere': 'Cohere',
+ 'test_databricks': 'Databricks',
+ 'test_groq': 'Groq',
+ 'test_together': 'Together AI',
+ 'test_mistral': 'Mistral',
+ 'test_deepseek': 'DeepSeek',
+ 'test_replicate': 'Replicate',
+ 'test_huggingface': 'HuggingFace',
+ 'test_fireworks': 'Fireworks AI',
+ 'test_perplexity': 'Perplexity',
+ 'test_cloudflare': 'Cloudflare',
+ 'test_voyage': 'Voyage AI',
+ 'test_xai': 'xAI',
+ 'test_nvidia': 'NVIDIA',
+ 'test_watsonx': 'IBM watsonx',
+ 'test_azure_ai': 'Azure AI',
+ 'test_snowflake': 'Snowflake',
+ 'test_infinity': 'Infinity',
+ 'test_jina': 'Jina AI',
+ 'test_deepgram': 'Deepgram',
+ 'test_clarifai': 'Clarifai',
+ 'test_triton': 'Triton',
+ }
+
+ for key, provider in provider_mapping.items():
+ if key in test_file:
+ return provider
+
+ # For cross-provider test files
+ if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory',
+ 'test_router', 'test_text_completion']):
+ return f'Cross-Provider Tests ({test_file})'
+
+ return 'Other Tests'
+
+def format_duration(seconds: float) -> str:
+ """Format duration in human-readable format"""
+ if seconds < 60:
+ return f"{seconds:.2f}s"
+ elif seconds < 3600:
+ minutes = int(seconds // 60)
+ secs = seconds % 60
+ return f"{minutes}m {secs:.0f}s"
+ else:
+ hours = int(seconds // 3600)
+ minutes = int((seconds % 3600) // 60)
+ return f"{hours}h {minutes}m"
+
+
+def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
+ """Generate a beautiful markdown report from JUnit XML"""
+ try:
+ tree = ET.parse(junit_xml_path)
+ root = tree.getroot()
+
+ # Handle both testsuite and testsuites root
+ if root.tag == 'testsuites':
+ suites = root.findall('testsuite')
+ else:
+ suites = [root]
+
+ # Overall statistics
+ total_tests = 0
+ total_failures = 0
+ total_errors = 0
+ total_skipped = 0
+ total_time = 0.0
+
+ # Provider breakdown
+ provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
+ provider_tests = defaultdict(list)
+
+ for suite in suites:
+ total_tests += int(suite.get('tests', 0))
+ total_failures += int(suite.get('failures', 0))
+ total_errors += int(suite.get('errors', 0))
+ total_skipped += int(suite.get('skipped', 0))
+ total_time += float(suite.get('time', 0))
+
+ for testcase in suite.findall('testcase'):
+ classname = testcase.get('classname', '')
+ test_name = testcase.get('name', '')
+ test_time = float(testcase.get('time', 0))
+
+ # Extract test file name from classname
+ if '.' in classname:
+ parts = classname.split('.')
+ test_file = parts[-2] if len(parts) > 1 else 'unknown'
+ else:
+ test_file = 'unknown'
+
+ provider = get_provider_from_test_file(test_file)
+ provider_stats[provider]['time'] += test_time
+
+ # Check test status
+ if testcase.find('failure') is not None:
+ provider_stats[provider]['failed'] += 1
+ failure = testcase.find('failure')
+ failure_msg = failure.get('message', '') if failure is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'FAILED',
+ 'time': test_time,
+ 'message': failure_msg
+ })
+ elif testcase.find('error') is not None:
+ provider_stats[provider]['errors'] += 1
+ error = testcase.find('error')
+ error_msg = error.get('message', '') if error is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'ERROR',
+ 'time': test_time,
+ 'message': error_msg
+ })
+ elif testcase.find('skipped') is not None:
+ provider_stats[provider]['skipped'] += 1
+ skip = testcase.find('skipped')
+ skip_msg = skip.get('message', '') if skip is not None else ''
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'SKIPPED',
+ 'time': test_time,
+ 'message': skip_msg
+ })
+ else:
+ provider_stats[provider]['passed'] += 1
+ provider_tests[provider].append({
+ 'name': test_name,
+ 'status': 'PASSED',
+ 'time': test_time,
+ 'message': ''
+ })
+
+ passed = total_tests - total_failures - total_errors - total_skipped
+
+ # Generate the markdown report
+ with open(output_path, 'w') as f:
+ # Header
+ f.write("# LLM Translation Test Results\n\n")
+
+ # Metadata table
+ f.write("## Test Run Information\n\n")
+ f.write("| Field | Value |\n")
+ f.write("|-------|-------|\n")
+ f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
+ f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
+ f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
+ f.write(f"| **Duration** | {format_duration(total_time)} |\n")
+ f.write("\n")
+
+ # Overall statistics with visual elements
+ f.write("## Overall Statistics\n\n")
+
+ # Summary box
+ f.write("```\n")
+ f.write(f"Total Tests: {total_tests}\n")
+ f.write(f"├── Passed: {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"├── Failed: {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"├── Errors: {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+ f.write("```\n\n")
+
+
+ # Provider summary table
+ f.write("## Results by Provider\n\n")
+ f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
+ f.write("|----------|-------|------|------|-------|------|-----------|----------|")
+
+ # Sort providers: specific providers first, then cross-provider tests
+ sorted_providers = []
+ cross_provider = []
+ for p in sorted(provider_stats.keys()):
+ if 'Cross-Provider' in p or p == 'Other Tests':
+ cross_provider.append(p)
+ else:
+ sorted_providers.append(p)
+
+ all_providers = sorted_providers + cross_provider
+
+ for provider in all_providers:
+ stats = provider_stats[provider]
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+ pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
+
+ f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
+ f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
+ f.write(f"{format_duration(stats['time'])} |")
+
+ # Detailed test results by provider
+ f.write("\n\n## Detailed Test Results\n\n")
+
+ for provider in sorted_providers:
+ if provider_tests[provider]:
+ stats = provider_stats[provider]
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+
+ f.write(f"### {provider}\n\n")
+ f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+ f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
+ f.write(f"in {format_duration(stats['time'])}\n\n")
+
+ # Group tests by status
+ tests_by_status = defaultdict(list)
+ for test in provider_tests[provider]:
+ tests_by_status[test['status']].append(test)
+
+ # Show failed tests first (if any)
+ if tests_by_status['FAILED']:
+ f.write("\nFailed Tests\n\n")
+ for test in tests_by_status['FAILED']:
+ f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+ if test['message']:
+ # Truncate long error messages
+ msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
+ f.write(f" > {msg}\n")
+ f.write("\n\n\n")
+
+ # Show errors (if any)
+ if tests_by_status['ERROR']:
+ f.write("\nError Tests\n\n")
+ for test in tests_by_status['ERROR']:
+ f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+ f.write("\n\n\n")
+
+ # Show passed tests in collapsible section
+ if tests_by_status['PASSED']:
+ f.write("\nPassed Tests\n\n")
+ for test in tests_by_status['PASSED']:
+ f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+ f.write("\n\n\n")
+
+ # Show skipped tests (if any)
+ if tests_by_status['SKIPPED']:
+ f.write("\nSkipped Tests\n\n")
+ for test in tests_by_status['SKIPPED']:
+ f.write(f"- `{test['name']}`\n")
+ f.write("\n\n\n")
+
+ # Cross-provider tests in a separate section
+ if cross_provider:
+ f.write("### Cross-Provider Tests\n\n")
+ for provider in cross_provider:
+ if provider_tests[provider]:
+ stats = provider_stats[provider]
+ total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+
+ f.write(f"#### {provider}\n\n")
+ f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+ f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
+
+ # For cross-provider tests, just show counts
+ f.write(f"- Passed: {stats['passed']}\n")
+ if stats['failed'] > 0:
+ f.write(f"- Failed: {stats['failed']}\n")
+ if stats['errors'] > 0:
+ f.write(f"- Errors: {stats['errors']}\n")
+ if stats['skipped'] > 0:
+ f.write(f"- Skipped: {stats['skipped']}\n")
+ f.write("\n")
+
+
+ print_colored(f"Report generated: {output_path}", Colors.GREEN)
+
+ except Exception as e:
+ print_colored(f"Error generating report: {e}", Colors.RED)
+ raise
+
+def run_tests(test_path: str = "tests/llm_translation/",
+ junit_xml: str = "test-results/junit.xml",
+ report_path: str = "test-results/llm_translation_report.md",
+ tag: str = None,
+ commit: str = None) -> int:
+ """Run the LLM translation tests and generate report"""
+
+ # Create test results directory
+ os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
+
+ print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
+ print_colored(f"Test directory: {test_path}", Colors.CYAN)
+ print_colored(f"Output: {junit_xml}", Colors.CYAN)
+ print()
+
+ # Run pytest
+ cmd = [
+ "poetry", "run", "pytest", test_path,
+ f"--junitxml={junit_xml}",
+ "-v",
+ "--tb=short",
+ "--maxfail=500",
+ "-n", "auto"
+ ]
+
+ # Add timeout if pytest-timeout is installed
+ try:
+ subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"],
+ capture_output=True, check=True)
+ cmd.extend(["--timeout=300"])
+ except:
+ print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
+
+ print_colored("Running pytest with command:", Colors.YELLOW)
+ print(f" {' '.join(cmd)}")
+ print()
+
+ # Run the tests
+ result = subprocess.run(cmd, capture_output=False)
+
+ # Generate the report regardless of test outcome
+ if os.path.exists(junit_xml):
+ print()
+ print_colored("Generating test report...", Colors.BLUE)
+ generate_markdown_report(junit_xml, report_path, tag, commit)
+
+ # Print summary to console
+ print()
+ print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
+
+ # Parse XML for quick summary
+ tree = ET.parse(junit_xml)
+ root = tree.getroot()
+
+ if root.tag == 'testsuites':
+ suites = root.findall('testsuite')
+ else:
+ suites = [root]
+
+ total = sum(int(s.get('tests', 0)) for s in suites)
+ failures = sum(int(s.get('failures', 0)) for s in suites)
+ errors = sum(int(s.get('errors', 0)) for s in suites)
+ skipped = sum(int(s.get('skipped', 0)) for s in suites)
+ passed = total - failures - errors - skipped
+
+ print(f" Total: {total}")
+ print_colored(f" Passed: {passed}", Colors.GREEN)
+ if failures > 0:
+ print_colored(f" Failed: {failures}", Colors.RED)
+ if errors > 0:
+ print_colored(f" Errors: {errors}", Colors.RED)
+ if skipped > 0:
+ print_colored(f" Skipped: {skipped}", Colors.YELLOW)
+
+ if total > 0:
+ pass_rate = (passed / total) * 100
+ color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
+ print_colored(f" Pass Rate: {pass_rate:.1f}%", color)
+ else:
+ print_colored("No test results found!", Colors.RED)
+
+ print()
+ print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
+
+ return result.returncode
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
+ parser.add_argument("--test-path", default="tests/llm_translation/",
+ help="Path to test directory")
+ parser.add_argument("--junit-xml", default="test-results/junit.xml",
+ help="Path for JUnit XML output")
+ parser.add_argument("--report", default="test-results/llm_translation_report.md",
+ help="Path for markdown report")
+ parser.add_argument("--tag", help="Git tag or version")
+ parser.add_argument("--commit", help="Git commit SHA")
+
+ args = parser.parse_args()
+
+ # Get git info if not provided
+ if not args.commit:
+ try:
+ result = subprocess.run(["git", "rev-parse", "HEAD"],
+ capture_output=True, text=True)
+ if result.returncode == 0:
+ args.commit = result.stdout.strip()
+ except:
+ pass
+
+ if not args.tag:
+ try:
+ result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"],
+ capture_output=True, text=True)
+ if result.returncode == 0:
+ args.tag = result.stdout.strip()
+ except:
+ pass
+
+ exit_code = run_tests(
+ test_path=args.test_path,
+ junit_xml=args.junit_xml,
+ report_path=args.report,
+ tag=args.tag,
+ commit=args.commit
+ )
+
+ sys.exit(exit_code)
\ No newline at end of file
diff --git a/.github/workflows/simple_pypi_publish.yml b/.github/workflows/simple_pypi_publish.yml
new file mode 100644
index 00000000000..e1830556819
--- /dev/null
+++ b/.github/workflows/simple_pypi_publish.yml
@@ -0,0 +1,67 @@
+name: Simple PyPI Publish
+
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ description: 'Version to publish (e.g., 1.74.10)'
+ required: true
+ type: string
+
+env:
+ TWINE_USERNAME: __token__
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+ if: github.repository == 'BerriAI/litellm'
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.8'
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install toml build wheel twine
+
+ - name: Update version in pyproject.toml
+ run: |
+ python -c "
+ import toml
+
+ with open('pyproject.toml', 'r') as f:
+ data = toml.load(f)
+
+ data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
+
+ with open('pyproject.toml', 'w') as f:
+ toml.dump(data, f)
+
+ print(f'Updated version to ${{ github.event.inputs.version }}')
+ "
+
+ - name: Copy model prices file
+ run: |
+ cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
+
+ - name: Build package
+ run: |
+ rm -rf build dist
+ python -m build
+
+ - name: Publish to PyPI
+ env:
+ TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+ run: |
+ twine upload dist/*
+
+ - name: Output success
+ run: |
+ echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
+ echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/"
\ No newline at end of file
diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml
index ceeedbe7e13..9638c00e453 100644
--- a/.github/workflows/test-linting.yml
+++ b/.github/workflows/test-linting.yml
@@ -11,6 +11,9 @@ jobs:
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ clean: true
- name: Set up Python
uses: actions/setup-python@v4
@@ -20,13 +23,15 @@ jobs:
- name: Install Poetry
uses: snok/install-poetry@v1
+ - name: Clean Python cache
+ run: |
+ find . -type d -name "__pycache__" -exec rm -rf {} + || true
+ find . -name "*.pyc" -delete || true
+
- name: Install dependencies
run: |
- pip install openai==1.81.0
poetry install --with dev
- pip install openai==1.81.0
-
-
+ poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@@ -34,16 +39,29 @@ jobs:
poetry run black .
cd ..
+ - name: Debug - Check file state
+ run: |
+ echo "Current branch:"
+ git branch --show-current
+ echo "Last 3 commits:"
+ git log --oneline -3
+ echo "File content around line 43:"
+ head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
+
- name: Run Ruff linting
run: |
cd litellm
poetry run ruff check .
cd ..
+ - name: Print OpenAI version
+ run: |
+ poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+
- name: Run MyPy type checking
run: |
cd litellm
- poetry run mypy . --ignore-missing-imports
+ poetry run mypy .
cd ..
- name: Check for circular imports
diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml
index 66471e07320..b7f4a25d593 100644
--- a/.github/workflows/test-litellm.yml
+++ b/.github/workflows/test-litellm.yml
@@ -7,7 +7,7 @@ on:
jobs:
test:
runs-on: ubuntu-latest
- timeout-minutes: 15
+ timeout-minutes: 25
steps:
- uses: actions/checkout@v4
@@ -27,9 +27,12 @@ jobs:
- name: Install dependencies
run: |
- poetry install --with dev,proxy-dev --extras proxy
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
+ poetry run pip install "google-genai==1.22.0"
+ poetry run pip install "google-cloud-aiplatform>=1.38"
+ poetry run pip install "fastapi-offline==1.7.3"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
@@ -37,4 +40,4 @@ jobs:
cd ..
- name: Run tests
run: |
- poetry run pytest tests/test_litellm -x -vv -n 4
+ poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4
diff --git a/.github/workflows/test-mcp.yml b/.github/workflows/test-mcp.yml
new file mode 100644
index 00000000000..2da6980951a
--- /dev/null
+++ b/.github/workflows/test-mcp.yml
@@ -0,0 +1,48 @@
+name: LiteLLM MCP Tests (folder - tests/mcp_tests)
+
+on:
+ pull_request:
+ branches: [ main ]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 25
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Thank You Message
+ run: |
+ echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+ echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.12'
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+
+ - name: Install dependencies
+ run: |
+ poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+ poetry run pip install "pytest==7.3.1"
+ poetry run pip install "pytest-retry==1.6.3"
+ poetry run pip install "pytest-cov==5.0.0"
+ poetry run pip install "pytest-asyncio==0.21.1"
+ poetry run pip install "respx==0.22.0"
+ poetry run pip install "pydantic==2.10.2"
+ poetry run pip install "mcp==1.10.1"
+ poetry run pip install pytest-xdist
+
+ - name: Setup litellm-enterprise as local package
+ run: |
+ cd enterprise
+ python -m pip install -e .
+ cd ..
+
+ - name: Run MCP tests
+ run: |
+ poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
diff --git a/.gitignore b/.gitignore
index 93134dabbf4..c2ac5137cbe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,7 +86,14 @@ litellm/proxy/db/migrations/0_init/migration.sql
litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
+litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
test.py
+
+litellm_config.yaml
+.cursor
+.vscode/launch.json
+litellm/proxy/to_delete_loadtest_work/*
+update_model_cost_map.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index dd98498e3be..9396f323e45 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,17 +14,17 @@ repos:
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- - id: black
- name: black
- entry: poetry run black
- language: system
- types: [python]
- files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+ # - id: black
+ # name: black
+ # entry: poetry run black
+ # language: system
+ # types: [python]
+ # files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
- exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/
+ exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000000..8e7b5f2bd2e
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,144 @@
+# INSTRUCTIONS FOR LITELLM
+
+This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
+
+## OVERVIEW
+
+LiteLLM is a unified interface for 100+ LLMs that:
+- Translates inputs to provider-specific completion, embedding, and image generation endpoints
+- Provides consistent OpenAI-format output across all providers
+- Includes retry/fallback logic across multiple deployments (Router)
+- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
+- Supports advanced features like function calling, streaming, caching, and observability
+
+## REPOSITORY STRUCTURE
+
+### Core Components
+- `litellm/` - Main library code
+ - `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
+ - `proxy/` - Proxy server implementation (LLM Gateway)
+ - `router_utils/` - Load balancing and fallback logic
+ - `types/` - Type definitions and schemas
+ - `integrations/` - Third-party integrations (observability, caching, etc.)
+
+### Key Directories
+- `tests/` - Comprehensive test suites
+- `docs/my-website/` - Documentation website
+- `ui/litellm-dashboard/` - Admin dashboard UI
+- `enterprise/` - Enterprise-specific features
+
+## DEVELOPMENT GUIDELINES
+
+### MAKING CODE CHANGES
+
+1. **Provider Implementations**: When adding/modifying LLM providers:
+ - Follow existing patterns in `litellm/llms/{provider}/`
+ - Implement proper transformation classes that inherit from `BaseConfig`
+ - Support both sync and async operations
+ - Handle streaming responses appropriately
+ - Include proper error handling with provider-specific exceptions
+
+2. **Type Safety**:
+ - Use proper type hints throughout
+ - Update type definitions in `litellm/types/`
+ - Ensure compatibility with both Pydantic v1 and v2
+
+3. **Testing**:
+ - Add tests in appropriate `tests/` subdirectories
+ - Include both unit tests and integration tests
+ - Test provider-specific functionality thoroughly
+ - Consider adding load tests for performance-critical changes
+
+### IMPORTANT PATTERNS
+
+1. **Function/Tool Calling**:
+ - LiteLLM standardizes tool calling across providers
+ - OpenAI format is the standard, with transformations for other providers
+ - See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
+
+2. **Streaming**:
+ - All providers should support streaming where possible
+ - Use consistent chunk formatting across providers
+ - Handle both sync and async streaming
+
+3. **Error Handling**:
+ - Use provider-specific exception classes
+ - Maintain consistent error formats across providers
+ - Include proper retry logic and fallback mechanisms
+
+4. **Configuration**:
+ - Support both environment variables and programmatic configuration
+ - Use `BaseConfig` classes for provider configurations
+ - Allow dynamic parameter passing
+
+## PROXY SERVER (LLM GATEWAY)
+
+The proxy server is a critical component that provides:
+- Authentication and authorization
+- Rate limiting and budget management
+- Load balancing across multiple models/deployments
+- Observability and logging
+- Admin dashboard UI
+- Enterprise features
+
+Key files:
+- `litellm/proxy/proxy_server.py` - Main server implementation
+- `litellm/proxy/auth/` - Authentication logic
+- `litellm/proxy/management_endpoints/` - Admin API endpoints
+
+## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
+
+LiteLLM supports MCP for agent workflows:
+- MCP server integration for tool calling
+- Transformation between OpenAI and MCP tool formats
+- Support for external MCP servers (Zapier, Jira, Linear, etc.)
+- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
+
+## TESTING CONSIDERATIONS
+
+1. **Provider Tests**: Test against real provider APIs when possible
+2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
+3. **Performance Tests**: Load testing for high-throughput scenarios
+4. **Integration Tests**: End-to-end workflows including tool calling
+
+## DOCUMENTATION
+
+- Keep documentation in sync with code changes
+- Update provider documentation when adding new providers
+- Include code examples for new features
+- Update changelog and release notes
+
+## SECURITY CONSIDERATIONS
+
+- Handle API keys securely
+- Validate all inputs, especially for proxy endpoints
+- Consider rate limiting and abuse prevention
+- Follow security best practices for authentication
+
+## ENTERPRISE FEATURES
+
+- Some features are enterprise-only
+- Check `enterprise/` directory for enterprise-specific code
+- Maintain compatibility between open-source and enterprise versions
+
+## COMMON PITFALLS TO AVOID
+
+1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
+2. **Provider Specifics**: Each provider has unique quirks - handle them properly
+3. **Rate Limits**: Respect provider rate limits in tests
+4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
+5. **Dependencies**: Keep dependencies minimal and well-justified
+
+## HELPFUL RESOURCES
+
+- Main documentation: https://docs.litellm.ai/
+- Provider-specific docs in `docs/my-website/docs/providers/`
+- Admin UI for testing proxy features
+
+## WHEN IN DOUBT
+
+- Follow existing patterns in the codebase
+- Check similar provider implementations
+- Ensure comprehensive test coverage
+- Update documentation appropriately
+- Consider backward compatibility impact
\ No newline at end of file
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 00000000000..50bed6e43e2
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,89 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..ad58a4976d6
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,275 @@
+# Contributing to LiteLLM
+
+Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
+
+## **Checklist before submitting a PR**
+
+Here are the core requirements for any PR submitted to LiteLLM:
+
+- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
+- [ ] **Ensure your PR passes all checks**:
+ - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
+ - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+## **Contributor License Agreement (CLA)**
+
+Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
+
+**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
+
+## Quick Start
+
+### 1. Setup Your Local Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+
+# Create a new branch for your feature
+git checkout -b your-feature-branch
+
+# Install development dependencies
+make install-dev
+
+# Verify your setup works
+make help
+```
+
+That's it! Your local development environment is ready.
+
+### 2. Development Workflow
+
+Here's the recommended workflow for making changes:
+
+```bash
+# Make your changes to the code
+# ...
+
+# Format your code (auto-fixes formatting issues)
+make format
+
+# Run all linting checks (matches CI exactly)
+make lint
+
+# Run unit tests to ensure nothing is broken
+make test-unit
+
+# Commit your changes
+git add .
+git commit -m "Your descriptive commit message"
+
+# Push and create a PR
+git push origin your-feature-branch
+```
+
+## Adding Testing
+
+**Adding at least 1 test is a hard requirement for all PRs.**
+
+### Where to Add Tests
+
+Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+
+- This directory mirrors the structure of the `litellm/` directory
+- **Only add mocked tests** - no real LLM API calls in this directory
+- For integration tests with real APIs, use the appropriate test directories
+
+### File Naming Convention
+
+The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+
+- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
+- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+
+### Example Test
+
+```python
+import pytest
+from litellm import completion
+
+def test_your_feature():
+ """Test your feature with a descriptive docstring."""
+ # Arrange
+ messages = [{"role": "user", "content": "Hello"}]
+
+ # Act
+ # Use mocked responses, not real API calls
+
+ # Assert
+ assert expected_result == actual_result
+```
+
+## Running Tests and Checks
+
+### Running Unit Tests
+
+Run all unit tests (uses parallel execution for speed):
+
+```bash
+make test-unit
+```
+
+Run specific test files:
+```bash
+poetry run pytest tests/test_litellm/test_your_file.py -v
+```
+
+### Running Linting and Formatting Checks
+
+Run all linting checks (matches CI exactly):
+
+```bash
+make lint
+```
+
+Individual linting commands:
+```bash
+make format-check # Check Black formatting
+make lint-ruff # Run Ruff linting
+make lint-mypy # Run MyPy type checking
+make check-circular-imports # Check for circular imports
+make check-import-safety # Check import safety
+```
+
+Apply formatting (auto-fixes issues):
+```bash
+make format
+```
+
+### CI Compatibility
+
+To ensure your changes will pass CI, run the exact same checks locally:
+
+```bash
+# This runs the same checks as the GitHub workflows
+make lint
+make test-unit
+```
+
+For exact CI compatibility (pins OpenAI version like CI):
+```bash
+make install-dev-ci # Installs exact CI dependencies
+```
+
+## Available Make Commands
+
+Run `make help` to see all available commands:
+
+```bash
+make help # Show all available commands
+make install-dev # Install development dependencies
+make install-proxy-dev # Install proxy development dependencies
+make install-test-deps # Install test dependencies (for running tests)
+make format # Apply Black code formatting
+make format-check # Check Black formatting (matches CI)
+make lint # Run all linting checks
+make test-unit # Run unit tests
+make test-integration # Run integration tests
+make test-unit-helm # Run Helm unit tests
+```
+
+## Code Quality Standards
+
+LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
+
+Our automated quality checks include:
+- **Black** for consistent code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for static type checking
+- **Circular import detection**
+- **Import safety validation**
+
+All checks must pass before your PR can be merged.
+
+## Common Issues and Solutions
+
+### 1. Linting Failures
+
+If `make lint` fails:
+
+1. **Formatting issues**: Run `make format` to auto-fix
+2. **Ruff issues**: Check the output and fix manually
+3. **MyPy issues**: Add proper type hints
+4. **Circular imports**: Refactor import dependencies
+5. **Import safety**: Fix any unprotected imports
+
+### 2. Test Failures
+
+If `make test-unit` fails:
+
+1. Check if you broke existing functionality
+2. Add tests for your new code
+3. Ensure tests use mocks, not real API calls
+4. Check test file naming conventions
+
+### 3. Common Development Tips
+
+- **Use type hints**: MyPy requires proper type annotations
+- **Write descriptive commit messages**: Help reviewers understand your changes
+- **Keep PRs focused**: One feature/fix per PR
+- **Test edge cases**: Don't just test the happy path
+- **Update documentation**: If you change APIs, update docs
+
+## Building and Running Locally
+
+### LiteLLM Proxy Server
+
+To run the proxy server locally:
+
+```bash
+# Install proxy dependencies
+make install-proxy-dev
+
+# Start the proxy server
+poetry run litellm --config your_config.yaml
+```
+
+### Docker Development
+
+If you want to build the Docker image yourself:
+
+```bash
+# Build using the non-root Dockerfile
+docker build -f docker/Dockerfile.non_root -t litellm_dev .
+
+# Run with your config
+docker run \
+ -v $(pwd)/proxy_config.yaml:/app/config.yaml \
+ -e LITELLM_MASTER_KEY="sk-1234" \
+ -p 4000:4000 \
+ litellm_dev \
+ --config /app/config.yaml --detailed_debug
+```
+
+## Submitting Your PR
+
+1. **Push your branch**: `git push origin your-feature-branch`
+2. **Create a PR**: Go to GitHub and create a pull request
+3. **Fill out the PR template**: Provide clear description of changes
+4. **Wait for review**: Maintainers will review and provide feedback
+5. **Address feedback**: Make requested changes and push updates
+6. **Merge**: Once approved, your PR will be merged!
+
+## Getting Help
+
+If you need help:
+
+- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
+- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
+- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
+
+## What to Contribute
+
+Looking for ideas? Check out:
+
+- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
+- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
+- 📚 Documentation improvements
+- 🧪 Test coverage improvements
+- 🔌 New LLM provider integrations
+
+Thank you for contributing to LiteLLM! 🚀
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index b972aab0961..6ab78d85e33 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@ USER root
RUN apk add --no-cache gcc python3-dev openssl openssl-dev
-RUN pip install --upgrade pip && \
+RUN pip install --upgrade pip>=24.3.1 && \
pip install build
# Copy the current directory contents into the container at /app
@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -53,6 +50,9 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache openssl tzdata
+# Upgrade pip to fix CVE-2025-8869
+RUN pip install --upgrade pip>=24.3.1
+
WORKDIR /app
# Copy the current directory contents into the container at /app
COPY . .
@@ -65,6 +65,9 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
# Generate prisma client
RUN prisma generate
RUN chmod +x docker/entrypoint.sh
@@ -72,6 +75,9 @@ RUN chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
diff --git a/GEMINI.md b/GEMINI.md
new file mode 100644
index 00000000000..efcee04d4c3
--- /dev/null
+++ b/GEMINI.md
@@ -0,0 +1,89 @@
+# GEMINI.md
+
+This file provides guidance to Gemini when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features
\ No newline at end of file
diff --git a/MCP_SSL_CHANGES_SUMMARY.md b/MCP_SSL_CHANGES_SUMMARY.md
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/Makefile b/Makefile
index b25c95cd8ff..159fe4fa2ef 100644
--- a/Makefile
+++ b/Makefile
@@ -1,35 +1,103 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
-.PHONY: help test test-unit test-integration lint format
+.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@echo "Available commands:"
+ @echo " make install-dev - Install development dependencies"
+ @echo " make install-proxy-dev - Install proxy development dependencies"
+ @echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
+ @echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
+ @echo " make install-test-deps - Install test dependencies"
+ @echo " make install-helm-unittest - Install helm unittest plugin"
+ @echo " make format - Apply Black code formatting"
+ @echo " make format-check - Check Black code formatting (matches CI)"
+ @echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
+ @echo " make lint-ruff - Run Ruff linting only"
+ @echo " make lint-mypy - Run MyPy type checking only"
+ @echo " make lint-black - Check Black formatting (matches CI)"
+ @echo " make check-circular-imports - Check for circular imports"
+ @echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
- @echo " make test-unit - Run unit tests"
+ @echo " make test-unit - Run unit tests (tests/test_litellm)"
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
+# Installation targets
install-dev:
poetry install --with dev
install-proxy-dev:
- poetry install --with dev,proxy-dev
+ poetry install --with dev,proxy-dev --extras proxy
-lint: install-dev
+# CI-compatible installations (matches GitHub workflows exactly)
+install-dev-ci:
+ pip install openai==1.99.5
+ poetry install --with dev
+ pip install openai==1.99.5
+
+install-proxy-dev-ci:
+ poetry install --with dev,proxy-dev --extras proxy
+ pip install openai==1.99.5
+
+install-test-deps: install-proxy-dev
+ poetry run pip install "pytest-retry==1.6.3"
+ poetry run pip install pytest-xdist
+ cd enterprise && python -m pip install -e . && cd ..
+
+install-helm-unittest:
+ helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
+
+# Formatting
+format: install-dev
+ cd litellm && poetry run black . && cd ..
+
+format-check: install-dev
+ cd litellm && poetry run black --check . && cd ..
+
+# Linting targets
+lint-ruff: install-dev
+ cd litellm && poetry run ruff check . && cd ..
+
+lint-mypy: install-dev
poetry run pip install types-requests types-setuptools types-redis types-PyYAML
- cd litellm && poetry run mypy . --ignore-missing-imports
+ cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
-# Testing
+lint-black: format-check
+
+check-circular-imports: install-dev
+ cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
+
+check-import-safety: install-dev
+ poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+# Combined linting (matches test-linting.yml workflow)
+lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
+
+# Testing targets
test:
poetry run pytest tests/
-test-unit:
- poetry run pytest tests/test_litellm/
+test-unit: install-test-deps
+ poetry run pytest tests/test_litellm -x -vv -n 4
test-integration:
poetry run pytest tests/ -k "not test_litellm"
-test-unit-helm:
- helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
\ No newline at end of file
+test-unit-helm: install-helm-unittest
+ helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+
+# LLM Translation testing targets
+test-llm-translation: install-test-deps
+ @echo "Running LLM translation tests..."
+ @python .github/workflows/run_llm_translation_tests.py
+
+test-llm-translation-single: install-test-deps
+ @echo "Running single LLM translation test file..."
+ @if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
+ @mkdir -p test-results
+ poetry run pytest tests/llm_translation/$(FILE) \
+ --junitxml=test-results/junit.xml \
+ -v --tb=short --maxfail=100 --timeout=300
\ No newline at end of file
diff --git a/README.md b/README.md
index cbb6b4807bd..c785ee82ffa 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,9 @@
+
+
+
LiteLLM manages:
@@ -34,7 +37,7 @@ LiteLLM manages:
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
@@ -44,7 +47,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
> [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
+> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
@@ -69,7 +72,7 @@ messages = [{ "content": "Hello, how are you?","role": "user"}]
response = completion(model="openai/gpt-4o", messages=messages)
# anthropic call
-response = completion(model="anthropic/claude-3-sonnet-20240229", messages=messages)
+response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
print(response)
```
@@ -77,9 +80,9 @@ print(response)
```json
{
- "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
- "created": 1734366691,
- "model": "claude-3-sonnet-20240229",
+ "id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
+ "created": 1751494488,
+ "model": "claude-sonnet-4-20250514",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
@@ -87,7 +90,7 @@ print(response)
"finish_reason": "stop",
"index": 0,
"message": {
- "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
+ "content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
"role": "assistant",
"tool_calls": null,
"function_call": null
@@ -95,9 +98,9 @@ print(response)
}
],
"usage": {
- "completion_tokens": 43,
+ "completion_tokens": 39,
"prompt_tokens": 13,
- "total_tokens": 56,
+ "total_tokens": 52,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
@@ -129,7 +132,7 @@ print(response)
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
-liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
+liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
@@ -138,8 +141,8 @@ response = completion(model="openai/gpt-4o", messages=messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
-# claude 2
-response = completion('anthropic/claude-3-sonnet-20240229', messages, stream=True)
+# claude sonnet 4
+response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
for part in response:
print(part)
```
@@ -148,9 +151,9 @@ for part in response:
```json
{
- "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
- "created": 1734366925,
- "model": "claude-3-sonnet-20240229",
+ "id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
+ "created": 1751494808,
+ "model": "claude-sonnet-4-20250514",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"choices": [
@@ -158,6 +161,7 @@ for part in response:
"finish_reason": null,
"index": 0,
"delta": {
+ "provider_specific_fields": null,
"content": "Hello",
"role": "assistant",
"function_call": null,
@@ -166,7 +170,10 @@ for part in response:
},
"logprobs": null
}
- ]
+ ],
+ "provider_specific_fields": null,
+ "stream_options": null,
+ "citations": null
}
```
@@ -227,7 +234,7 @@ $ litellm --model huggingface/bigcode/starcoder
> [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
+> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
```python
import openai # openai v1.0.0+
@@ -259,14 +266,14 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/
+# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' > .env
+echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
-docker-compose up
+docker compose up
```
@@ -309,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
+| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | ✅ | | |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
@@ -333,22 +341,37 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | |
+| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | | | | |
| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | |
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
+| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | |
+| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and contributing LLM integrations are both accepted and highly encouraged! [See our Contribution Guide for more details](https://docs.litellm.ai/docs/extras/contributing_code)
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
+
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-This covers:
+This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
@@ -356,24 +379,46 @@ This covers:
- ✅ **Custom SLAs**
- ✅ **Secure access with Single Sign-On**
-# Code Quality / Linting
+# Contributing
+
+We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
+
+## Quick Start for Contributors
+
+This requires poetry to be installed.
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+make install-dev # Install development dependencies
+make format # Format your code
+make lint # Run all linting checks
+make test-unit # Run unit tests
+make format-check # Check formatting only
+```
+
+For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Code Quality / Linting
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
-We run:
-- Ruff for [formatting and linting checks](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L320)
-- Mypy + Pyright for typing [1](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L90), [2](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L4)
-- Black for [formatting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L79)
-- isort for [import sorting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L10)
+Our automated checks include:
+- **Black** for code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for type checking
+- **Circular import detection**
+- **Import safety checks**
-If you have suggestions on how to improve the code quality feel free to open an issue or a PR.
+All these checks must pass before your PR can be merged.
# Support / talk with founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
+- [Community Slack 💭](https://www.litellm.ai/support)
- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
@@ -397,18 +442,3 @@ If you have suggestions on how to improve the code quality feel free to open an
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard
diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh
new file mode 100755
index 00000000000..4255885bcbb
--- /dev/null
+++ b/ci_cd/security_scans.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Security Scans Script for LiteLLM
+# This script runs comprehensive security scans including Trivy and Grype
+
+set -e
+
+echo "Starting security scans for LiteLLM..."
+
+# Function to install Trivy and required tools
+install_trivy() {
+ echo "Installing Trivy and required tools..."
+ sudo apt-get update
+ sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+ wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+ echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+ sudo apt-get update
+ sudo apt-get install trivy
+ echo "Trivy and required tools installed successfully"
+}
+
+# Function to install Grype
+install_grype() {
+ echo "Installing Grype..."
+ curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
+ echo "Grype installed successfully"
+}
+
+# Function to run Trivy scans
+run_trivy_scans() {
+ echo "Running Trivy scans..."
+
+ echo "Scanning LiteLLM Docs..."
+ trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+
+ echo "Scanning LiteLLM UI..."
+ trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+
+ echo "Trivy scans completed successfully"
+}
+
+# Function to build and scan Docker images with Grype
+run_grype_scans() {
+ echo "Running Grype scans..."
+
+ # Temporarily add wheel files to .dockerignore for security scans
+ echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
+ cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
+ echo "/*.whl" >> .dockerignore
+
+ # Build and scan Dockerfile.database
+ echo "Building and scanning Dockerfile.database..."
+ docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
+ grype litellm-database:latest --fail-on critical
+
+ # Build and scan main Dockerfile
+ echo "Building and scanning main Dockerfile..."
+ docker build --no-cache -t litellm:latest .
+ grype litellm:latest --fail-on critical
+
+ # Restore original .dockerignore
+ echo "Restoring original .dockerignore..."
+ mv .dockerignore.backup .dockerignore
+
+ # Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
+ echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
+ echo "Using locally built image: litellm:latest"
+
+ # Allowlist of CVEs to be ignored in failure threshold/reporting
+ # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
+ ALLOWED_CVES=(
+ "CVE-2025-8869"
+ )
+
+ # Build JSON array of allowlisted CVE IDs for jq
+ ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
+
+ echo "Checking for vulnerabilities with CVSS score >= 4.0..."
+ echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
+
+ HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+ .matches[]
+ | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+ | select((.vulnerability.id as $id | $allow | index($id) | not))
+ | .vulnerability.id' | wc -l)
+
+ if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
+ echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
+ echo "Detailed vulnerability report:"
+ grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+ ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
+ (.matches[]
+ | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+ | select((.vulnerability.id as $id | $allow | index($id) | not))
+ | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
+ | @tsv' | column -t -s $'\t'
+ exit 1
+ else
+ echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
+ fi
+
+ echo "Grype scans completed successfully"
+}
+
+# Main execution
+main() {
+ echo "Installing security scanning tools..."
+ install_trivy
+ install_grype
+
+ echo "Running filesystem vulnerability scans..."
+ run_trivy_scans
+
+ echo "Running Docker image vulnerability scans..."
+ run_grype_scans
+
+ echo "All security scans completed successfully!"
+}
+
+# Execute main function
+main "$@"
diff --git a/ci_cd/security_scans_readme.md b/ci_cd/security_scans_readme.md
new file mode 100644
index 00000000000..dd64b01c296
--- /dev/null
+++ b/ci_cd/security_scans_readme.md
@@ -0,0 +1,9 @@
+# Security Scans
+
+## Scans that run:
+
+- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
+- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM)
+- Grype scan on `Dockerfile.database` (fails on CRITICAL)
+- Grype scan on main `Dockerfile` (fails on CRITICAL)
+- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)
diff --git a/cookbook/liteLLM_Baseten.ipynb b/cookbook/liteLLM_Baseten.ipynb
index e03bb3254a5..0a5bc5f1df7 100644
--- a/cookbook/liteLLM_Baseten.ipynb
+++ b/cookbook/liteLLM_Baseten.ipynb
@@ -6,19 +6,21 @@
"id": "gZx-wHJapG5w"
},
"source": [
- "# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
+ "# LiteLLM with Baseten Model APIs\n",
"\n",
- "* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
- "* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
- "* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
+ "This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
- "\n",
- "## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
- "Example call\n",
+ "## Example Usage\n",
"```python\n",
- "model = \"q841o8w\" # baseten model version ID\n",
- "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
- "```"
+ "response = completion(\n",
+ " model=\"baseten/openai/gpt-oss-120b\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
+ " max_tokens=1000,\n",
+ " temperature=0.7\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "## Setup"
]
},
{
@@ -29,20 +31,25 @@
},
"outputs": [],
"source": [
- "!pip install litellm==0.1.399\n",
- "!pip install baseten urllib3"
+ "%pip install litellm"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
- "from litellm import completion"
+ "from litellm import completion\n",
+ "\n",
+ "# Set your Baseten API key\n",
+ "os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
+ "\n",
+ "# Test message\n",
+ "messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
@@ -51,19 +58,31 @@
"id": "4STYM2OHFNlc"
},
"source": [
- "## Setup"
+ "## Example 1: Basic Completion\n",
+ "\n",
+ "Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
- "os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
- "messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
+ "print(\"=== Basic Completion ===\")\n",
+ "response = completion(\n",
+ " model=\"baseten/openai/gpt-oss-120b\",\n",
+ " messages=messages,\n",
+ " max_tokens=1000,\n",
+ " temperature=0.7,\n",
+ " top_p=0.9,\n",
+ " presence_penalty=0.1,\n",
+ " frequency_penalty=0.1,\n",
+ ")\n",
+ "print(f\"Response: {response.choices[0].message.content}\")\n",
+ "print(f\"Usage: {response.usage}\")"
]
},
{
@@ -72,13 +91,14 @@
"id": "syF3dTdKFSQQ"
},
"source": [
- "## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
- "### Pass Your Baseten model `Version ID` as `model`"
+ "## Example 2: Streaming Completion\n",
+ "\n",
+ "Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -86,137 +106,26 @@
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\u001b[32mINFO\u001b[0m API key set.\n",
- "INFO:baseten:API key set.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "{'choices': [{'finish_reason': 'stop',\n",
- " 'index': 0,\n",
- " 'message': {'role': 'assistant',\n",
- " 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
- " 'created': 1692135883.699066,\n",
- " 'model': 'qvv0xeq'}"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "model = \"qvv0xeq\"\n",
- "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
- "response"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "7n21UroEGCGa"
- },
- "source": [
- "## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
- "### Pass Your Baseten model `Version ID` as `model`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 19,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "uLVWFH899lAF",
- "outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\u001b[32mINFO\u001b[0m API key set.\n",
- "INFO:baseten:API key set.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "{'choices': [{'finish_reason': 'stop',\n",
- " 'index': 0,\n",
- " 'message': {'role': 'assistant',\n",
- " 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
- " 'created': 1692135900.2806294,\n",
- " 'model': 'q841o8w'}"
- ]
- },
- "execution_count": 19,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "model = \"q841o8w\"\n",
- "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
- "response"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "6-TFwmPAGPXq"
- },
- "source": [
- "## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
- "### Pass Your Baseten model `Version ID` as `model`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "gbeYZOrUE_Bp",
- "outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\u001b[32mINFO\u001b[0m API key set.\n",
- "INFO:baseten:API key set.\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "{'choices': [{'finish_reason': 'stop',\n",
- " 'index': 0,\n",
- " 'message': {'role': 'assistant',\n",
- " 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
- " 'created': 1692135914.7472186,\n",
- " 'model': '31dxrj3'}"
- ]
- },
- "execution_count": 20,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "model = \"31dxrj3\"\n",
- "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
- "response"
+ "print(\"=== Streaming Completion ===\")\n",
+ "response = completion(\n",
+ " model=\"baseten/openai/gpt-oss-120b\",\n",
+ " messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
+ " stream=True,\n",
+ " max_tokens=500,\n",
+ " temperature=0.8,\n",
+ " stream_options={\n",
+ " \"include_usage\": True,\n",
+ " \"continuous_usage_stats\": True\n",
+ " },\n",
+ ")\n",
+ "\n",
+ "print(\"Streaming response:\")\n",
+ "for chunk in response:\n",
+ " if chunk.choices and chunk.choices[0].delta.content:\n",
+ " print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
+ "print(\"\\n\")"
]
}
],
@@ -234,4 +143,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
new file mode 100644
index 00000000000..615baa422eb
--- /dev/null
+++ b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
@@ -0,0 +1,25 @@
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+# Upload file
+batch_input_file = client.files.create(
+ file=open("./bedrock_batch_completions.jsonl", "rb"),
+ purpose="batch",
+ extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+)
+print(batch_input_file)
+
+# Create batch
+batch = client.batches.create(
+ input_file_id=batch_input_file.id,
+ endpoint="/v1/chat/completions",
+ completion_window="24h",
+ metadata={"description": "Test batch job"},
+)
+print(batch)
\ No newline at end of file
diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl
new file mode 100644
index 00000000000..adef9ac2dd5
--- /dev/null
+++ b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl
@@ -0,0 +1,128 @@
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py
new file mode 100644
index 00000000000..6ee5555695e
--- /dev/null
+++ b/cookbook/litellm_proxy_server/cli_token_usage.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Example: Using CLI token with LiteLLM SDK
+
+This example shows how to use the CLI authentication token
+in your Python scripts after running `litellm-proxy login`.
+"""
+
+from textwrap import indent
+import litellm
+LITELLM_BASE_URL = "http://localhost:4000/"
+
+
+def main():
+ """Using CLI token with LiteLLM SDK"""
+ print("🚀 Using CLI Token with LiteLLM SDK")
+ print("=" * 40)
+ #litellm._turn_on_debug()
+
+ # Get the CLI token
+ api_key = litellm.get_litellm_gateway_api_key()
+
+ if not api_key:
+ print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
+ return
+
+ print("✅ Found CLI token.")
+
+ available_models = litellm.get_valid_models(
+ check_provider_endpoint=True,
+ custom_llm_provider="litellm_proxy",
+ api_key=api_key,
+ api_base=LITELLM_BASE_URL
+ )
+
+ print("✅ Available models:")
+ if available_models:
+ for i, model in enumerate(available_models, 1):
+ print(f" {i:2d}. {model}")
+ else:
+ print(" No models available")
+
+ # Use with LiteLLM
+ try:
+ response = litellm.completion(
+ model="litellm_proxy/gemini/gemini-2.5-flash",
+ messages=[{"role": "user", "content": "Hello from CLI token!"}],
+ api_key=api_key,
+ base_url=LITELLM_BASE_URL
+ )
+ print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
+ except Exception as e:
+ print(f"❌ Error: {e}")
+
+
+if __name__ == "__main__":
+ main()
+
+ print("\n💡 Tips:")
+ print("1. Run 'litellm-proxy login' to authenticate first")
+ print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
+ print("3. The token is stored locally at ~/.litellm/token.json")
diff --git a/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py b/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
new file mode 100644
index 00000000000..351b0920eb8
--- /dev/null
+++ b/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
@@ -0,0 +1,36 @@
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234", # paste your litellm proxy api key here
+ base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+ model="gpt-5",
+ input=[
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ tools=[
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never"
+ }
+ ],
+ stream=True,
+ tool_choice="required"
+)
+
+for chunk in response:
+ print("response chunk: ", chunk)
diff --git a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py b/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
index 689f105bc5f..1dc2d914857 100644
--- a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
+++ b/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py
@@ -5,7 +5,7 @@ import os
import litellm
from litellm import Router
from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
load_dotenv()
diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
index a8aa506e8a2..76d5d3913f5 100644
--- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
+++ b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py
@@ -12,7 +12,7 @@ sys.path.insert(
import litellm
from litellm import Router
from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
load_dotenv()
diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
index a8aa506e8a2..76d5d3913f5 100644
--- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
+++ b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py
@@ -12,7 +12,7 @@ sys.path.insert(
import litellm
from litellm import Router
from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
load_dotenv()
diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
new file mode 100644
index 00000000000..d47de5b0871
--- /dev/null
+++ b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
@@ -0,0 +1,400 @@
+# LiteLLM Release Notes Generation Instructions
+
+This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
+
+## Required Inputs
+
+1. **Release Version** (e.g., `v1.77.3-stable`)
+2. **PR Diff/Changelog** - List of PRs with titles and contributors
+3. **Previous Version Commit Hash** - To compare model pricing changes
+4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
+
+## Step-by-Step Process
+
+### 1. Initial Setup and Analysis
+
+```bash
+# Check git diff for model pricing changes
+git diff HEAD -- model_prices_and_context_window.json
+```
+
+**Key Analysis Points:**
+- New models added (look for new entries)
+- Deprecated models removed (look for deleted entries)
+- Pricing updates (look for cost changes)
+- Feature support changes (tool calling, reasoning, etc.)
+
+### 2. Release Notes Structure
+
+Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
+
+```markdown
+---
+title: "v1.77.X-stable - [Key Theme]"
+slug: "v1-77-X"
+date: YYYY-MM-DDTHH:mm:ss
+authors: [standard author block]
+hide_table_of_contents: false
+---
+
+## Deploy this version
+[Docker and pip installation tabs]
+
+## Key Highlights
+[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
+
+## New Models / Updated Models
+#### New Model Support
+[Model pricing table]
+
+#### Features
+[Provider-specific features organized by provider]
+
+### Bug Fixes
+[Provider-specific bug fixes organized by provider]
+
+#### New Provider Support
+[New provider integrations]
+
+## LLM API Endpoints
+#### Features
+[API-specific features organized by API type]
+
+#### Bugs
+[General bug fixes]
+
+## Management Endpoints / UI
+#### Features
+[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
+
+#### Bugs
+[Management-related bug fixes]
+
+## Logging / Guardrail / Prompt Management Integrations
+#### Features
+[Organized by integration provider with proper doc links]
+
+#### Guardrails
+[Guardrail-specific features and fixes]
+
+#### Prompt Management
+[Prompt management integrations like BitBucket]
+
+## Spend Tracking, Budgets and Rate Limiting
+[Cost tracking, service tier pricing, rate limiting improvements]
+
+## MCP Gateway
+[MCP-specific features, OAuth 2.0, configuration improvements]
+
+## Performance / Loadbalancing / Reliability improvements
+[Infrastructure improvements, memory fixes, performance optimizations]
+
+## Documentation Updates
+[Documentation improvements, guides, corrections - separate section for visibility]
+
+## New Contributors
+[List of first-time contributors]
+
+## Full Changelog
+[Link to GitHub comparison]
+```
+
+### 3. Categorization Rules
+
+**Performance Improvements:**
+- RPS improvements
+- Memory optimizations
+- CPU usage optimizations
+- Timeout controls
+- Worker configuration
+- Memory leak fixes
+- Cache performance improvements
+- Database connection management
+- Dependency management (fastuuid, etc.)
+- Configuration management
+
+**New Models/Updated Models:**
+- Extract from model_prices_and_context_window.json diff
+- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
+- **Structure:**
+ - `#### New Model Support` - pricing table
+ - `#### Features` - organized by provider with documentation links
+ - `### Bug Fixes` - provider-specific bug fixes
+ - `#### New Provider Support` - major new provider integrations
+- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
+- Use bullet points under each provider for multiple features
+- Separate features from bug fixes clearly
+
+**LLM API Endpoints:**
+- **Structure:**
+ - `#### Features` - organized by API type (Responses API, Batch API, etc.)
+ - `#### Bugs` - general bug fixes under **General** category
+- **API Categories:**
+ - Responses API
+ - Batch API
+ - CountTokens API
+ - Images API
+ - Video Generation (if applicable)
+ - General (miscellaneous improvements)
+- Use proper documentation links for each API type
+
+**UI/Management:**
+- Authentication changes
+- Dashboard improvements
+- Team management
+- Key management
+- Proxy CLI authentication and improvements
+- Virtual key management and scheduled rotations
+- SSO configuration fixes
+- Admin settings updates
+- Management routes and endpoints
+
+**Logging / Guardrail / Prompt Management Integrations:**
+- **Structure:**
+ - `#### Features` - organized by integration provider with proper doc links
+ - `#### Guardrails` - guardrail-specific features and fixes
+ - `#### Prompt Management` - prompt management integrations
+ - `#### New Integration` - major new integrations
+- **Integration Categories:**
+ - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
+ - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
+ - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
+ - **[PostHog](../../docs/observability/posthog)** - observability integration
+ - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
+ - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
+ - Other logging providers with proper doc links
+- **Guardrail Categories:**
+ - LakeraAI, Presidio, Noma, and other guardrail providers
+- **Prompt Management:**
+ - BitBucket, GitHub, and other prompt management integrations
+- Use bullet points under each provider for multiple features
+- Separate logging features from guardrails and prompt management clearly
+
+### 4. Documentation Linking Strategy
+
+**Link to docs when:**
+- New provider support added
+- Significant feature additions
+- API endpoint changes
+- Integration additions
+
+**Link format:** `../../docs/[category]/[specific_doc]`
+
+**Common doc paths:**
+- `../../docs/providers/[provider]` - Provider-specific docs
+- `../../docs/image_generation` - Image generation
+- `../../docs/video_generation` - Video generation (if exists)
+- `../../docs/response_api` - Responses API
+- `../../docs/proxy/logging` - Logging integrations
+- `../../docs/proxy/guardrails` - Guardrails
+- `../../docs/pass_through/[provider]` - Passthrough endpoints
+
+### 5. Model Table Generation
+
+From git diff analysis, create tables like:
+
+```markdown
+| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
+| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
+```
+
+**Extract from JSON:**
+- `max_input_tokens` → Context Window
+- `input_cost_per_token` × 1,000,000 → Input cost
+- `output_cost_per_token` × 1,000,000 → Output cost
+- `supports_*` fields → Features
+- Special pricing fields (per image, per second) for generation models
+
+### 6. PR Categorization Logic
+
+**By Keywords in PR Title:**
+- `[Perf]`, `Performance`, `RPS` → Performance Improvements
+- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
+- `[Feat]`, `[Feature]`, `Add support` → Features section
+- `[Docs]` → Documentation Updates section
+- Provider names (Gemini, OpenAI, etc.) → Group under provider
+- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
+- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
+
+**By PR Content Analysis:**
+- New model additions → New Models section
+- UI changes → Management Endpoints/UI
+- Logging/observability → Logging/Guardrail/Prompt Management Integrations
+- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
+- Authentication → Management Endpoints/UI
+- MCP-related changes → MCP Gateway
+- Documentation updates → Documentation Updates
+- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
+
+**Special Categorization Rules:**
+- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
+- **Cost breakdown in logging** → Spend Tracking section
+- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
+- **All documentation PRs** → Documentation Updates section for visibility
+
+### 7. Writing Style Guidelines
+
+**Tone:**
+- Professional but accessible
+- Focus on user impact
+- Highlight breaking changes clearly
+- Use active voice
+
+**Formatting:**
+- Use consistent markdown formatting
+- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
+- Use code blocks for configuration examples
+- Bold important terms and section headers
+
+**Warnings/Notes:**
+- Add warning boxes for breaking changes
+- Include migration instructions when needed
+- Provide override options for default changes
+
+### 8. Quality Checks
+
+**Before finalizing:**
+- Verify all PR links work
+- Check documentation links are valid
+- Ensure model pricing is accurate
+- Confirm provider names are consistent
+- Review for typos and formatting issues
+- **Count PRs by section** - Provide final count like:
+ ```
+ ## MM/DD/YYYY
+ * New Models / Updated Models: XX
+ * LLM API Endpoints: XX
+ * Management Endpoints / UI: XX
+ * Logging / Guardrail / Prompt Management Integrations: XX
+ * Spend Tracking, Budgets and Rate Limiting: XX
+ * MCP Gateway: XX
+ * Performance / Loadbalancing / Reliability improvements: XX
+ * Documentation Updates: XX
+ ```
+
+### 9. Common Patterns to Follow
+
+**Performance Changes:**
+```markdown
+- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
+```
+
+**New Models:**
+Always include pricing table and feature highlights
+
+**Breaking Changes:**
+```markdown
+:::warning
+This release has a known issue...
+:::
+```
+
+**Provider Features (New Models / Updated Models section):**
+```markdown
+#### Features
+
+- **[Provider Name](../../docs/providers/provider)**
+ - Feature description - [PR #XXXXX](link)
+ - Another feature description - [PR #YYYYY](link)
+```
+
+**API Features (LLM API Endpoints section):**
+```markdown
+#### Features
+
+- **[API Name](../../docs/api_path)**
+ - Feature description - [PR #XXXXX](link)
+ - Another feature - [PR #YYYYY](link)
+- **General**
+ - Miscellaneous improvements - [PR #ZZZZZ](link)
+```
+
+**Integration Features (Logging / Guardrail Integrations section):**
+```markdown
+#### Features
+
+- **[Integration Name](../../docs/proxy/logging#integration)**
+ - Feature description - [PR #XXXXX](link)
+ - Bug fix description - [PR #YYYYY](link)
+```
+
+**Bug Fixes Pattern:**
+```markdown
+### Bug Fixes
+
+- **[Provider/Component Name](../../docs/providers/provider)**
+ - Bug fix description - [PR #XXXXX](link)
+```
+
+### 10. Missing Documentation Check
+
+**Review for missing docs:**
+- New providers without documentation
+- New API endpoints without examples
+- Complex features without guides
+- Integration setup instructions
+
+**Flag for documentation needs:**
+- New provider integrations
+- Significant API changes
+- Complex configuration options
+- Migration requirements
+
+### 11. New Sections and Categories (Added in v1.77.5)
+
+**MCP Gateway Section:**
+- All MCP-related changes go here (not in General Proxy Improvements)
+- OAuth 2.0 flow improvements
+- MCP configuration and tools
+- Server management features
+
+**Spend Tracking, Budgets and Rate Limiting Section:**
+- Service tier pricing (OpenAI priority/flex pricing)
+- Cost tracking and breakdown features
+- Rate limiting improvements (Parallel Request Limiter v3)
+- Priority reservation fixes
+- Metadata handling for rate limiting
+
+**Documentation Updates Section:**
+- Create separate section for all documentation improvements
+- Include provider documentation fixes
+- Model reference updates
+- New guides and tutorials
+- Documentation corrections and clarifications
+- This gives documentation changes proper visibility
+
+**Management Endpoints / UI Grouping:**
+- Group related features under sub-categories:
+ - **Proxy CLI Auth** - CLI authentication improvements
+ - **Virtual Keys** - Key rotation and management
+ - **Models + Endpoints** - Provider and endpoint management
+
+**Logging Section Expansion:**
+- Rename to "Logging / Guardrail / Prompt Management Integrations"
+- Add **Prompt Management** subsection for BitBucket, GitHub integrations
+- Keep guardrails separate from logging features
+
+## Example Command Workflow
+
+```bash
+# 1. Get model changes
+git diff HEAD -- model_prices_and_context_window.json
+
+# 2. Analyze PR list for categorization
+# 3. Create release notes following template
+# 4. Link to appropriate documentation
+# 5. Review for missing documentation needs
+```
+
+## Output Requirements
+
+- Follow exact markdown structure from reference
+- Include all PR links and contributors
+- Provide accurate model pricing tables
+- Link to relevant documentation
+- Highlight breaking changes with warnings
+- Include deployment instructions
+- End with full changelog link
+
+This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.
diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py
new file mode 100644
index 00000000000..5fd19c6f66f
--- /dev/null
+++ b/cookbook/misc/test_responses_api.py
@@ -0,0 +1,53 @@
+import base64
+from openai import OpenAI
+import time
+client = OpenAI(
+ base_url="http://0.0.0.0:4001",
+ api_key="sk-1234"
+)
+
+# Function to encode the image
+def encode_image(image_path):
+ with open(image_path, "rb") as image_file:
+ return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+# Path to your image
+image_path = "litellm/proxy/logo.jpg"
+
+# Getting the Base64 string
+base64_image = encode_image(image_path)
+
+
+response = client.responses.create(
+ model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+ input=[
+ {
+ "role": "user",
+ "content": [
+ { "type": "input_text", "text": "what color is the image"},
+ {
+ "type": "input_image",
+ "image_url": f"data:image/jpeg;base64,{base64_image}",
+ },
+ ],
+ }
+ ],
+)
+
+
+
+print(response.output_text)
+print("response1 id===", response.id)
+print("sleeping for 20 seconds...")
+time.sleep(20)
+print("making follow up request for existing id")
+response2 = client.responses.create(
+ model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+ previous_response_id=response.id,
+ input="ok, and what objects are in the image?"
+)
+
+print(response2.output_text)
+
+
diff --git a/cookbook/veo_video_generation.py b/cookbook/veo_video_generation.py
new file mode 100644
index 00000000000..64a7207feb1
--- /dev/null
+++ b/cookbook/veo_video_generation.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Complete example for Veo video generation through LiteLLM proxy.
+
+This script demonstrates how to:
+1. Generate videos using Google's Veo model
+2. Poll for completion status
+3. Download the generated video file
+
+Requirements:
+- LiteLLM proxy running with Google AI Studio pass-through configured
+- Google AI Studio API key with Veo access
+"""
+
+import json
+import os
+import time
+import requests
+from typing import Optional
+
+
+class VeoVideoGenerator:
+ """Complete Veo video generation client using LiteLLM proxy."""
+
+ def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta",
+ api_key: str = "sk-1234"):
+ """
+ Initialize the Veo video generator.
+
+ Args:
+ base_url: Base URL for the LiteLLM proxy with Gemini pass-through
+ api_key: API key for LiteLLM proxy authentication
+ """
+ self.base_url = base_url
+ self.api_key = api_key
+ self.headers = {
+ "x-goog-api-key": api_key,
+ "Content-Type": "application/json"
+ }
+
+ def generate_video(self, prompt: str) -> Optional[str]:
+ """
+ Initiate video generation with Veo.
+
+ Args:
+ prompt: Text description of the video to generate
+
+ Returns:
+ Operation name if successful, None otherwise
+ """
+ print(f"🎬 Generating video with prompt: '{prompt}'")
+
+ url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
+ payload = {
+ "instances": [{
+ "prompt": prompt
+ }]
+ }
+
+ try:
+ response = requests.post(url, headers=self.headers, json=payload)
+ response.raise_for_status()
+
+ data = response.json()
+ operation_name = data.get("name")
+
+ if operation_name:
+ print(f"✅ Video generation started: {operation_name}")
+ return operation_name
+ else:
+ print("❌ No operation name returned")
+ print(f"Response: {json.dumps(data, indent=2)}")
+ return None
+
+ except requests.RequestException as e:
+ print(f"❌ Failed to start video generation: {e}")
+ if hasattr(e, 'response') and e.response is not None:
+ try:
+ error_data = e.response.json()
+ print(f"Error details: {json.dumps(error_data, indent=2)}")
+ except:
+ print(f"Error response: {e.response.text}")
+ return None
+
+ def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+ """
+ Poll operation status until video generation is complete.
+
+ Args:
+ operation_name: Name of the operation to monitor
+ max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
+
+ Returns:
+ Video URI if successful, None otherwise
+ """
+ print("⏳ Waiting for video generation to complete...")
+
+ operation_url = f"{self.base_url}/{operation_name}"
+ start_time = time.time()
+ poll_interval = 10 # Start with 10 seconds
+
+ while time.time() - start_time < max_wait_time:
+ try:
+ print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
+
+ response = requests.get(operation_url, headers=self.headers)
+ response.raise_for_status()
+
+ data = response.json()
+
+ # Check for errors
+ if "error" in data:
+ print("❌ Error in video generation:")
+ print(json.dumps(data["error"], indent=2))
+ return None
+
+ # Check if operation is complete
+ is_done = data.get("done", False)
+
+ if is_done:
+ print("🎉 Video generation complete!")
+
+ try:
+ # Extract video URI from nested response
+ video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+ print(f"📹 Video URI: {video_uri}")
+ return video_uri
+ except KeyError as e:
+ print(f"❌ Could not extract video URI: {e}")
+ print("Full response:")
+ print(json.dumps(data, indent=2))
+ return None
+
+ # Wait before next poll, with exponential backoff
+ time.sleep(poll_interval)
+ poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds
+
+ except requests.RequestException as e:
+ print(f"❌ Error polling operation status: {e}")
+ time.sleep(poll_interval)
+
+ print(f"⏰ Timeout after {max_wait_time} seconds")
+ return None
+
+ def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+ """
+ Download the generated video file.
+
+ Args:
+ video_uri: URI of the video to download (from Google's response)
+ output_filename: Local filename to save the video
+
+ Returns:
+ True if download successful, False otherwise
+ """
+ print(f"⬇️ Downloading video...")
+ print(f"Original URI: {video_uri}")
+
+ # Convert Google URI to LiteLLM proxy URI
+ # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
+ if video_uri.startswith("files/"):
+ download_path = f"{video_uri}:download?alt=media"
+ else:
+ download_path = video_uri
+
+ litellm_download_url = f"{self.base_url}/{download_path}"
+ print(f"Download URL: {litellm_download_url}")
+
+ try:
+ # Download with streaming and redirect handling
+ response = requests.get(
+ litellm_download_url,
+ headers=self.headers,
+ stream=True,
+ allow_redirects=True # Handle redirects automatically
+ )
+ response.raise_for_status()
+
+ # Save video file
+ with open(output_filename, 'wb') as f:
+ downloaded_size = 0
+ for chunk in response.iter_content(chunk_size=8192):
+ if chunk:
+ f.write(chunk)
+ downloaded_size += len(chunk)
+
+ # Progress indicator for large files
+ if downloaded_size % (1024 * 1024) == 0: # Every MB
+ print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
+
+ # Verify file was created and has content
+ if os.path.exists(output_filename):
+ file_size = os.path.getsize(output_filename)
+ if file_size > 0:
+ print(f"✅ Video downloaded successfully!")
+ print(f"📁 Saved as: {output_filename}")
+ print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
+ return True
+ else:
+ print("❌ Downloaded file is empty")
+ os.remove(output_filename)
+ return False
+ else:
+ print("❌ File was not created")
+ return False
+
+ except requests.RequestException as e:
+ print(f"❌ Download failed: {e}")
+ if hasattr(e, 'response') and e.response is not None:
+ print(f"Status code: {e.response.status_code}")
+ print(f"Response headers: {dict(e.response.headers)}")
+ return False
+
+ def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
+ """
+ Complete workflow: generate video and download it.
+
+ Args:
+ prompt: Text description for video generation
+ output_filename: Output filename (auto-generated if None)
+
+ Returns:
+ True if successful, False otherwise
+ """
+ # Auto-generate filename if not provided
+ if output_filename is None:
+ timestamp = int(time.time())
+ safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
+ output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+
+ print("=" * 60)
+ print("🎬 VEO VIDEO GENERATION WORKFLOW")
+ print("=" * 60)
+
+ # Step 1: Generate video
+ operation_name = self.generate_video(prompt)
+ if not operation_name:
+ return False
+
+ # Step 2: Wait for completion
+ video_uri = self.wait_for_completion(operation_name)
+ if not video_uri:
+ return False
+
+ # Step 3: Download video
+ success = self.download_video(video_uri, output_filename)
+
+ if success:
+ print("=" * 60)
+ print("🎉 SUCCESS! Video generation complete!")
+ print(f"📁 Video saved as: {output_filename}")
+ print("=" * 60)
+ else:
+ print("=" * 60)
+ print("❌ FAILED! Video generation or download failed")
+ print("=" * 60)
+
+ return success
+
+
+def main():
+ """
+ Example usage of the VeoVideoGenerator.
+
+ Configure these environment variables:
+ - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
+ - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
+ """
+
+ # Configuration from environment or defaults
+ base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
+ api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+
+ print("🚀 Starting Veo Video Generation Example")
+ print(f"📡 Using LiteLLM proxy at: {base_url}")
+
+ # Initialize generator
+ generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
+
+ # Example prompts - try different ones!
+ example_prompts = [
+ "A cat playing with a ball of yarn in a sunny garden",
+ "Ocean waves crashing against rocky cliffs at sunset",
+ "A bustling city street with people walking and cars passing by",
+ "A peaceful forest with sunlight filtering through the trees"
+ ]
+
+ # Use first example or get from user
+ prompt = example_prompts[0]
+ print(f"🎬 Using prompt: '{prompt}'")
+
+ # Generate and download video
+ success = generator.generate_and_download(prompt)
+
+ if success:
+ print("\n✅ Example completed successfully!")
+ print("💡 Try modifying the prompt in the script for different videos!")
+ else:
+ print("\n❌ Example failed!")
+ print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
+
+ # Troubleshooting tips
+ print("\n🔍 Troubleshooting:")
+ print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
+ print("2. Verify your Google AI Studio API key has Veo access")
+ print("3. Check that your prompt meets Veo's content guidelines")
+ print("4. Review the LiteLLM proxy logs for detailed error information")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/db_scripts/migrate_keys.py b/db_scripts/migrate_keys.py
new file mode 100644
index 00000000000..5c940e069b3
--- /dev/null
+++ b/db_scripts/migrate_keys.py
@@ -0,0 +1,187 @@
+from prisma import Prisma
+import csv
+import json
+import asyncio
+from datetime import datetime
+from typing import Optional, List, Dict, Any
+
+import os
+
+## VARIABLES
+DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/litellm"
+CSV_FILE_PATH = "./path_to_csv.csv"
+
+os.environ["DATABASE_URL"] = DATABASE_URL
+
+
+async def parse_csv_value(value: str, field_type: str) -> Any:
+ """Parse CSV values according to their expected types"""
+ if value == "NULL" or value == "" or value is None:
+ return None
+
+ if field_type == "boolean":
+ return value.lower() == "true"
+ elif field_type == "float":
+ return float(value)
+ elif field_type == "int":
+ return int(value) if value.isdigit() else None
+ elif field_type == "bigint":
+ return int(value) if value.isdigit() else None
+ elif field_type == "datetime":
+ try:
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except:
+ return None
+ elif field_type == "json":
+ try:
+ return value if value else json.dumps({})
+ except:
+ return json.dumps({})
+ elif field_type == "string_array":
+ # Handle string arrays like {default-models}
+ if value.startswith("{") and value.endswith("}"):
+ content = value[1:-1] # Remove braces
+ if content:
+ return [item.strip() for item in content.split(",")]
+ else:
+ return []
+ return []
+ else:
+ return value
+
+
+async def migrate_verification_tokens():
+ """Main migration function"""
+ prisma = Prisma()
+ await prisma.connect()
+
+ try:
+ # Read CSV file
+ csv_file_path = CSV_FILE_PATH
+
+ with open(csv_file_path, "r", encoding="utf-8") as file:
+ csv_reader = csv.DictReader(file)
+
+ processed_count = 0
+ error_count = 0
+
+ for row in csv_reader:
+ try:
+ # Replace 'default-team' with the specified UUID
+ team_id = row.get("team_id")
+ if team_id == "NULL" or team_id == "":
+ team_id = None
+
+ # Prepare data for insertion
+ verification_token_data = {
+ "token": row["token"],
+ "key_name": await parse_csv_value(row["key_name"], "string"),
+ "key_alias": await parse_csv_value(row["key_alias"], "string"),
+ "soft_budget_cooldown": await parse_csv_value(
+ row["soft_budget_cooldown"], "boolean"
+ ),
+ "spend": await parse_csv_value(row["spend"], "float"),
+ "expires": await parse_csv_value(row["expires"], "datetime"),
+ "models": await parse_csv_value(row["models"], "string_array"),
+ "aliases": await parse_csv_value(row["aliases"], "json"),
+ "config": await parse_csv_value(row["config"], "json"),
+ "user_id": await parse_csv_value(row["user_id"], "string"),
+ "team_id": team_id,
+ "permissions": await parse_csv_value(
+ row["permissions"], "json"
+ ),
+ "max_parallel_requests": await parse_csv_value(
+ row["max_parallel_requests"], "int"
+ ),
+ "metadata": await parse_csv_value(row["metadata"], "json"),
+ "tpm_limit": await parse_csv_value(row["tpm_limit"], "bigint"),
+ "rpm_limit": await parse_csv_value(row["rpm_limit"], "bigint"),
+ "max_budget": await parse_csv_value(row["max_budget"], "float"),
+ "budget_duration": await parse_csv_value(
+ row["budget_duration"], "string"
+ ),
+ "budget_reset_at": await parse_csv_value(
+ row["budget_reset_at"], "datetime"
+ ),
+ "allowed_cache_controls": await parse_csv_value(
+ row["allowed_cache_controls"], "string_array"
+ ),
+ "model_spend": await parse_csv_value(
+ row["model_spend"], "json"
+ ),
+ "model_max_budget": await parse_csv_value(
+ row["model_max_budget"], "json"
+ ),
+ "budget_id": await parse_csv_value(row["budget_id"], "string"),
+ "blocked": await parse_csv_value(row["blocked"], "boolean"),
+ "created_at": await parse_csv_value(
+ row["created_at"], "datetime"
+ ),
+ "updated_at": await parse_csv_value(
+ row["updated_at"], "datetime"
+ ),
+ "allowed_routes": await parse_csv_value(
+ row["allowed_routes"], "string_array"
+ ),
+ "object_permission_id": await parse_csv_value(
+ row["object_permission_id"], "string"
+ ),
+ "created_by": await parse_csv_value(
+ row["created_by"], "string"
+ ),
+ "updated_by": await parse_csv_value(
+ row["updated_by"], "string"
+ ),
+ "organization_id": await parse_csv_value(
+ row["organization_id"], "string"
+ ),
+ }
+
+ # Remove None values to use database defaults
+ verification_token_data = {
+ k: v
+ for k, v in verification_token_data.items()
+ if v is not None
+ }
+
+ # Check if token already exists
+ existing_token = await prisma.litellm_verificationtoken.find_unique(
+ where={"token": verification_token_data["token"]}
+ )
+
+ if existing_token:
+ print(
+ f"Token {verification_token_data['token']} already exists, skipping..."
+ )
+ continue
+
+ # Insert the record
+ await prisma.litellm_verificationtoken.create(
+ data=verification_token_data
+ )
+
+ processed_count += 1
+ print(
+ f"Successfully migrated token: {verification_token_data['token']}"
+ )
+
+ except Exception as e:
+ error_count += 1
+ print(
+ f"Error processing row with token {row.get('token', 'unknown')}: {str(e)}"
+ )
+ continue
+
+ print(f"\nMigration completed!")
+ print(f"Successfully processed: {processed_count} records")
+ print(f"Errors encountered: {error_count} records")
+
+ except Exception as e:
+ print(f"Migration failed: {str(e)}")
+
+ finally:
+ await prisma.disconnect()
+
+
+if __name__ == "__main__":
+ asyncio.run(migrate_verification_tokens())
diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml
index bd63ca6bfca..e361ee226b7 100644
--- a/deploy/charts/litellm-helm/Chart.yaml
+++ b/deploy/charts/litellm-helm/Chart.yaml
@@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.4
+version: 0.4.6
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md
index 31bda3f7d79..352c3e9ddff 100644
--- a/deploy/charts/litellm-helm/README.md
+++ b/deploy/charts/litellm-helm/README.md
@@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
-| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
+| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
@@ -36,11 +36,50 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
-| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
-| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |
+| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` |
+| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` |
+| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
+| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
+| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
+| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
+| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
+| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
+| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
+| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
+
+#### Example `proxy_config` ConfigMap from values (default):
+
+
+```
+proxyConfigMap:
+ create: true
+ key: "config.yaml"
+
+proxy_config:
+ general_settings:
+ master_key: os.environ/PROXY_MASTER_KEY
+ model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: gpt-3.5-turbo
+ api_key: eXaMpLeOnLy
+```
+
+#### Example using existing `proxyConfigMap` instead of creating it:
+
+
+```
+proxyConfigMap:
+ create: false
+ name: my-litellm-config
+ key: config.yaml
+
+# proxy_config is ignored in this mode
+```
#### Example `environmentSecrets` Secret
+
```
apiVersion: v1
kind: Secret
@@ -110,6 +149,22 @@ data:
Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
+### Migration Job Settings
+
+The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
+
+| Name | Description | Value |
+| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` |
+| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` |
+| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` |
+| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` |
+| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` |
+| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` |
+| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
+| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A |
+
+
## Accessing the Admin UI
When browsing to the URL published per the settings in `ingress.*`, you will
be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
@@ -119,7 +174,7 @@ service, the **Proxy Endpoint** should be set to `http://-litellm:4000`
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
-generated string stored in the `-litellm-masterkey` Kubernetes Secret.
+generated string in the `sk-...` format stored in the `-litellm-masterkey` Kubernetes Secret.
```bash
kubectl -n litellm get secret -litellm-masterkey -o jsonpath="{.data.masterkey}"
diff --git a/deploy/charts/litellm-helm/templates/NOTES.txt b/deploy/charts/litellm-helm/templates/NOTES.txt
index e72c9916080..017bbfa78bd 100644
--- a/deploy/charts/litellm-helm/templates/NOTES.txt
+++ b/deploy/charts/litellm-helm/templates/NOTES.txt
@@ -20,3 +20,4 @@
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
+PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*
\ No newline at end of file
diff --git a/deploy/charts/litellm-helm/templates/configmap-litellm.yaml b/deploy/charts/litellm-helm/templates/configmap-litellm.yaml
index 4598054a9d0..cf35917da03 100644
--- a/deploy/charts/litellm-helm/templates/configmap-litellm.yaml
+++ b/deploy/charts/litellm-helm/templates/configmap-litellm.yaml
@@ -1,7 +1,9 @@
+{{- if .Values.proxyConfigMap.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
-{{ .Values.proxy_config | toYaml | indent 6 }}
\ No newline at end of file
+{{ .Values.proxy_config | toYaml | indent 6 }}
+{{- end }}
\ No newline at end of file
diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml
index 5b9488c19bf..6a5a6e87577 100644
--- a/deploy/charts/litellm-helm/templates/deployment.yaml
+++ b/deploy/charts/litellm-helm/templates/deployment.yaml
@@ -1,6 +1,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
+ annotations:
+ {{- toYaml .Values.deploymentAnnotations | nindent 4 }}
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
@@ -14,7 +16,9 @@ spec:
template:
metadata:
annotations:
+ {{- if .Values.proxyConfigMap.create }}
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
+ {{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@@ -69,7 +73,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
+ {{- if .Values.db.secret.endpointKey }}
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.db.secret.name }}
+ key: {{ .Values.db.secret.endpointKey }}
+ {{- else }}
value: {{ .Values.db.endpoint }}
+ {{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
@@ -97,6 +108,12 @@ spec:
value: {{ $val | quote }}
{{- end }}
{{- end }}
+ {{- if .Values.separateHealthApp }}
+ - name: SEPARATE_HEALTH_APP
+ value: "1"
+ - name: SEPARATE_HEALTH_PORT
+ value: {{ .Values.separateHealthPort | default "8081" | quote }}
+ {{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
@@ -116,19 +133,23 @@ spec:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
+ {{- if .Values.separateHealthApp }}
+ - name: health
+ containerPort: {{ .Values.separateHealthPort | default 8081 }}
+ protocol: TCP
+ {{- end }}
livenessProbe:
httpGet:
path: /health/liveliness
- port: http
+ port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
readinessProbe:
httpGet:
path: /health/readiness
- port: http
- # Give the container time to start up. Up to 5 minutes (10 * 30 seconds)
+ port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
startupProbe:
httpGet:
path: /health/readiness
- port: http
+ port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
failureThreshold: 30
periodSeconds: 10
resources:
@@ -164,9 +185,13 @@ spec:
{{- end }}
- name: litellm-config
configMap:
+ {{- if .Values.proxyConfigMap.create }}
name: {{ include "litellm.fullname" . }}-config
+ {{- else }}
+ name: {{ .Values.proxyConfigMap.name }}
+ {{- end }}
items:
- - key: "config.yaml"
+ - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
path: "config.yaml"
{{- with .Values.volumes }}
{{- toYaml . | nindent 8 }}
diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml
index ba69f0fef8d..7a6893f28f1 100644
--- a/deploy/charts/litellm-helm/templates/migrations-job.yaml
+++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml
@@ -1,16 +1,27 @@
{{- if .Values.migrationJob.enabled }}
-# This job runs the prisma migrations for the LiteLLM DB.
+# This job runs the Prisma migrations for the LiteLLM DB.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
+ labels:
+ {{- include "litellm.labels" . | nindent 4 }}
annotations:
+ {{- if .Values.migrationJob.hooks.argocd.enabled }}
argocd.argoproj.io/hook: PreSync
- argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates
+ argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+ {{- end }}
+ {{- if .Values.migrationJob.hooks.helm.enabled }}
+ helm.sh/hook: "pre-install,pre-upgrade"
+ helm.sh/hook-delete-policy: "before-hook-creation"
+ helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
+ {{- end }}
checksum/config: {{ toYaml .Values | sha256sum }}
spec:
template:
metadata:
+ labels:
+ {{- include "litellm.labels" . | nindent 8 }}
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}
@@ -38,21 +49,44 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
+ {{- if .Values.db.secret.endpointKey }}
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.db.secret.name }}
+ key: {{ .Values.db.secret.endpointKey }}
+ {{- else }}
value: {{ .Values.db.endpoint }}
+ {{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
- {{- else }}
+ {{- else if .Values.db.deployStandalone }}
- name: DATABASE_URL
value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
{{- end }}
+ {{- if .Values.envVars }}
+ {{- range $key, $val := .Values.envVars }}
+ - name: {{ $key }}
+ value: {{ $val | quote }}
+ {{- end }}
+ {{- end }}
+ {{- with .Values.extraEnvVars }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
- name: DISABLE_SCHEMA_UPDATE
value: "false" # always run the migration from the Helm PreSync hook, override the value set
{{- with .Values.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
+ {{- with .Values.migrationJob.resources }}
+ resources:
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ {{- with .Values.migrationJob.extraContainers }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
{{- with .Values.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
diff --git a/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml b/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml
new file mode 100644
index 00000000000..1715b94c1f6
--- /dev/null
+++ b/deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml
@@ -0,0 +1,33 @@
+{{- /*
+PodDisruptionBudget for LiteLLM proxy
+Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
+Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
+*/ -}}
+{{- if .Values.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: {{ include "litellm.fullname" . }}
+ labels:
+ {{- include "litellm.labels" . | nindent 4 }}
+ {{- with .Values.pdb.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.pdb.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ selector:
+ matchLabels:
+ {{- /* Match the Deployment selector to target the same pod set */ -}}
+ {{- include "litellm.selectorLabels" . | nindent 6 }}
+ {{- if .Values.pdb.minAvailable }}
+ minAvailable: {{ .Values.pdb.minAvailable }}
+ {{- else if .Values.pdb.maxUnavailable }}
+ maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+ {{- else }}
+ # Safe default if enabled but not configured
+ maxUnavailable: 1
+ {{- end }}
+{{- end }}
diff --git a/deploy/charts/litellm-helm/templates/secret-masterkey.yaml b/deploy/charts/litellm-helm/templates/secret-masterkey.yaml
index 5632957dc05..7c8560cc2cc 100644
--- a/deploy/charts/litellm-helm/templates/secret-masterkey.yaml
+++ b/deploy/charts/litellm-helm/templates/secret-masterkey.yaml
@@ -1,5 +1,5 @@
{{- if not .Values.masterkeySecretName }}
-{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
+{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
apiVersion: v1
kind: Secret
metadata:
diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml
index b71f91377f1..f9c83966696 100644
--- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml
+++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml
@@ -115,3 +115,25 @@ tests:
content:
name: EXTRA_ENV_VAR
value: EXTRA_ENV_VAR_VALUE
+ - it: should mount existing configmap when create=false
+ template: deployment.yaml
+ set:
+ proxyConfigMap:
+ create: false
+ name: my-litellm-config
+ key: custom.yaml
+ asserts:
+ - contains:
+ path: spec.template.spec.volumes
+ content:
+ name: litellm-config
+ configMap:
+ name: my-litellm-config
+ items:
+ - key: custom.yaml
+ path: config.yaml
+ - contains:
+ path: spec.template.spec.containers[0].volumeMounts
+ content:
+ name: litellm-config
+ mountPath: /etc/litellm/
\ No newline at end of file
diff --git a/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml b/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
index eb1d3c3967f..bbbade9d802 100644
--- a/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
+++ b/deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
@@ -2,13 +2,19 @@ suite: test masterkey secret
templates:
- secret-masterkey.yaml
tests:
- - it: should create a secret if masterkeySecretName is not set
+ - it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
asserts:
- isKind:
of: Secret
+ - matchRegex:
+ path: data.masterkey
+ pattern: ^c2st
+ # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
+ # but stored as base64 encoded in Kubernetes secret (requirement).
+ # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:
diff --git a/deploy/charts/litellm-helm/tests/migrations-job_tests.yaml b/deploy/charts/litellm-helm/tests/migrations-job_tests.yaml
new file mode 100644
index 00000000000..3a7bfa5eb0c
--- /dev/null
+++ b/deploy/charts/litellm-helm/tests/migrations-job_tests.yaml
@@ -0,0 +1,127 @@
+suite: test migrations job
+templates:
+ - migrations-job.yaml
+tests:
+ - it: should work with envVars
+ template: migrations-job.yaml
+ set:
+ envVars:
+ TEST_ENV_VAR: "test_value"
+ ANOTHER_VAR: "another_value"
+ migrationJob:
+ enabled: true
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: TEST_ENV_VAR
+ value: "test_value"
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: ANOTHER_VAR
+ value: "another_value"
+
+ - it: should work with extraEnvVars
+ template: migrations-job.yaml
+ set:
+ extraEnvVars:
+ - name: EXTRA_ENV_VAR
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.labels['env']
+ - name: SIMPLE_EXTRA_VAR
+ value: "simple_value"
+ migrationJob:
+ enabled: true
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: EXTRA_ENV_VAR
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.labels['env']
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: SIMPLE_EXTRA_VAR
+ value: "simple_value"
+
+ - it: should work with both envVars and extraEnvVars
+ template: migrations-job.yaml
+ set:
+ envVars:
+ ENV_VAR: "env_var_value"
+ extraEnvVars:
+ - name: EXTRA_ENV_VAR
+ value: "extra_env_var_value"
+ migrationJob:
+ enabled: true
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: ENV_VAR
+ value: "env_var_value"
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: EXTRA_ENV_VAR
+ value: "extra_env_var_value"
+
+ - it: should not render when migrations job is disabled
+ template: migrations-job.yaml
+ set:
+ migrationJob:
+ enabled: false
+ asserts:
+ - hasDocuments:
+ count: 0
+
+ - it: should still include default env vars
+ template: migrations-job.yaml
+ set:
+ envVars:
+ CUSTOM_VAR: "custom_value"
+ migrationJob:
+ enabled: true
+ db:
+ useExisting: true
+ endpoint: "test-db"
+ database: "testdb"
+ url: "postgresql://user:pass@test-db:5432/testdb"
+ secret:
+ name: "test-secret"
+ usernameKey: "username"
+ passwordKey: "password"
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: DISABLE_SCHEMA_UPDATE
+ value: "false"
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: DATABASE_HOST
+ value: "test-db"
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: CUSTOM_VAR
+ value: "custom_value"
+
+ - it: should not include DATABASE_URL when deployStandalone is false
+ template: migrations-job.yaml
+ set:
+ migrationJob:
+ enabled: true
+ db:
+ deployStandalone: false
+ useExisting: false
+ asserts:
+ - notContains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: DATABASE_URL
\ No newline at end of file
diff --git a/deploy/charts/litellm-helm/tests/pdb_tests.yaml b/deploy/charts/litellm-helm/tests/pdb_tests.yaml
new file mode 100644
index 00000000000..5e042e80bd3
--- /dev/null
+++ b/deploy/charts/litellm-helm/tests/pdb_tests.yaml
@@ -0,0 +1,45 @@
+suite: "pdb enabled"
+templates:
+ - poddisruptionbudget.yaml
+tests:
+ - it: "renders a PDB with maxUnavailable=1"
+ set:
+ pdb.enabled: true
+ pdb.maxUnavailable: 1
+ asserts:
+ - hasDocuments: { count: 1 }
+ - isKind: { of: PodDisruptionBudget }
+ - equal: { path: apiVersion, value: policy/v1 }
+ - equal: { path: spec.maxUnavailable, value: 1 }
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/name: litellm
+ app.kubernetes.io/instance: RELEASE-NAME
+
+---
+suite: "pdb disabled"
+templates:
+ - poddisruptionbudget.yaml
+tests:
+ - it: "does not render when disabled"
+ set:
+ pdb.enabled: false
+ asserts:
+ - hasDocuments: { count: 0 }
+
+---
+suite: "pdb minAvailable precedence"
+templates:
+ - poddisruptionbudget.yaml
+tests:
+ - it: "uses minAvailable when both are set"
+ set:
+ pdb.enabled: true
+ pdb.minAvailable: "50%"
+ pdb.maxUnavailable: 1
+ asserts:
+ - isKind: { of: PodDisruptionBudget }
+ - equal: { path: apiVersion, value: policy/v1 }
+ - equal: { path: spec.minAvailable, value: "50%" }
+ - isNull: { path: spec.maxUnavailable }
diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml
index 213db35a20f..c1792497d29 100644
--- a/deploy/charts/litellm-helm/values.yaml
+++ b/deploy/charts/litellm-helm/values.yaml
@@ -27,6 +27,9 @@ serviceAccount:
# If not set and create is true, a name is generated using the fullname template
name: ""
+# annotations for litellm deployment
+deploymentAnnotations: {}
+# annotations for litellm pods
podAnnotations: {}
podLabels: {}
@@ -60,6 +63,12 @@ service:
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
+# Separate health app configuration
+# When enabled, health checks will use a separate port and the application
+# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
+separateHealthApp: false
+separateHealthPort: 8081
+
ingress:
enabled: false
className: "nginx"
@@ -84,6 +93,14 @@ masterkeySecretName: ""
# if set, use this secret key for the master key; otherwise, use the default key
masterkeySecretKey: ""
+proxyConfigMap:
+ # when true, creates a new configmap
+ create: true
+ # if create is false and name is set, use existing ConfigMap
+ # create: false
+ # name: ""
+ # key: "config.yaml"
+
# The elements within proxy_config are rendered as config.yaml for the proxy
# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
# Reference: https://docs.litellm.ai/docs/proxy/configs
@@ -152,6 +169,8 @@ db:
name: postgres
usernameKey: username
passwordKey: password
+ # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
+ endpointKey: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@@ -197,6 +216,18 @@ migrationJob:
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
ttlSecondsAfterFinished: 120
+ resources: {}
+ # requests:
+ # cpu: 100m
+ # memory: 100Mi
+ extraContainers: []
+
+ # Hook configuration
+ hooks:
+ argocd:
+ enabled: true
+ helm:
+ enabled: false
# Additional environment variables to be added to the deployment as a map of key-value pairs
envVars: {
@@ -209,4 +240,11 @@ extraEnvVars: {
# value: EXTRA_ENV_VAR_VALUE
}
-
+# Pod Disruption Budget
+pdb:
+ enabled: false
+ # Set exactly one of the following. If both are set, minAvailable takes precedence.
+ minAvailable: null # e.g. "50%" or 1
+ maxUnavailable: null # e.g. 1 or "20%"
+ annotations: {}
+ labels: {}
diff --git a/dist/litellm-1.57.6.tar.gz b/dist/litellm-1.57.6.tar.gz
deleted file mode 100644
index 01a039cf6ee..00000000000
Binary files a/dist/litellm-1.57.6.tar.gz and /dev/null differ
diff --git a/docker-compose.yml b/docker-compose.yml
index 2e90d897f21..366fbe51b5a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,68 +1,66 @@
-version: "3.11"
-services:
- litellm:
- build:
- context: .
- args:
- target: runtime
- image: ghcr.io/berriai/litellm:main-stable
- #########################################
- ## Uncomment these lines to start proxy with a config.yaml file ##
- # volumes:
- # - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
- # command:
- # - "--config=/app/config.yaml"
- ##############################################
- ports:
- - "4000:4000" # Map the container port to the host, change the host port if necessary
- environment:
- DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
- STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
- env_file:
- - .env # Load local .env file
- depends_on:
- - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
- healthcheck: # Defines the health check configuration for the container
- test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ] # Command to execute for health check
- interval: 30s # Perform health check every 30 seconds
- timeout: 10s # Health check command times out after 10 seconds
- retries: 3 # Retry up to 3 times if health check fails
- start_period: 40s # Wait 40 seconds after container start before beginning health checks
-
- db:
- image: postgres:16
- restart: always
- container_name: litellm_db
- environment:
- POSTGRES_DB: litellm
- POSTGRES_USER: llmproxy
- POSTGRES_PASSWORD: dbpassword9090
- ports:
- - "5432:5432"
- volumes:
- - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
- healthcheck:
- test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
- interval: 1s
- timeout: 5s
- retries: 10
-
- prometheus:
- image: prom/prometheus
- volumes:
- - prometheus_data:/prometheus
- - ./prometheus.yml:/etc/prometheus/prometheus.yml
- ports:
- - "9090:9090"
- command:
- - "--config.file=/etc/prometheus/prometheus.yml"
- - "--storage.tsdb.path=/prometheus"
- - "--storage.tsdb.retention.time=15d"
- restart: always
-
-volumes:
- prometheus_data:
- driver: local
- postgres_data:
- name: litellm_postgres_data # Named volume for Postgres data persistence
-
+services:
+ litellm:
+ build:
+ context: .
+ args:
+ target: runtime
+ image: ghcr.io/berriai/litellm:main-stable
+ #########################################
+ ## Uncomment these lines to start proxy with a config.yaml file ##
+ # volumes:
+ # - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
+ # command:
+ # - "--config=/app/config.yaml"
+ ##############################################
+ ports:
+ - "4000:4000" # Map the container port to the host, change the host port if necessary
+ environment:
+ DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+ STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
+ env_file:
+ - .env # Load local .env file
+ depends_on:
+ - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
+ healthcheck: # Defines the health check configuration for the container
+ test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ] # Command to execute for health check
+ interval: 30s # Perform health check every 30 seconds
+ timeout: 10s # Health check command times out after 10 seconds
+ retries: 3 # Retry up to 3 times if health check fails
+ start_period: 40s # Wait 40 seconds after container start before beginning health checks
+
+ db:
+ image: postgres:16
+ restart: always
+ container_name: litellm_db
+ environment:
+ POSTGRES_DB: litellm
+ POSTGRES_USER: llmproxy
+ POSTGRES_PASSWORD: dbpassword9090
+ ports:
+ - "5432:5432"
+ volumes:
+ - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
+ interval: 1s
+ timeout: 5s
+ retries: 10
+
+ prometheus:
+ image: prom/prometheus
+ volumes:
+ - prometheus_data:/prometheus
+ - ./prometheus.yml:/etc/prometheus/prometheus.yml
+ ports:
+ - "9090:9090"
+ command:
+ - "--config.file=/etc/prometheus/prometheus.yml"
+ - "--storage.tsdb.path=/prometheus"
+ - "--storage.tsdb.retention.time=15d"
+ restart: always
+
+volumes:
+ prometheus_data:
+ driver: local
+ postgres_data:
+ name: litellm_postgres_data # Named volume for Postgres data persistence
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index da0326fd2cd..351c4f6bc48 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -57,6 +57,9 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
@@ -71,8 +74,12 @@ RUN chmod +x docker/entrypoint.sh
RUN chmod +x docker/prod_entrypoint.sh
EXPOSE 4000/tcp
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
# # Set your entrypoint and command
+
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
new file mode 100644
index 00000000000..2e886915203
--- /dev/null
+++ b/docker/Dockerfile.dev
@@ -0,0 +1,87 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.11-slim
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
+
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+USER root
+
+# Install build dependencies in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ gcc \
+ python3-dev \
+ libssl-dev \
+ pkg-config \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip install --upgrade pip build
+
+# Copy requirements first for better layer caching
+COPY requirements.txt .
+
+# Install Python dependencies with cache mount for faster rebuilds
+RUN --mount=type=cache,target=/root/.cache/pip \
+ pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Fix JWT dependency conflicts early
+RUN pip uninstall jwt -y || true && \
+ pip uninstall PyJWT -y || true && \
+ pip install PyJWT==2.9.0 --no-cache-dir
+
+# Copy only necessary files for build
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+COPY litellm/ ./litellm/
+COPY enterprise/ ./enterprise/
+COPY docker/ ./docker/
+
+# Build Admin UI once
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# Install the built package
+RUN pip install dist/*.whl
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Ensure runtime stage runs as root
+USER root
+
+# Install only runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ libssl3 \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy only necessary runtime files
+COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/
+COPY litellm/ ./litellm/
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+
+# Copy pre-built wheels and install everything at once
+COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app/dist/*.whl .
+
+# Install all dependencies in one step with no-cache for smaller image
+RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
+ rm -f *.whl && \
+ rm -rf /wheels
+
+# Generate prisma client and set permissions
+RUN prisma generate && \
+ chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs
+CMD ["--port", "4000"]
\ No newline at end of file
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 079778cafb8..4178724e6e4 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -1,94 +1,106 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.13.1-slim
+# Base images
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.13.1-slim
-# Builder stage
+# -----------------
+# Builder Stage
+# -----------------
FROM $LITELLM_BUILD_IMAGE AS builder
-
-# Set the working directory to /app
WORKDIR /app
-# Set the shell to bash
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
# Install build dependencies
-RUN apt-get clean && apt-get update && \
- apt-get install -y gcc g++ python3-dev && \
- rm -rf /var/lib/apt/lists/*
+USER root
+RUN apk add --no-cache build-base bash \
+ && pip install --no-cache-dir --upgrade pip build
-RUN pip install --no-cache-dir --upgrade pip && \
- pip install --no-cache-dir build
-
-# Copy the current directory contents into the container at /app
+# Copy project files
COPY . .
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-# Build the package
-RUN rm -rf dist/* && python -m build
+# Build package and wheel dependencies
+RUN rm -rf dist/* && python -m build && \
+ pip install dist/*.whl && \
+ pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
-
-# Install the package
-RUN pip install dist/*.whl
-
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# Runtime stage
+# -----------------
+# Runtime Stage
+# -----------------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-# Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
-
WORKDIR /app
-# Copy the current directory contents into the container at /app
-COPY . .
-RUN ls -la /app
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
+# Install runtime dependencies
+USER root
+RUN apk upgrade --no-cache && \
+ apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
+
+# Copy only necessary artifacts from builder stage for runtime
+COPY . .
+COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
+COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
+COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# Install package from wheel and dependencies
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
+ && rm -f *.whl \
+ && rm -rf /wheels
-# ensure pyjwt is used, not jwt
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
+# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
- pip uninstall PyJWT -y && \
- pip install PyJWT==2.9.0 --no-cache-dir
+ pip uninstall PyJWT -y && \
+ pip install PyJWT==2.9.0 --no-cache-dir
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-### Prisma Handling for Non-Root #################################################
-# Prisma allows you to specify the binary cache directory to use
+# --- Prisma Handling for Non-Root User ---
+# Set Prisma cache directories
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
+ENV NPM_CONFIG_CACHE=/.npm
-RUN pip install --no-cache-dir nodejs-bin prisma
+# Install prisma and make entrypoints executable
+RUN pip install --no-cache-dir prisma && \
+ chmod +x docker/entrypoint.sh && \
+ chmod +x docker/prod_entrypoint.sh
-# Make a /non-existent folder and assign chown to nobody
-RUN mkdir -p /nonexistent && \
- chown -R nobody:nogroup /app && \
- chown -R nobody:nogroup /nonexistent && \
- chown -R nobody:nogroup /usr/local/lib/python3.13/site-packages/prisma/
+# Create directories and set permissions for non-root user
+RUN mkdir -p /nonexistent /.npm && \
+ chown -R nobody:nogroup /app && \
+ chown -R nobody:nogroup /nonexistent /.npm && \
+ PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+ chown -R nobody:nogroup $PRISMA_PATH && \
+ LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
+ [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
+# Get paths for directories that need write access at runtime
+RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+ LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+ # Set group ownership to 0 (root group) for OpenShift compatibility && \
+ chgrp -R 0 $PRISMA_PATH && \
+ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
+ # Mirror owner permissions to group (g=u) as recommended by Red Hat && \
+ chmod -R g=u $PRISMA_PATH && \
+ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
+ # Ensure directories are writable by group && \
+ chmod -R g+w $PRISMA_PATH && \
+ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
-# Run Prisma generate as user = nobody
+# Switch to non-root user
USER nobody
+# Set HOME for prisma generate to have a writable directory
+ENV HOME=/app
RUN prisma generate
-### End of Prisma Handling for Non-Root #########################################
+# --- End of Prisma Handling ---
EXPOSE 4000/tcp
-# # Set your entrypoint and command
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
+# Set entrypoint and command
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# CMD ["--port", "4000", "--detailed_debug"]
diff --git a/docker/README.md b/docker/README.md
index 8dbc59d01bf..ce478dfe0dd 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,3 +1,65 @@
-# LiteLLM Docker
+# Docker Development Guide
-This is a minimal Docker Compose setup for self-hosting LiteLLM.
\ No newline at end of file
+This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
+
+## Prerequisites
+
+- Docker
+- Docker Compose
+
+## Building and Running the Application
+
+To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment.
+
+### 1. Set the Master Key
+
+The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
+
+Create a `.env` file in the root of the project and add the following line:
+
+```
+MASTER_KEY=your-secret-key
+```
+
+Replace `your-secret-key` with a strong, randomly generated secret.
+
+### 2. Build and Run the Containers
+
+Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
+
+```bash
+docker compose up -d --build
+```
+
+This command will:
+
+- Build the Docker image using `Dockerfile.non_root`.
+- Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`).
+- The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code.
+
+### 3. Verifying the Application is Running
+
+You can check the status of the running containers with the following command:
+
+```bash
+docker compose ps
+```
+
+To view the logs of the `litellm` container, run:
+
+```bash
+docker compose logs -f litellm
+```
+
+### 4. Stopping the Application
+
+To stop the running containers, use the following command:
+
+```bash
+docker compose down
+```
+
+## Troubleshooting
+
+- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
+- **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.
diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip
index b8a0f2a2c6c..aeb19bce21f 100644
--- a/docker/build_from_pip/Dockerfile.build_from_pip
+++ b/docker/build_from_pip/Dockerfile.build_from_pip
@@ -13,10 +13,16 @@ RUN apk update && \
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
-COPY requirements.txt .
+COPY docker/build_from_pip/requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
+# Copy Prisma schema file
+COPY schema.prisma .
+
+# Generate prisma client
+RUN prisma generate
+
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
diff --git a/docker/build_from_pip/requirements.txt b/docker/build_from_pip/requirements.txt
index 71e038b6267..cc14b99727f 100644
--- a/docker/build_from_pip/requirements.txt
+++ b/docker/build_from_pip/requirements.txt
@@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
+openai==1.99.9
ddtrace==2.19.0 # for advanced DD tracing / profiling
diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh
new file mode 100755
index 00000000000..794f9a2bbce
--- /dev/null
+++ b/docker/install_auto_router.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install semantic_router==0.1.11 --no-deps
+pip install aurelio-sdk==0.0.19
\ No newline at end of file
diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh
index ea94c343801..1fc09d2c864 100644
--- a/docker/prod_entrypoint.sh
+++ b/docker/prod_entrypoint.sh
@@ -1,5 +1,10 @@
#!/bin/sh
+if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
+ export LITELLM_ARGS="$@"
+ exec supervisord -c /etc/supervisord.conf
+fi
+
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
diff --git a/docker/supervisord.conf b/docker/supervisord.conf
new file mode 100644
index 00000000000..c6855fe652b
--- /dev/null
+++ b/docker/supervisord.conf
@@ -0,0 +1,42 @@
+[supervisord]
+nodaemon=true
+loglevel=info
+
+[group:litellm]
+programs=main,health
+
+[program:main]
+command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
+autostart=true
+autorestart=true
+startretries=3
+priority=1
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[program:health]
+command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
+autostart=true
+autorestart=true
+startretries=3
+priority=2
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[eventlistener:process_monitor]
+command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+autostart=true
+autorestart=true
\ No newline at end of file
diff --git a/docs/my-website/.gitignore b/docs/my-website/.gitignore
index c5090458cda..7bc0252433b 100644
--- a/docs/my-website/.gitignore
+++ b/docs/my-website/.gitignore
@@ -10,6 +10,7 @@
# Misc
.DS_Store
+.env
.env.local
.env.development.local
.env.test.local
diff --git a/docs/my-website/docs/adding_provider/new_rerank_provider.md b/docs/my-website/docs/adding_provider/new_rerank_provider.md
index 84c363261cd..628c0994434 100644
--- a/docs/my-website/docs/adding_provider/new_rerank_provider.md
+++ b/docs/my-website/docs/adding_provider/new_rerank_provider.md
@@ -17,7 +17,7 @@ class YourProviderRerankConfig(BaseRerankConfig):
# ... other supported params
]
- def transform_rerank_request(self, model: str, optional_rerank_params: OptionalRerankParams, headers: dict) -> dict:
+ def transform_rerank_request(self, model: str, optional_rerank_params: Dict, headers: dict) -> dict:
# Transform request to RerankRequest spec
return rerank_request.model_dump(exclude_none=True)
diff --git a/docs/my-website/docs/anthropic_unified.md b/docs/my-website/docs/anthropic_unified.md
index 8a34db52482..03ba8a68847 100644
--- a/docs/my-website/docs/anthropic_unified.md
+++ b/docs/my-website/docs/anthropic_unified.md
@@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# /v1/messages [BETA]
+# /v1/messages
Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format.
@@ -14,20 +14,20 @@ Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format.
| Logging | ✅ | works across all integrations |
| End-user Tracking | ✅ | |
| Streaming | ✅ | |
-| Fallbacks | ✅ | between anthropic models |
-| Loadbalancing | ✅ | between anthropic models |
-| Support llm providers | - `anthropic` - `bedrock` (only Anthropic models) | |
-
-Planned improvement:
-- Vertex AI Anthropic support
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
## Usage
---
### LiteLLM Python SDK
+
+
+
#### Non-streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
+```python showLineNumbers title="Anthropic Example using LiteLLM Python SDK"
import litellm
response = await litellm.anthropic.messages.acreate(
messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
@@ -37,6 +37,179 @@ response = await litellm.anthropic.messages.acreate(
)
```
+#### Streaming example
+```python showLineNumbers title="Anthropic Streaming Example using LiteLLM Python SDK"
+import litellm
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ api_key=api_key,
+ model="anthropic/claude-3-haiku-20240307",
+ max_tokens=100,
+ stream=True,
+)
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
+
+#### Non-streaming example
+```python showLineNumbers title="OpenAI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="openai/gpt-4",
+ max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="OpenAI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="openai/gpt-4",
+ max_tokens=100,
+ stream=True,
+)
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
+
+#### Non-streaming example
+```python showLineNumbers title="Google Gemini Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="gemini/gemini-2.0-flash-exp",
+ max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Google Gemini Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="gemini/gemini-2.0-flash-exp",
+ max_tokens=100,
+ stream=True,
+)
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
+
+#### Non-streaming example
+```python showLineNumbers title="Vertex AI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="vertex_ai/gemini-2.0-flash-exp",
+ max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Vertex AI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="vertex_ai/gemini-2.0-flash-exp",
+ max_tokens=100,
+ stream=True,
+)
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
+
+#### Non-streaming example
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2" # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+ max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="AWS Bedrock Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2" # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+ max_tokens=100,
+ stream=True,
+)
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
Example response:
```json
{
@@ -61,22 +234,10 @@ Example response:
}
```
-#### Streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
-import litellm
-response = await litellm.anthropic.messages.acreate(
- messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
- api_key=api_key,
- model="anthropic/claude-3-haiku-20240307",
- max_tokens=100,
- stream=True,
-)
-async for chunk in response:
- print(chunk)
-```
-
### LiteLLM Proxy Server
+
+
1. Setup config.yaml
@@ -85,6 +246,7 @@ model_list:
- model_name: anthropic-claude
litellm_params:
model: claude-3-7-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start proxy
@@ -95,10 +257,7 @@ litellm --config /path/to/config.yaml
3. Test it!
-
-
-
-```python showLineNumbers title="Example using LiteLLM Proxy Server"
+```python showLineNumbers title="Anthropic Example using LiteLLM Proxy Server"
import anthropic
# point anthropic sdk to litellm proxy
@@ -113,8 +272,165 @@ response = client.messages.create(
max_tokens=100,
)
```
+
-
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: openai-gpt4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python showLineNumbers title="OpenAI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy
+client = anthropic.Anthropic(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+response = client.messages.create(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="openai-gpt4",
+ max_tokens=100,
+)
+```
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: gemini-2-flash
+ litellm_params:
+ model: gemini/gemini-2.0-flash-exp
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python showLineNumbers title="Google Gemini Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy
+client = anthropic.Anthropic(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+response = client.messages.create(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="gemini-2-flash",
+ max_tokens=100,
+)
+```
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: vertex-gemini
+ litellm_params:
+ model: vertex_ai/gemini-2.0-flash-exp
+ vertex_project: your-gcp-project-id
+ vertex_location: us-central1
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python showLineNumbers title="Vertex AI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy
+client = anthropic.Anthropic(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+response = client.messages.create(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="vertex-gemini",
+ max_tokens=100,
+)
+```
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: bedrock-claude
+ litellm_params:
+ model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-west-2
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy
+client = anthropic.Anthropic(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+response = client.messages.create(
+ messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+ model="bedrock-claude",
+ max_tokens=100,
+)
+```
+
+
+
+
```bash showLineNumbers title="Example using LiteLLM Proxy Server"
curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
@@ -136,7 +452,6 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
-
## Request Format
---
@@ -189,7 +504,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
- **system** (string or array):
A system prompt providing context or specific instructions to the model.
- **temperature** (number):
- Controls randomness in the model’s responses. Valid range: `0 < temperature < 1`.
+ Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
- **thinking** (object):
Configuration for enabling extended thinking. If enabled, it includes:
- **budget_tokens** (integer):
@@ -201,7 +516,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
- **tools** (array of objects):
Definitions for tools available to the model. Each tool includes:
- **name** (string):
- The tool’s name.
+ The tool's name.
- **description** (string):
A detailed description of the tool.
- **input_schema** (object):
diff --git a/docs/my-website/docs/assistants.md b/docs/my-website/docs/assistants.md
index 4032c74557f..d262b492a70 100644
--- a/docs/my-website/docs/assistants.md
+++ b/docs/my-website/docs/assistants.md
@@ -279,7 +279,7 @@ with run as run:
curl -X POST 'http://0.0.0.0:4000/threads/{thread_id}/runs' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
--D '{
+-d '{
"assistant_id": "asst_6xVZQFFy1Kw87NbnYeNebxTf",
"stream": true
}'
diff --git a/docs/my-website/docs/audio_transcription.md b/docs/my-website/docs/audio_transcription.md
index 22517f68e43..8cbc567180c 100644
--- a/docs/my-website/docs/audio_transcription.md
+++ b/docs/my-website/docs/audio_transcription.md
@@ -3,13 +3,22 @@ import TabItem from '@theme/TabItem';
# /audio/transcriptions
-Use this to loadbalance across Azure + OpenAI.
+## Overview
+
+| Feature | Supported | Notes |
+|-------|-------|-------|
+| Cost Tracking | ✅ | |
+| Logging | ✅ | works across all integrations |
+| End-user Tracking | ✅ | |
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
## Quick Start
### LiteLLM Python SDK
-```python showLineNumbers
+```python showLineNumbers title="Python SDK Example"
from litellm import transcription
import os
@@ -30,7 +39,7 @@ print(f"response: {response}")
-```yaml showLineNumbers
+```yaml showLineNumbers title="OpenAI Configuration"
model_list:
- model_name: whisper
litellm_params:
@@ -45,7 +54,7 @@ general_settings:
-```yaml showLineNumbers
+```yaml showLineNumbers title="OpenAI + Azure Configuration"
model_list:
- model_name: whisper
litellm_params:
@@ -71,7 +80,7 @@ general_settings:
### Start proxy
-```bash
+```bash showLineNumbers title="Start Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:8000
@@ -82,7 +91,7 @@ litellm --config /path/to/config.yaml
-```bash
+```bash showLineNumbers title="Test with cURL"
curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
--header 'Authorization: Bearer sk-1234' \
--form 'file=@"/Users/krrishdholakia/Downloads/gettysburg.wav"' \
@@ -92,7 +101,7 @@ curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
-```python showLineNumbers
+```python showLineNumbers title="Test with OpenAI Python SDK"
from openai import OpenAI
client = openai.OpenAI(
api_key="sk-1234",
@@ -115,4 +124,82 @@ transcript = client.audio.transcriptions.create(
- Azure
- [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
- [Groq](./providers/groq.md#speech-to-text---whisper)
-- [Deepgram](./providers/deepgram.md)
\ No newline at end of file
+- [Deepgram](./providers/deepgram.md)
+
+---
+
+## Fallbacks
+
+You can configure fallbacks for audio transcription to automatically retry with different models if the primary model fails.
+
+
+
+
+```bash showLineNumbers title="Test with cURL and Fallbacks"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"'
+```
+
+
+
+
+```python showLineNumbers title="Test with OpenAI Python SDK and Fallbacks"
+from openai import OpenAI
+client = OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+ model="groq/whisper-large-v3",
+ file=audio_file,
+ extra_body={
+ "fallbacks": ["openai/whisper-1"]
+ }
+)
+```
+
+
+
+### Testing Fallbacks
+
+You can test your fallback configuration using `mock_testing_fallbacks=true` to simulate failures:
+
+
+
+
+```bash showLineNumbers title="Test Fallbacks with Mock Testing"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"' \
+--form 'mock_testing_fallbacks=true'
+```
+
+
+
+
+```python showLineNumbers title="Test Fallbacks with Mock Testing"
+from openai import OpenAI
+client = OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+ model="groq/whisper-large-v3",
+ file=audio_file,
+ extra_body={
+ "fallbacks": ["openai/whisper-1"],
+ "mock_testing_fallbacks": True
+ }
+)
+```
+
+
\ No newline at end of file
diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md
index 7029697699c..1bd4c700ae7 100644
--- a/docs/my-website/docs/batches.md
+++ b/docs/my-website/docs/batches.md
@@ -7,7 +7,7 @@ Covers Batches, Files
| Feature | Supported | Notes |
|-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex | - |
+| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
| ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
| Logging | ✅ | Works across all logging integrations |
@@ -116,10 +116,38 @@ print("response from litellm.create_batch=", create_batch_response)
**Retrieve the Specific Batch and File Content**
```python
+ # Maximum wait time before we give up
+ MAX_WAIT_TIME = 300
+
+ # Time to wait between each status check
+ POLL_INTERVAL = 5
+
+ #Time waited till now
+ waited = 0
+
+ # Wait for the batch to finish processing before trying to retrieve output
+ # This loop checks the batch status every few seconds (polling)
+
+ while True:
+ retrieved_batch = await litellm.aretrieve_batch(
+ batch_id=create_batch_response.id,
+ custom_llm_provider="openai"
+ )
+
+ status = retrieved_batch.status
+ print(f"⏳ Batch status: {status}")
+
+ if status == "completed" and retrieved_batch.output_file_id:
+ print("✅ Batch complete. Output file ID:", retrieved_batch.output_file_id)
+ break
+ elif status in ["failed", "cancelled", "expired"]:
+ raise RuntimeError(f"❌ Batch failed with status: {status}")
+
+ await asyncio.sleep(POLL_INTERVAL)
+ waited += POLL_INTERVAL
+ if waited > MAX_WAIT_TIME:
+ raise TimeoutError("❌ Timed out waiting for batch to complete.")
-retrieved_batch = await litellm.aretrieve_batch(
- batch_id=create_batch_response.id, custom_llm_provider="openai"
-)
print("retrieved batch=", retrieved_batch)
# just assert that we retrieved a non None batch
@@ -150,6 +178,7 @@ print("list_batches_response=", list_batches_response)
### [Azure OpenAI](./providers/azure#azure-batches-api)
### [OpenAI](#quick-start)
### [Vertex AI](./providers/vertex#batch-apis)
+### [Bedrock](./providers/bedrock_batches)
## How Cost Tracking for Batches API Works
diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md
index 817d70b87c2..43ab82b8e61 100644
--- a/docs/my-website/docs/benchmarks.md
+++ b/docs/my-website/docs/benchmarks.md
@@ -18,13 +18,17 @@ model_list:
### 1 Instance LiteLLM Proxy
-In these tests the median latency of directly calling the fake-openai-endpoint is 60ms.
+In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
-| Metric | Litellm Proxy (1 Instance) |
-|--------|------------------------|
-| RPS | 475 |
-| Median Latency (ms) | 100 |
-| Latency overhead added by LiteLLM Proxy | 40ms |
+#### Performance Metrics
+
+| Metric | Value |
+|--------|-------|
+| **Requests per Second (RPS)** | 475 |
+| **End-to-End Latency P50 (ms)** | 100 |
+| **LiteLLM Overhead P50 (ms)** | 3 |
+| **LiteLLM Overhead P90 (ms)** | 17 |
+| **LiteLLM Overhead P99 (ms)** | 31 |
@@ -33,7 +37,8 @@ In these tests the median latency of directly calling the fake-openai-endpoint i
-->
#### Key Findings
-- Single instance: 475 RPS @ 100ms latency
+- Single instance: 475 RPS @ 100ms median latency
+- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
- 2 LiteLLM instances: 950 RPS @ 100ms latency
- 4 LiteLLM instances: 1900 RPS @ 100ms latency
@@ -54,6 +59,62 @@ Each machine deploying LiteLLM had the following specs:
- 2 CPU
- 4GB RAM
+## How to measure LiteLLM Overhead
+
+All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy.
+
+
+If you want to measure this on locust you can use the following code:
+
+```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead"
+import os
+import uuid
+from locust import HttpUser, task, between, events
+
+# Custom metric to track LiteLLM overhead duration
+overhead_durations = []
+
+@events.request.add_listener
+def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs):
+ if response and hasattr(response, 'headers'):
+ overhead_duration = response.headers.get('x-litellm-overhead-duration-ms')
+ if overhead_duration:
+ try:
+ duration_ms = float(overhead_duration)
+ overhead_durations.append(duration_ms)
+ # Report as custom metric
+ events.request.fire(
+ request_type="Custom",
+ name="LiteLLM Overhead Duration (ms)",
+ response_time=duration_ms,
+ response_length=0,
+ )
+ except (ValueError, TypeError):
+ pass
+
+class MyUser(HttpUser):
+ wait_time = between(0.5, 1) # Random wait time between requests
+
+ def on_start(self):
+ self.api_key = os.getenv('API_KEY', 'sk-1234567890')
+ self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
+
+ @task
+ def litellm_completion(self):
+ # no cache hits with this
+ payload = {
+ "model": "db-openai-endpoint",
+ "messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}],
+ "user": "my-new-end-user-1"
+ }
+ response = self.client.post("chat/completions", json=payload)
+
+ if response.status_code != 200:
+ # log the errors in error.txt
+ with open("error.txt", "a") as error_log:
+ error_log.write(response.text + "\n")
+```
+
## Logging Callbacks
diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md
index a14170beefa..0548c331f80 100644
--- a/docs/my-website/docs/caching/all_caches.md
+++ b/docs/my-website/docs/caching/all_caches.md
@@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk
+# Caching - In-Memory, Redis, s3, gcs, Redis Semantic Cache, Disk
[**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching/caching.py)
@@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem';
:::
-## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
+## Initialize Cache - In Memory, Redis, s3 Bucket, gcs Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
@@ -28,6 +28,8 @@ pip install redis
For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/
+**Basic Redis Cache**
+
```python
import litellm
from litellm import completion
@@ -48,6 +50,91 @@ response2 = completion(
# response1 == response2, response 1 is cached
```
+**GCP IAM Redis Authentication**
+
+For GCP Memorystore Redis with IAM authentication:
+
+```shell
+pip install google-cloud-iam
+```
+
+```python
+import litellm
+from litellm import completion
+# For Redis Cluster with GCP IAM
+from litellm.caching.redis_cluster_cache import RedisClusterCache
+
+litellm.cache = RedisClusterCache(
+ startup_nodes=[
+ {"host": "10.128.0.2", "port": 6379},
+ {"host": "10.128.0.2", "port": 11008},
+ ],
+ gcp_service_account="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com",
+ ssl=True,
+ ssl_cert_reqs=None,
+ ssl_check_hostname=False,
+)
+
+# Make completion calls
+response1 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
+**Environment Variables for GCP IAM Redis**
+
+You can also set these as environment variables:
+
+```shell
+export REDIS_HOST="10.128.0.2"
+export REDIS_PORT="6379"
+export REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
+export REDIS_SSL="False"
+```
+
+Then simply initialize:
+
+```python
+litellm.cache = Cache(type="redis")
+```
+
+
+
+
+
+Set environment variables
+
+```shell
+GCS_BUCKET_NAME="my-cache-bucket"
+GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json"
+```
+
+```python
+import litellm
+from litellm import completion
+from litellm.caching.caching import Cache
+
+litellm.cache = Cache(type="gcs", gcs_bucket_name="my-cache-bucket", gcs_path_service_account="/path/to/service_account.json")
+
+response1 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
@@ -88,6 +175,37 @@ response2 = completion(
+
+
+Install azure-storage-blob and azure-identity
+```shell
+pip install azure-storage-blob azure-identity
+```
+
+```python
+import litellm
+from litellm import completion
+from litellm.caching.caching import Cache
+from azure.identity import DefaultAzureCredential
+
+# pass Azure Blob Storage account URL and container name
+litellm.cache = Cache(type="azure-blob", azure_account_url="https://example.blob.core.windows.net", azure_blob_container="litellm")
+
+# Make completion calls
+response1 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
+
+
@@ -236,10 +354,10 @@ response2 = completion(
### Quick Start
-Install diskcache:
+Install the disk caching extra:
```shell
-pip install diskcache
+pip install "litellm[caching]"
```
Then you can use the disk cache as follows.
@@ -374,7 +492,7 @@ Advanced Params
```python
litellm.enable_cache(
- type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+ type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
host: Optional[str] = None,
port: Optional[str] = None,
password: Optional[str] = None,
@@ -398,7 +516,7 @@ Update the Cache params
```python
litellm.update_cache(
- type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+ type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
host: Optional[str] = None,
port: Optional[str] = None,
password: Optional[str] = None,
@@ -459,7 +577,7 @@ cache.get_cache = get_cache
```python
def __init__(
self,
- type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local",
+ type: Optional[Literal["local", "redis", "redis-semantic", "s3", "gcs", "disk"]] = "local",
supported_call_types: Optional[
List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]]
] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"],
@@ -473,6 +591,13 @@ def __init__(
namespace: Optional[str] = None,
default_in_redis_ttl: Optional[float] = None,
redis_flush_size=None,
+
+ # GCP IAM Redis authentication params
+ gcp_service_account: Optional[str] = None,
+ gcp_ssl_ca_certs: Optional[str] = None,
+ ssl: Optional[bool] = None,
+ ssl_cert_reqs: Optional[Union[str, None]] = None,
+ ssl_check_hostname: Optional[bool] = None,
# redis semantic cache params
similarity_threshold: Optional[float] = None,
diff --git a/docs/my-website/docs/completion/computer_use.md b/docs/my-website/docs/completion/computer_use.md
new file mode 100644
index 00000000000..ed09a73b219
--- /dev/null
+++ b/docs/my-website/docs/completion/computer_use.md
@@ -0,0 +1,446 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Computer Use
+
+Computer use allows models to interact with computer interfaces by taking screenshots and performing actions like clicking, typing, and scrolling. This enables AI models to autonomously operate desktop environments.
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+- Bedrock (Anthropic) (`bedrock/`)
+- Vertex AI (Anthropic) (`vertex_ai/`)
+
+**Supported Tool Types:**
+- `computer` - Computer interaction tool with display parameters
+- `bash` - Bash shell tool
+- `text_editor` - Text editor tool
+- `web_search` - Web search tool
+
+LiteLLM will standardize the computer use tools across all supported providers.
+
+## Quick Start
+
+
+
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Computer use tool
+ tools = [
+ {
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_height_px": 768,
+ "display_width_px": 1024,
+ "display_number": 0,
+ }
+ ]
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Take a screenshot and tell me what you see"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+ }
+ }
+ ]
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+
+
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+ - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
+ - model_name: claude-bedrock # Bedrock Anthropic model
+ litellm_params:
+ model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-west-2
+ model_info:
+ supports_computer_use: True # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="sk-1234", # your litellm proxy api key
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-3-5-sonnet-latest",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Take a screenshot and tell me what you see"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+ }
+ }
+ ]
+ }
+ ],
+ tools=[
+ {
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_height_px": 768,
+ "display_width_px": 1024,
+ "display_number": 0,
+ }
+ ]
+)
+
+print(response)
+```
+
+
+
+
+## Checking if a model supports `computer use`
+
+
+
+
+Use `litellm.supports_computer_use(model="")` -> returns `True` if model supports computer use and `False` if not
+
+```python
+import litellm
+
+assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
+assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
+assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
+assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
+assert litellm.supports_computer_use(model="openai/gpt-4") == False
+```
+
+
+
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+ - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
+ - model_name: claude-bedrock # Bedrock Anthropic model
+ litellm_params:
+ model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-west-2
+ model_info:
+ supports_computer_use: True # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Call `/model_group/info` to check if your model supports `computer use`
+
+```shell
+curl -X 'GET' \
+ 'http://localhost:4000/model_group/info' \
+ -H 'accept: application/json' \
+ -H 'x-api-key: sk-1234'
+```
+
+Expected Response
+
+```json
+{
+ "data": [
+ {
+ "model_group": "claude-3-5-sonnet-latest",
+ "providers": ["anthropic"],
+ "max_input_tokens": 200000,
+ "max_output_tokens": 8192,
+ "mode": "chat",
+ "supports_computer_use": true, # 👈 supports_computer_use is true
+ "supports_vision": true,
+ "supports_function_calling": true
+ },
+ {
+ "model_group": "claude-bedrock",
+ "providers": ["bedrock"],
+ "max_input_tokens": 200000,
+ "max_output_tokens": 8192,
+ "mode": "chat",
+ "supports_computer_use": true, # 👈 supports_computer_use is true
+ "supports_vision": true,
+ "supports_function_calling": true
+ }
+ ]
+}
+```
+
+
+
+
+## Different Tool Types
+
+Computer use supports several different tool types for various interaction modes:
+
+
+
+
+The `computer_20241022` tool provides direct screen interaction capabilities.
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_height_px": 768,
+ "display_width_px": 1024,
+ "display_number": 0,
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Click on the search button in the screenshot"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+ }
+ }
+ ]
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+
+
+
+The `bash_20241022` tool provides command line interface access.
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "bash_20241022",
+ "name": "bash"
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "List the files in the current directory using bash"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+
+
+
+The `text_editor_20250124` tool provides text file editing capabilities.
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "text_editor_20250124",
+ "name": "str_replace_editor"
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Create a simple Python hello world script"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+
+
+
+## Advanced Usage with Multiple Tools
+
+You can combine different computer use tools in a single request:
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_height_px": 768,
+ "display_width_px": 1024,
+ "display_number": 0,
+ },
+ {
+ "type": "bash_20241022",
+ "name": "bash"
+ },
+ {
+ "type": "text_editor_20250124",
+ "name": "str_replace_editor"
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Take a screenshot, then create a file describing what you see, and finally use bash to show the file contents"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+ }
+ }
+ ]
+ }
+ ]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Computer Tool (`computer_20241022`)
+
+```json
+{
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_height_px": 768, // Required: Screen height in pixels
+ "display_width_px": 1024, // Required: Screen width in pixels
+ "display_number": 0 // Optional: Display number (default: 0)
+}
+```
+
+### Bash Tool (`bash_20241022`)
+
+```json
+{
+ "type": "bash_20241022",
+ "name": "bash" // Required: Tool name
+}
+```
+
+### Text Editor Tool (`text_editor_20250124`)
+
+```json
+{
+ "type": "text_editor_20250124",
+ "name": "str_replace_editor" // Required: Tool name
+}
+```
+
+### Web Search Tool (`web_search_20250305`)
+
+```json
+{
+ "type": "web_search_20250305",
+ "name": "web_search" // Required: Tool name
+}
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/completion/document_understanding.md b/docs/my-website/docs/completion/document_understanding.md
index 04047a5909a..172e0792801 100644
--- a/docs/my-website/docs/completion/document_understanding.md
+++ b/docs/my-website/docs/completion/document_understanding.md
@@ -9,6 +9,8 @@ Works for:
- Vertex AI models (Gemini + Anthropic)
- Bedrock Models
- Anthropic API Models
+- OpenAI API Models
+- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
## Quick Start
@@ -278,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+## Mistral Example
+
+Here is a sample payload for using the Mistral model for document understanding:
+
+
+
+
+
+```python
+from litellm.utils import completion
+
+# pdf file_id received from files endpoint
+file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
+
+# model
+model = "mistral/mistral-large-latest"
+
+file_content = [
+ {"type": "text", "text": "What's this file about?"},
+ {
+ "type": "file",
+ "file": {
+ "file_id": file_id,
+ }
+ },
+]
+
+response = completion(
+ model=model,
+ messages=[{"role": "user", "content": file_content}],
+)
+assert response is not None
+```
+
+
+
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model": "mistral/mistral-large-latest",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What is the content of the file?"
+ },
+ {
+ "type": "file",
+ "file": {
+ "file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
+ }
+ }
+ ]
+ }
+ ]
+}
+```
+
+
+
## Checking if a model supports pdf input
diff --git a/docs/my-website/docs/completion/http_handler_config.md b/docs/my-website/docs/completion/http_handler_config.md
new file mode 100644
index 00000000000..d4a25ce2043
--- /dev/null
+++ b/docs/my-website/docs/completion/http_handler_config.md
@@ -0,0 +1,145 @@
+# Custom HTTP Handler
+
+Configure custom aiohttp sessions for better performance and control in LiteLLM completions.
+
+## Overview
+
+You can now inject custom `aiohttp.ClientSession` instances into LiteLLM for:
+- Custom connection pooling and timeouts
+- Corporate proxy and SSL configurations
+- Performance optimization
+- Request monitoring
+
+## Basic Usage
+
+### Default (No Changes Required)
+```python
+import litellm
+
+# Works exactly as before
+response = await litellm.acompletion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Custom Session
+```python
+import aiohttp
+import litellm
+from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+# Create optimized session
+session = aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=180),
+ connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+)
+
+# Replace global handler
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+
+# All completions now use your session
+response = await litellm.acompletion(model="gpt-3.5-turbo", messages=[...])
+```
+
+## Common Patterns
+
+### FastAPI Integration
+```python
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ # Startup
+ session = aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=180),
+ connector=aiohttp.TCPConnector(limit=300)
+ )
+ litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(
+ client_session=session
+ )
+ yield
+ # Shutdown
+ await session.close()
+
+app = FastAPI(lifespan=lifespan)
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+ return await litellm.acompletion(model="gpt-3.5-turbo", messages=messages)
+```
+
+### Corporate Proxy
+```python
+import ssl
+
+# Custom SSL context
+ssl_context = ssl.create_default_context()
+ssl_context.load_cert_chain('cert.pem', 'key.pem')
+
+# Proxy session
+session = aiohttp.ClientSession(
+ connector=aiohttp.TCPConnector(ssl=ssl_context),
+ trust_env=True # Use environment proxy settings
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+### High Performance
+```python
+# Optimized for high throughput
+session = aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=300),
+ connector=aiohttp.TCPConnector(
+ limit=1000, # High connection limit
+ limit_per_host=200, # Per host limit
+ ttl_dns_cache=600, # DNS cache
+ keepalive_timeout=60, # Keep connections alive
+ enable_cleanup_closed=True
+ )
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+## Constructor Options
+
+```python
+BaseLLMAIOHTTPHandler(
+ client_session=None, # Custom aiohttp.ClientSession
+ transport=None, # Advanced transport control
+ connector=None, # Custom aiohttp.BaseConnector
+)
+```
+
+## Resource Management
+
+- **User sessions**: You manage the lifecycle (call `await session.close()`)
+- **Auto-created sessions**: Automatically cleaned up by the handler
+- **100% backward compatible**: Existing code works unchanged
+
+## Configuration Tips
+
+### Development
+```python
+session = aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=60),
+ connector=aiohttp.TCPConnector(limit=50)
+)
+```
+
+### Production
+```python
+session = aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=300),
+ connector=aiohttp.TCPConnector(
+ limit=1000,
+ limit_per_host=200,
+ keepalive_timeout=60
+ )
+)
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/completion/image_generation_chat.md b/docs/my-website/docs/completion/image_generation_chat.md
new file mode 100644
index 00000000000..58ae70e2fff
--- /dev/null
+++ b/docs/my-website/docs/completion/image_generation_chat.md
@@ -0,0 +1,232 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Image Generation in Chat Completions, Responses API
+
+This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
+
+:::info
+
+Requires LiteLLM v1.76.1+
+
+:::
+
+Supported Providers:
+- Google AI Studio (`gemini`)
+- Vertex AI (`vertex_ai/`)
+
+LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
+
+```python title="Example response from litellm"
+"message": {
+ ...
+ "content": "Here's the image you requested:",
+ "image": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+ "detail": "auto"
+ }
+}
+```
+
+## Quick Start
+
+
+
+
+```python showLineNumbers title="Image generation with chat completion"
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+ model="gemini/gemini-2.5-flash-image-preview",
+ messages=[
+ {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+ ],
+)
+
+print(response.choices[0].message.content) # Text response
+print(response.choices[0].message.image) # Image data
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gemini-image-gen
+ litellm_params:
+ model: gemini/gemini-2.5-flash-image-preview
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Run proxy server
+
+```bash showLineNumbers title="Start the proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash showLineNumbers title="Make request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "gemini-image-gen",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+ }
+ ]
+ }'
+```
+
+
+
+
+**Expected Response**
+
+```bash
+{
+ "id": "chatcmpl-3b66124d79a708e10c603496b363574c",
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "content": "Here's the image you requested:",
+ "role": "assistant",
+ "image": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+ "detail": "auto"
+ }
+ }
+ }
+ ],
+ "created": 1723323084,
+ "model": "gemini/gemini-2.5-flash-image-preview",
+ "object": "chat.completion",
+ "usage": {
+ "completion_tokens": 12,
+ "prompt_tokens": 16,
+ "total_tokens": 28
+ }
+}
+```
+
+## Streaming Support
+
+
+
+
+```python showLineNumbers title="Streaming image generation"
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+ model="gemini/gemini-2.5-flash-image-preview",
+ messages=[
+ {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+ ],
+ stream=True,
+)
+
+for chunk in response:
+ if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
+ print("Generated image:", chunk.choices[0].delta.image["url"])
+ break
+```
+
+
+
+
+```bash showLineNumbers title="Streaming request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "gemini-image-gen",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+ }
+ ],
+ "stream": true
+ }'
+```
+
+
+
+
+**Expected Streaming Response**
+
+```bash
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+## Async Support
+
+```python showLineNumbers title="Async image generation"
+from litellm import acompletion
+import asyncio
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+async def generate_image():
+ response = await acompletion(
+ model="gemini/gemini-2.5-flash-image-preview",
+ messages=[
+ {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+ ],
+ )
+
+ print(response.choices[0].message.content) # Text response
+ print(response.choices[0].message.image) # Image data
+
+ return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+## Supported Models
+
+| Provider | Model |
+|----------|--------|
+| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
+| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
+
+## Spec
+
+The `image` field in the response follows this structure:
+
+```python
+"image": {
+ "url": "data:image/png;base64,",
+ "detail": "auto"
+}
+```
+
+- `url` - str: Base64 encoded image data in data URI format
+- `detail` - str: Image detail level (always "auto" for generated images)
+
+The image is returned as a base64-encoded data URI that can be directly used in HTML `` tags or saved to a file.
diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md
index fb0fc390ad0..91d9cc72cf3 100644
--- a/docs/my-website/docs/completion/input.md
+++ b/docs/my-website/docs/completion/input.md
@@ -39,31 +39,34 @@ This is a list of openai params we translate across providers.
Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider
-| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
-|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
-|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | | ✅ | ✅ | | | ✅ |
-|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
-|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
-|xAI| ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
-|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
-|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-|Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |
-|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
-|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ | | | |
-|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |
-|VertexAI| ✅ | ✅ | ✅ | | ✅ | ✅ | | | | | | | | | ✅ | ✅ | | |
-|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (model dependent) | |
-|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
-|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | | | ✅ | | ✅ | ✅ | | | |
-|Sambanova| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | | ✅ | ✅ | | | |
-|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
-|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
-|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | |
-|Ollama| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | |✅| | | | | | |
-|Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | |
-|ClarifAI| ✅ | ✅ | ✅ | |✅ | ✅ | | | | | | | | | | |
-|Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | |
-|Novita AI| ✅ | ✅ | | ✅ | ✅ | ✅ | | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | | | |
+| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed| tools | tool_choice | logprobs | top_logprobs | extra_headers |
+|--------------|-------------|------------------------|------------|-------|--------|----------------|------|-----|------------------|-------------------|-----------|----------------|-------------|------|------------------|-------------------|--------|--------------|----------|---------------|----------------------|
+| Anthropic| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || | ✅ | ✅ | | ✅ | ✅ || | ✅|
+| OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| Azure OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| xAI| ✅|| ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅||
+| Replicate| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Anyscale | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Cohere | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| Huggingface| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| Openrouter | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| ||| ✅| ✅ ||| ||
+| AI21 | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| VertexAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || || ✅ | ✅|||| ||
+| Bedrock| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || || ✅ (model dependent) | |||| ||
+| Sagemaker| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| TogetherAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | ✅|| || ✅ | | ✅ | ✅ || ||
+| Sambanova| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || || ✅ | | ✅ | ✅ || ||
+| AlephAlpha | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| NLP Cloud| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Petals | ✅| ✅ || ✅| ✅ ||| || | || ||| |||| ||
+| Ollama | ✅| ✅ | ✅ | ✅| ✅ | ✅ || ✅|| | || ✅||| | ✅ ||| ||
+| Databricks | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| ClarifAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || ||| |||| ||
+| Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
+| Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
+| Bytez | ✅| ✅ || ✅| ✅ | | | ✅|| || || || || || ||
+| OVHCloud AI Endpoints | ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
+
:::note
By default, LiteLLM raises an exception if the openai param being passed in isn't supported.
@@ -104,6 +107,7 @@ def completion(
parallel_tool_calls: Optional[bool] = None,
logprobs: Optional[bool] = None,
top_logprobs: Optional[int] = None,
+ safety_identifier: Optional[str] = None,
deployment_id=None,
# soon to be deprecated params by OpenAI
functions: Optional[List] = None,
@@ -194,6 +198,8 @@ def completion(
- `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
+- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking.
+
- `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
- `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request.
diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md
index 033dccea200..ee0e3086785 100644
--- a/docs/my-website/docs/completion/knowledgebase.md
+++ b/docs/my-website/docs/completion/knowledgebase.md
@@ -17,6 +17,9 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
## Supported Vector Stores
- [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/)
+- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
+- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores)
+- [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
## Quick Start
@@ -157,6 +160,129 @@ print(response.choices[0].message.content)
+## Provider Specific Guides
+
+This section covers how to add your vector stores to LiteLLM. If you want support for a new provider, please file an issue [here](https://github.com/BerriAI/litellm/issues).
+
+### Bedrock Knowledge Bases
+
+**1. Set up your Bedrock Knowledge Base**
+
+Ensure you have a Bedrock Knowledge Base created in your AWS account with the appropriate permissions configured.
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"Bedrock"** as the provider
+3. Enter your Bedrock Knowledge Base ID in the **"Vector Store ID"** field
+
+
+
+
+### Vertex AI RAG Engine
+
+**1. Get your Vertex AI RAG Engine ID**
+
+1. Navigate to your RAG Engine Corpus in the [Google Cloud Console](https://console.cloud.google.com/vertex-ai/rag/corpus)
+2. Select the **RAG Engine** you want to integrate with LiteLLM
+
+
+
+
+
+3. Click the **"Details"** button and copy the UUID for the RAG Engine
+4. The ID should look like: `6917529027641081856`
+
+
+
+
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"Vertex AI RAG Engine"** as the provider
+3. Enter your Vertex AI RAG Engine ID in the **"Vector Store ID"** field
+
+
+
+
+
+### PG Vector
+
+**1. Deploy the litellm-pg-vector-store connector**
+
+LiteLLM provides a server that exposes OpenAI-compatible `vector_store` endpoints for PG Vector. The LiteLLM Proxy server connects to your deployed service and uses it as a vector store when querying.
+
+1. Follow the deployment instructions for the litellm-pg-vector-store connector [here](https://github.com/BerriAI/litellm-pgvector)
+2. For detailed configuration options, see the [configuration guide](https://github.com/BerriAI/litellm-pgvector?tab=readme-ov-file#configuration)
+
+**Example .env configuration for deploying litellm-pg-vector-store:**
+
+```env
+DATABASE_URL="postgresql://neondb_owner:xxxx"
+SERVER_API_KEY="sk-1234"
+HOST="0.0.0.0"
+PORT=8001
+EMBEDDING__MODEL="text-embedding-ada-002"
+EMBEDDING__BASE_URL="http://localhost:4000"
+EMBEDDING__API_KEY="sk-1234"
+EMBEDDING__DIMENSIONS=1536
+DB_FIELDS__ID_FIELD="id"
+DB_FIELDS__CONTENT_FIELD="content"
+DB_FIELDS__METADATA_FIELD="metadata"
+DB_FIELDS__EMBEDDING_FIELD="embedding"
+DB_FIELDS__VECTOR_STORE_ID_FIELD="vector_store_id"
+DB_FIELDS__CREATED_AT_FIELD="created_at"
+```
+
+**2. Add to LiteLLM UI**
+
+Once your litellm-pg-vector-store is deployed:
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"PG Vector"** as the provider
+3. Enter your **API Base URL** and **API Key** for your `litellm-pg-vector-store` container
+ - The API Key field corresponds to the `SERVER_API_KEY` from your .env configuration
+
+
+
+
+
+### OpenAI Vector Stores
+
+**1. Set up your OpenAI Vector Store**
+
+1. Create your Vector Store on the [OpenAI platform](https://platform.openai.com/storage/vector_stores)
+2. Note your Vector Store ID (format: `vs_687ae3b2439881918b433cb99d10662e`)
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"OpenAI"** as the provider
+3. Enter your **Vector Store ID** in the corresponding field
+4. Enter your **OpenAI API Key** in the API Key field
+
+
+
+
diff --git a/docs/my-website/docs/completion/provider_specific_params.md b/docs/my-website/docs/completion/provider_specific_params.md
index a8307fc8a20..250b410c9c4 100644
--- a/docs/my-website/docs/completion/provider_specific_params.md
+++ b/docs/my-website/docs/completion/provider_specific_params.md
@@ -423,7 +423,7 @@ model_list:
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
--D '{
+-d '{
"model": "llama-3-8b-instruct",
"messages": [
{
@@ -431,6 +431,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
"content": "What'\''s the weather like in Boston today?"
}
],
- "adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
- }'
-```
\ No newline at end of file
+ "adapater_id": "my-special-adapter-id"
+}'
+```
+
+## Provider-Specific Metadata Parameters
+
+| Provider | Parameter | Use Case |
+|----------|-----------|----------|
+| **AWS Bedrock** | `requestMetadata` | Cost attribution, logging |
+| **Gemini/Vertex AI** | `labels` | Resource labeling |
+| **Anthropic** | `metadata` | User identification |
+
+
+
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
+ messages=[{"role": "user", "content": "Hello!"}],
+ requestMetadata={"cost_center": "engineering"}
+)
+```
+
+
+
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="vertex_ai/gemini-pro",
+ messages=[{"role": "user", "content": "Hello!"}],
+ labels={"environment": "production"}
+)
+```
+
+
+
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="anthropic/claude-3-sonnet-20240229",
+ messages=[{"role": "user", "content": "Hello!"}],
+ metadata={"user_id": "user123"}
+)
+```
+
+
+
\ No newline at end of file
diff --git a/docs/my-website/docs/completion/shared_session.md b/docs/my-website/docs/completion/shared_session.md
new file mode 100644
index 00000000000..ff3da37f34f
--- /dev/null
+++ b/docs/my-website/docs/completion/shared_session.md
@@ -0,0 +1,213 @@
+# Shared Session Support
+
+## Overview
+
+LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
+
+## Usage
+
+### Basic Usage
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def main():
+ # Create a shared session
+ async with ClientSession() as shared_session:
+ # Use the same session for multiple calls
+ response1 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=shared_session
+ )
+
+ response2 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "How are you?"}],
+ shared_session=shared_session
+ )
+
+ # Both calls reuse the same session!
+
+asyncio.run(main())
+```
+
+### Without Shared Session (Default)
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def main():
+ # Each call creates a new session
+ response1 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+
+ response2 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "How are you?"}]
+ )
+ # Two separate sessions created
+
+asyncio.run(main())
+```
+
+## Benefits
+
+- **Performance**: Reuse HTTP connections across multiple calls
+- **Resource Efficiency**: Reduce memory and connection overhead
+- **Better Control**: Manage session lifecycle explicitly
+- **Debugging**: Easy to trace which calls use which sessions
+
+## Debug Logging
+
+Enable debug logging to see session reuse in action:
+
+```python
+import os
+import litellm
+
+# Enable debug logging
+os.environ['LITELLM_LOG'] = 'DEBUG'
+
+# You'll see logs like:
+# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
+# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
+```
+
+## Common Patterns
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+app = FastAPI()
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+ # Create session per request
+ async with aiohttp.ClientSession() as session:
+ return await litellm.acompletion(
+ model="gpt-4o",
+ messages=messages,
+ shared_session=session
+ )
+```
+
+### Batch Processing
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def process_batch(messages_list):
+ async with ClientSession() as shared_session:
+ tasks = []
+ for messages in messages_list:
+ task = acompletion(
+ model="gpt-4o",
+ messages=messages,
+ shared_session=shared_session
+ )
+ tasks.append(task)
+
+ # All tasks use the same session
+ results = await asyncio.gather(*tasks)
+ return results
+```
+
+### Custom Session Configuration
+
+```python
+import aiohttp
+import litellm
+
+# Create optimized session
+async with aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=180),
+ connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+) as shared_session:
+
+ response = await litellm.acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=shared_session
+ )
+```
+
+## Implementation Details
+
+The `shared_session` parameter is threaded through the entire LiteLLM call chain:
+
+1. **`acompletion()`** - Accepts `shared_session` parameter
+2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
+3. **`AsyncHTTPHandler`** - Uses existing session if provided
+4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
+
+## Backward Compatibility
+
+- **100% backward compatible** - Existing code works unchanged
+- **Optional parameter** - `shared_session=None` by default
+- **No breaking changes** - All existing functionality preserved
+
+## Testing
+
+Test the shared session functionality:
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def test_shared_session():
+ async with ClientSession() as session:
+ print(f"✅ Created session: {id(session)}")
+
+ try:
+ response = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=session,
+ api_key="your-api-key"
+ )
+ print(f"Response: {response.choices[0].message.content}")
+ except Exception as e:
+ print(f"✅ Expected error: {type(e).__name__}")
+
+ print("✅ Session control working!")
+
+asyncio.run(test_shared_session())
+```
+
+## Files Modified
+
+The shared session functionality was added to these files:
+
+- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
+- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
+- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
+- `litellm/llms/openai/openai.py` - OpenAI provider integration
+- `litellm/llms/openai/common_utils.py` - OpenAI client creation
+- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
+
+## Troubleshooting
+
+### Session Not Being Reused
+
+1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
+2. **Verify session is not closed**: Ensure the session is still active when making calls
+3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
+
+### Performance Issues
+
+1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
+2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
+3. **Timeout settings**: Configure appropriate timeouts for your environment
diff --git a/docs/my-website/docs/completion/usage.md b/docs/my-website/docs/completion/usage.md
index 2a9eab941ea..c388e5bfee1 100644
--- a/docs/my-website/docs/completion/usage.md
+++ b/docs/my-website/docs/completion/usage.md
@@ -26,6 +26,7 @@ response = completion(
print(response.usage)
```
+> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
## Streaming Usage
diff --git a/docs/my-website/docs/completion/web_fetch.md b/docs/my-website/docs/completion/web_fetch.md
new file mode 100644
index 00000000000..30a15e44495
--- /dev/null
+++ b/docs/my-website/docs/completion/web_fetch.md
@@ -0,0 +1,294 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Web Fetch
+
+The web fetch tool allows LLMs to retrieve full content from specified web pages and PDF documents. This enables AI models to access real-time information from the internet and incorporate web content into their responses.
+
+## Web Fetch vs Web Search
+
+**Web Fetch** retrieves the full content from specific web pages that you provide URLs for, while **Web Search** performs internet searches to find relevant information based on your queries.
+
+| Feature | Web Fetch | Web Search |
+|---------|-----------|------------|
+| **Purpose** | Retrieve content from specific URLs | Search the internet for information |
+| **Input** | You provide exact URLs to fetch | You provide search queries/questions |
+| **Output** | Full page content from specified URLs | Search results with relevant information |
+| **Use Cases** | - Analyzing specific articles - Comparing content from known websites - Extracting data from particular pages | - Finding current news/events - Researching topics - Getting real-time information |
+
+
+**Example Web Fetch**: "Fetch the content from https://example.com/pricing and summarize it"
+**Example Web Search**: "What are the latest AI developments this week?"
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+
+**Supported Tool Types:**
+- `web_fetch_20250910` - Web content retrieval tool with usage limits, domain filtering, and citation support
+
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Web fetch tool
+tools = [
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 5,
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Please analyze the content at https://example.com/article and summarize the main points"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+### LiteLLM Proxy
+
+1. Define web fetch models on config.yaml
+
+```yaml
+model_list:
+ - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="sk-1234", # your litellm proxy api key
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-3-5-sonnet-latest",
+ messages=[
+ {
+ "role": "user",
+ "content": "Please fetch and analyze the content from https://news.ycombinator.com and tell me about the top stories"
+ }
+ ],
+ tools=[
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 5,
+ }
+ ]
+)
+
+print(response)
+```
+
+## Supported Models
+
+Web fetch is available on the following Anthropic API models:
+
+- `claude-opus-4-1-20250805` (Claude Opus 4.1)
+- `claude-opus-4-20250514` (Claude Opus 4)
+- `claude-sonnet-4-20250514` (Claude Sonnet 4)
+- `claude-3-7-sonnet-20250219` (Claude Sonnet 3.7)
+- `claude-3-5-sonnet-latest` (Claude Sonnet 3.5 v2 - deprecated)
+- `claude-3-5-haiku-latest` (Claude Haiku 3.5)
+
+:::note
+The web fetch tool currently does not support websites dynamically rendered via JavaScript.
+:::
+
+## Usage Examples
+
+### Basic Web Content Retrieval
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 3,
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Fetch the latest news from https://techcrunch.com and summarize the top 3 articles"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+### Research and Analysis
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 10,
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Research the latest developments in AI by fetching content from multiple tech news websites and provide a comprehensive analysis"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+### Content Comparison
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 5,
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Compare the pricing information from https://openai.com/pricing and https://anthropic.com/pricing and create a comparison table"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+## Advanced Usage with Multiple Tools
+
+You can combine web fetch with other tools like computer use or text editor:
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+ {
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+ "max_uses": 5,
+ },
+ {
+ "type": "text_editor_20250124",
+ "name": "str_replace_editor"
+ }
+]
+
+messages = [
+ {
+ "role": "user",
+ "content": "Fetch the latest AI research papers from arXiv, analyze them, and create a detailed report file with your findings"
+ }
+]
+
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages,
+ tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Web Fetch Tool (`web_fetch_20250910`)
+
+The web fetch tool supports the following parameters:
+
+```json
+{
+ "type": "web_fetch_20250910",
+ "name": "web_fetch",
+
+ // Optional: Limit the number of fetches per request
+ "max_uses": 10,
+
+ // Optional: Only fetch from these domains
+ "allowed_domains": ["example.com", "docs.example.com"],
+
+ // Optional: Never fetch from these domains
+ "blocked_domains": ["private.example.com"],
+
+ // Optional: Enable citations for fetched content
+ "citations": {
+ "enabled": true
+ },
+
+ // Optional: Maximum content length in tokens
+ "max_content_tokens": 100000
+}
+```
+
diff --git a/docs/my-website/docs/completion/web_search.md b/docs/my-website/docs/completion/web_search.md
index 7a67dc265e4..b0d8fcdf4c0 100644
--- a/docs/my-website/docs/completion/web_search.md
+++ b/docs/my-website/docs/completion/web_search.md
@@ -1,17 +1,32 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# Using Web Search
+# Web Search
Use web search with litellm
| Feature | Details |
|---------|---------|
| Supported Endpoints | - `/chat/completions` - `/responses` |
-| Supported Providers | `openai` |
+| Supported Providers | `openai`, `xai`, `vertex_ai`, `anthropic`, `gemini`, `perplexity` |
| LiteLLM Cost Tracking | ✅ Supported |
-| LiteLLM Version | `v1.63.15-nightly` or higher |
+| LiteLLM Version | `v1.71.0+` |
+## Which Search Engine is Used?
+
+Each provider uses their own search backend:
+
+| Provider | Search Engine | Notes |
+|----------|---------------|-------|
+| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
+| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
+| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
+| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
+| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
+
+:::info
+**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
+:::
## `/chat/completions` (litellm.completion)
@@ -31,8 +46,12 @@ response = completion(
"content": "What was a positive news story from today?",
}
],
+ web_search_options={
+ "search_context_size": "medium" # Options: "low", "medium", "high"
+ }
)
```
+
@@ -40,10 +59,36 @@ response = completion(
```yaml
model_list:
+ # OpenAI
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
+
+ # xAI
+ - model_name: grok-3
+ litellm_params:
+ model: xai/grok-3
+ api_key: os.environ/XAI_API_KEY
+
+ # Anthropic
+ - model_name: claude-3-5-sonnet-latest
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ # VertexAI
+ - model_name: gemini-2-flash
+ litellm_params:
+ model: gemini-2.0-flash
+ vertex_project: your-project-id
+ vertex_location: us-central1
+
+ # Google AI Studio
+ - model_name: gemini-2-flash-studio
+ litellm_params:
+ model: gemini/gemini-2.0-flash
+ api_key: os.environ/GOOGLE_API_KEY
```
2. Start the proxy
@@ -64,7 +109,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
- model="gpt-4o-search-preview",
+ model="grok-3", # or any other web search enabled model
messages=[
{
"role": "user",
@@ -81,6 +126,7 @@ response = client.chat.completions.create(
+**OpenAI (using web_search_options)**
```python showLineNumbers
from litellm import completion
@@ -98,6 +144,69 @@ response = completion(
}
)
```
+
+**xAI (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for xAI
+response = completion(
+ model="xai/grok-3",
+ messages=[
+ {
+ "role": "user",
+ "content": "What was a positive news story from today?",
+ }
+ ],
+ web_search_options={
+ "search_context_size": "high" # Options: "low", "medium" (default), "high"
+ }
+)
+```
+
+**Anthropic (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Anthropic
+response = completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=[
+ {
+ "role": "user",
+ "content": "What was a positive news story from today?",
+ }
+ ],
+ web_search_options={
+ "search_context_size": "medium", # Options: "low", "medium" (default), "high"
+ "user_location": {
+ "type": "approximate",
+ "approximate": {
+ "city": "San Francisco",
+ },
+ }
+ }
+)
+```
+
+**VertexAI/Gemini (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Gemini
+response = completion(
+ model="gemini-2.0-flash",
+ messages=[
+ {
+ "role": "user",
+ "content": "What was a positive news story from today?",
+ }
+ ],
+ web_search_options={
+ "search_context_size": "low" # Options: "low", "medium" (default), "high"
+ }
+)
+```
@@ -112,7 +221,7 @@ client = OpenAI(
# Customize search context size
response = client.chat.completions.create(
- model="gpt-4o-search-preview",
+ model="grok-3", # works with any web search enabled model
messages=[
{
"role": "user",
@@ -127,6 +236,8 @@ response = client.chat.completions.create(
+
+
## `/responses` (litellm.responses)
### Quick Start
@@ -243,35 +354,130 @@ print(response.output_text)
+## Configuring Web Search in config.yaml
+You can set default web search options directly in your proxy config file:
+
+
+```yaml
+model_list:
+ # Enable web search by default for all requests to this model
+ - model_name: grok-3
+ litellm_params:
+ model: xai/grok-3
+ api_key: os.environ/XAI_API_KEY
+ web_search_options: {} # Enables web search with default settings
+```
+
+
+
+```yaml
+model_list:
+ # Set custom web search context size
+ - model_name: grok-3
+ litellm_params:
+ model: xai/grok-3
+ api_key: os.environ/XAI_API_KEY
+ web_search_options:
+ search_context_size: "high" # Options: "low", "medium", "high"
+
+ # Different context size for different models
+ - model_name: gpt-4o-search-preview
+ litellm_params:
+ model: openai/gpt-4o-search-preview
+ api_key: os.environ/OPENAI_API_KEY
+ web_search_options:
+ search_context_size: "low"
+
+ # Gemini with medium context (default)
+ - model_name: gemini-2-flash
+ litellm_params:
+ model: gemini-2.0-flash
+ vertex_project: your-project-id
+ vertex_location: us-central1
+ web_search_options:
+ search_context_size: "medium"
+```
+
+
+
+
+**Note:** When `web_search_options` is set in the config, it applies to all requests to that model. Users can still override these settings by passing `web_search_options` in their API requests.
## Checking if a model supports web search
-Use `litellm.supports_web_search(model="openai/gpt-4o-search-preview")` -> returns `True` if model can perform web searches
+Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model can perform web searches
```python showLineNumbers
+# Check OpenAI models
assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
+
+# Check xAI models
+assert litellm.supports_web_search(model="xai/grok-3") == True
+
+# Check Anthropic models
+assert litellm.supports_web_search(model="anthropic/claude-3-5-sonnet-latest") == True
+
+# Check VertexAI models
+assert litellm.supports_web_search(model="gemini-2.0-flash") == True
+
+# Check Google AI Studio models
+assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
```
-1. Define OpenAI models in config.yaml
+1. Define models in config.yaml
```yaml
model_list:
+ # OpenAI
- model_name: gpt-4o-search-preview
litellm_params:
model: openai/gpt-4o-search-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
supports_web_search: True
+
+ # xAI
+ - model_name: grok-3
+ litellm_params:
+ model: xai/grok-3
+ api_key: os.environ/XAI_API_KEY
+ model_info:
+ supports_web_search: True
+
+ # Anthropic
+ - model_name: claude-3-5-sonnet-latest
+ litellm_params:
+ model: anthropic/claude-3-5-sonnet-latest
+ api_key: os.environ/ANTHROPIC_API_KEY
+ model_info:
+ supports_web_search: True
+
+ # VertexAI
+ - model_name: gemini-2-flash
+ litellm_params:
+ model: gemini-2.0-flash
+ vertex_project: your-project-id
+ vertex_location: us-central1
+ model_info:
+ supports_web_search: True
+
+ # Google AI Studio
+ - model_name: gemini-2-flash-studio
+ litellm_params:
+ model: gemini/gemini-2.0-flash
+ api_key: os.environ/GOOGLE_API_KEY
+ model_info:
+ supports_web_search: True
```
2. Run proxy server
@@ -298,7 +504,19 @@ Expected Response
"model_group": "gpt-4o-search-preview",
"providers": ["openai"],
"max_tokens": 128000,
- "supports_web_search": true, # 👈 supports_web_search is true
+ "supports_web_search": true
+ },
+ {
+ "model_group": "grok-3",
+ "providers": ["xai"],
+ "max_tokens": 131072,
+ "supports_web_search": true
+ },
+ {
+ "model_group": "gemini-2-flash",
+ "providers": ["vertex_ai"],
+ "max_tokens": 8192,
+ "supports_web_search": true
}
]
}
diff --git a/docs/my-website/docs/contact.md b/docs/my-website/docs/contact.md
index d5309cd7373..947ec86991c 100644
--- a/docs/my-website/docs/contact.md
+++ b/docs/my-website/docs/contact.md
@@ -2,5 +2,6 @@
[](https://discord.gg/wuPM9dRgDw)
+* [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
* [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
* Contact us at ishaan@berri.ai / krrish@berri.ai
diff --git a/docs/my-website/docs/contributing.md b/docs/my-website/docs/contributing.md
index 8fc64b8f287..a88013ff1b3 100644
--- a/docs/my-website/docs/contributing.md
+++ b/docs/my-website/docs/contributing.md
@@ -13,7 +13,9 @@ git clone https://github.com/BerriAI/litellm.git
Tell the proxy where the UI is located
```bash
-export PROXY_BASE_URL="http://localhost:3000/"
+DATABASE_URL = "postgresql://:@:/"
+LITELLM_MASTER_KEY = "sk-1234"
+STORE_MODEL_IN_DB = "True"
```
```bash
@@ -25,7 +27,7 @@ python3 proxy_cli.py --config /path/to/config.yaml --port 4000
Set the mode as development (this will assume the proxy is running on localhost:4000)
```bash
-export NODE_ENV="development"
+npm install # install dependencies
```
```bash
diff --git a/docs/my-website/docs/data_security.md b/docs/my-website/docs/data_security.md
index 30128760f27..2c4b1247e2b 100644
--- a/docs/my-website/docs/data_security.md
+++ b/docs/my-website/docs/data_security.md
@@ -45,7 +45,7 @@ For security inquiries, please contact us at support@berri.ai
| **Certification** | **Status** |
|-------------------|-------------------------------------------------------------------------------------------------|
| SOC 2 Type I | Certified. Report available upon request on Enterprise plan. |
-| SOC 2 Type II | In progress. Certificate available by April 15th, 2025 |
+| SOC 2 Type II | Certified. Report available upon request on Enterprise plan. |
| ISO 27001 | Certified. Report available upon request on Enterprise |
diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md
index 1fd5a03e652..e63d9403665 100644
--- a/docs/my-website/docs/embedding/supported_embedding.md
+++ b/docs/my-website/docs/embedding/supported_embedding.md
@@ -266,7 +266,59 @@ print(response)
| Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
| Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
| Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |
+| TwelveLabs Marengo (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | [Async Invoke Docs](../providers/bedrock_embedding#async-invoke-embedding) |
+## TwelveLabs Bedrock Embedding Models
+
+TwelveLabs Marengo models support multimodal embeddings (text, image, video, audio) and require the `input_type` parameter to specify the input format.
+
+### Usage
+
+```python
+from litellm import embedding
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+# Text embedding
+response = embedding(
+ model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["Hello world from LiteLLM!"],
+ input_type="text" # Required parameter
+)
+
+# Image embedding (base64)
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."],
+ input_type="image", # Required parameter
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+# Video embedding (S3 URL)
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["s3://your-bucket/video.mp4"],
+ input_type="video", # Required parameter
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+```
+
+### Required Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `input_type` | Type of input content | `"text"`, `"image"`, `"video"`, `"audio"` |
+
+### Supported Models
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| TwelveLabs Marengo 2.7 (Sync) | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | Text embeddings only |
+| TwelveLabs Marengo 2.7 (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text/image/video/audio")` | All input types, requires `output_s3_uri` |
## Cohere Embedding Models
https://docs.cohere.com/reference/embed
diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md
index 706ca337144..cc3466fc103 100644
--- a/docs/my-website/docs/enterprise.md
+++ b/docs/my-website/docs/enterprise.md
@@ -1,12 +1,19 @@
import Image from '@theme/IdealImage';
# Enterprise
+
+:::info
+✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+:::
+
For companies that need SSO, user management and professional support for LiteLLM Proxy
:::info
-Get free 7-day trial key [here](https://www.litellm.ai/#trial)
+Get free 7-day trial key [here](https://www.litellm.ai/enterprise#trial)
:::
+## Enterprise Features
+
Includes all enterprise features.
@@ -18,32 +25,13 @@ This covers:
- [**Enterprise Features**](./proxy/enterprise)
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
-- ✅ **Professional Support - Dedicated discord + slack**
+- ✅ **Professional Support - Dedicated Slack/Teams channel**
-Deployment Options:
+## Self-Hosted
-**Self-Hosted**
-1. Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel.
+Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel.
-2. We Manage - you give us subscription access on your AWS/Azure/GCP account, and we manage the deployment.
-
-**Managed**
-
-You can use our cloud product where we setup a dedicated instance for you.
-
-## Frequently Asked Questions
-
-### SLA's + Professional Support
-
-Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.
-
-- 1 hour for Sev0 issues - 100% production traffic is failing
-- 6 hours for Sev1 - <100% production traffic is failing
-- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
-- 72h SLA for patching vulnerabilities in the software.
-
-**We can offer custom SLAs** based on your needs and the severity of the issue
### What’s the cost of the Self-Managed Enterprise edition?
@@ -58,8 +46,72 @@ You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) an
LITELLM_LICENSE="eyJ..."
```
-No data leaves your environment.
+**No data leaves your environment.**
+
+
+## Hosted LiteLLM Proxy
+
+LiteLLM maintains the proxy, so you can focus on your core products.
+
+We provide a dedicated proxy for your team, and manage the infrastructure.
+
+### **Status**: GA
+
+Our proxy is already used in production by customers.
+
+See our status page for [**live reliability**](https://status.litellm.ai/)
+
+### **Benefits**
+- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects.
+- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
+- **Secure**: LiteLLM is SOC-2 Type 2 and ISO 27001 certified, to make sure your data is as secure as possible.
+
+### Supported data regions for LiteLLM Cloud
+
+You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
+
+
+## Frequently Asked Questions
+
+### SLA's + Professional Support
+
+Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.
+
+- 1 hour for Sev0 issues - 100% production traffic is failing
+- 6 hours for Sev1 - < 100% production traffic is failing
+- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
+- 72h SLA for patching vulnerabilities in the software.
+
+**We can offer custom SLAs** based on your needs and the severity of the issue
## Data Security / Legal / Compliance FAQs
-[Data Security / Legal / Compliance FAQs](./data_security.md)
\ No newline at end of file
+[Data Security / Legal / Compliance FAQs](./data_security.md)
+
+
+### Pricing
+
+Pricing is based on usage. We can figure out a price that works for your team, on the call.
+
+[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+
+
+
+## **Screenshots**
+
+### 1. Create keys
+
+
+
+### 2. Add Models
+
+
+
+### 3. Track spend
+
+
+
+
+### 4. Configure load balancing
+
+
diff --git a/docs/my-website/docs/exception_mapping.md b/docs/my-website/docs/exception_mapping.md
index 13eda5b405a..2342f444e17 100644
--- a/docs/my-website/docs/exception_mapping.md
+++ b/docs/my-website/docs/exception_mapping.md
@@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
| 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
| 400 | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
| 400 | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
+| 400 | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
| 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
| 401 | AuthenticationError | openai.AuthenticationError |
| 403 | PermissionDeniedError | openai.PermissionDeniedError |
diff --git a/docs/my-website/docs/extras/contributing_code.md b/docs/my-website/docs/extras/contributing_code.md
index 8a617431960..f3a8271b14b 100644
--- a/docs/my-website/docs/extras/contributing_code.md
+++ b/docs/my-website/docs/extras/contributing_code.md
@@ -39,14 +39,14 @@ That's it, your local dev environment is ready!
## 2. Adding Testing to your PR
-- Add your test to the [`tests/litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
+- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
- This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
- Do not add real llm api calls to this directory.
-### 2.1 File Naming Convention for `tests/litellm/`
+### 2.1 File Naming Convention for `tests/test_litellm/`
-The `tests/litellm/` directory follows the same directory structure as `litellm/`.
+The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
- `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
- `test_{filename}.py` maps to `litellm/{filename}.py`
diff --git a/docs/my-website/docs/extras/gemini_img_migration.md b/docs/my-website/docs/extras/gemini_img_migration.md
new file mode 100644
index 00000000000..a29f301e382
--- /dev/null
+++ b/docs/my-website/docs/extras/gemini_img_migration.md
@@ -0,0 +1,220 @@
+# Gemini Image Generation Migration Guide
+
+## Who is impacted by this change?
+
+Anyone using the following models with /chat/completions:
+- `gemini/gemini-2.0-flash-exp-image-generation`
+- `vertex_ai/gemini-2.0-flash-exp-image-generation`
+
+## Key Change
+
+:::info
+From v1.77.0, LiteLLM will return the List of images in `response.choices[0].message.images` instead of a single image in `response.choices[0].message.image`.
+:::
+
+Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.images` with base64 data URLs.
+
+## Before and After
+
+### Before
+```python
+from litellm import completion
+
+response = completion(
+ model="gemini/gemini-2.0-flash-exp-image-generation",
+ messages=[{"role": "user", "content": "Generate an image of a cat"}],
+ modalities=["image", "text"],
+)
+
+
+base_64_image_data = response.choices[0].message.content
+```
+
+### After
+```python
+from litellm import completion
+
+response = completion(
+ model="gemini/gemini-2.0-flash-exp-image-generation",
+ messages=[{"role": "user", "content": "Generate an image of a cat"}],
+ modalities=["image", "text"],
+)
+
+# Image is now available in the response
+image_url = response.choices[0].message.images[0]["image_url"]["url"] # "data:image/png;base64,..."
+```
+
+### Why the change?
+
+Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
+
+**Why the change from `image` to `images`?**
+This is to be consistent with the OpenRouter API, making sure we are using simple, well-known interfaces where possible.
+
+## Usage
+
+### Using the Python SDK
+
+**Key Change:**
+```diff
+# Before
+-- base_64_image_data = response.choices[0].message.content
+
+# After
+++ image_url = response.choices[0].message.images[0]["image_url"]["url"]
+```
+
+#### Basic Image Generation
+
+```python
+from litellm import completion
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Generate an image
+response = completion(
+ model="gemini/gemini-2.0-flash-exp-image-generation",
+ messages=[{"role": "user", "content": "Generate an image of a cat"}],
+ modalities=["image", "text"],
+)
+
+# Access the generated image
+print(response.choices[0].message.content) # Text response (if any)
+print(response.choices[0].message.images[0]) # Image data
+```
+
+#### Response Format
+
+The image is returned in the `message.images` field:
+
+```python
+{
+ "image_url": {
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+ "detail": "auto"
+ },
+ "index": 0,
+ "type": "image_url"
+}
+```
+
+### Using the LiteLLM Proxy Server
+
+**Key Change:**
+```diff
+# Before
+-- "content": "base64-image-data..."
+
+# After
+++ "images": [{
+++ "image_url": {
+++ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+++ "detail": "auto"
+++ },
+++ "index": 0,
+++ "type": "image_url"
+++ }]
+```
+
+#### Configuration Setup
+
+1. **Configure your models in `config.yaml`:**
+
+```yaml
+model_list:
+ - model_name: gemini-image-gen
+ litellm_params:
+ model: gemini/gemini-2.0-flash-exp-image-generation
+ api_key: os.environ/GEMINI_API_KEY
+ - model_name: vertex-image-gen
+ litellm_params:
+ model: vertex_ai/gemini-2.5-flash-image-preview
+ vertex_project: your-project-id
+ vertex_location: us-central1
+
+general_settings:
+ master_key: sk-1234 # Your proxy API key
+```
+
+2. **Start the proxy server:**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### Making Requests
+
+**Using OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+# Point to your proxy server
+client = OpenAI(
+ api_key="sk-1234", # Your proxy API key
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="gemini-image-gen",
+ messages=[{"role": "user", "content": "Generate an image of a cat"}],
+ extra_body={"modalities": ["image", "text"]}
+)
+
+# Access the generated image
+print(response.choices[0].message.content) # Text response (if any)
+print(response.choices[0].message.image) # Image data
+```
+
+**Using curl:**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model": "gemini-image-gen",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Generate an image of a cat"
+ }
+ ],
+ "modalities": ["image", "text"]
+}'
+```
+
+**Response format from proxy:**
+
+```json
+{
+ "id": "chatcmpl-123",
+ "object": "chat.completion",
+ "created": 1704089632,
+ "model": "gemini-image-gen",
+ "choices": [
+ {
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": "Here's an image of a cat for you!",
+ "images": [{
+ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+ "detail": "auto"
+ }
+ },
+ "finish_reason": "stop"
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 8,
+ "total_tokens": 18
+ }
+}
+```
+
diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md
index f9a9297e062..f3f955cb01d 100644
--- a/docs/my-website/docs/fine_tuning.md
+++ b/docs/my-website/docs/fine_tuning.md
@@ -13,6 +13,8 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
| Feature | Supported | Notes |
|-------|-------|-------|
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
+
+#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
| Logging | ✅ | Works across all logging integrations |
diff --git a/docs/my-website/docs/generateContent.md b/docs/my-website/docs/generateContent.md
new file mode 100644
index 00000000000..e6823ebf05d
--- /dev/null
+++ b/docs/my-website/docs/generateContent.md
@@ -0,0 +1,236 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Google AI generateContent
+
+Use LiteLLM to call Google AI's generateContent endpoints for text generation, multimodal interactions, and streaming responses.
+
+## Overview
+
+| Feature | Supported | Notes |
+|-------|-------|-------|
+| Cost Tracking | ✅ | |
+| Logging | ✅ | works across all integrations |
+| End-user Tracking | ✅ | |
+| Streaming | ✅ | |
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+
+## Usage
+---
+
+### LiteLLM Python SDK
+
+
+
+
+#### Non-streaming example
+```python showLineNumbers title="Basic Text Generation"
+from litellm.google_genai import agenerate_content
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+ parts=[
+ PartDict(text="Hello, can you tell me a short joke?")
+ ],
+ role="user",
+)
+
+response = await agenerate_content(
+ contents=contents,
+ model="gemini/gemini-2.0-flash",
+ max_tokens=100,
+)
+print(response)
+```
+
+#### Streaming example
+```python showLineNumbers title="Streaming Text Generation"
+from litellm.google_genai import agenerate_content_stream
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+ parts=[
+ PartDict(text="Write a long story about space exploration")
+ ],
+ role="user",
+)
+
+response = await agenerate_content_stream(
+ contents=contents,
+ model="gemini/gemini-2.0-flash",
+ max_tokens=500,
+)
+
+async for chunk in response:
+ print(chunk)
+```
+
+
+
+
+
+#### Sync non-streaming example
+```python showLineNumbers title="Sync Text Generation"
+from litellm.google_genai import generate_content
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+ parts=[
+ PartDict(text="Hello, can you tell me a short joke?")
+ ],
+ role="user",
+)
+
+response = generate_content(
+ contents=contents,
+ model="gemini/gemini-2.0-flash",
+ max_tokens=100,
+)
+print(response)
+```
+
+#### Sync streaming example
+```python showLineNumbers title="Sync Streaming Text Generation"
+from litellm.google_genai import generate_content_stream
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+ parts=[
+ PartDict(text="Write a long story about space exploration")
+ ],
+ role="user",
+)
+
+response = generate_content_stream(
+ contents=contents,
+ model="gemini/gemini-2.0-flash",
+ max_tokens=500,
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+
+
+
+### LiteLLM Proxy Server
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: gemini-flash
+ litellm_params:
+ model: gemini/gemini-2.0-flash
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+
+
+
+```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
+from google.genai import Client
+import os
+
+# Configure Google GenAI SDK to use LiteLLM proxy
+os.environ["GOOGLE_GEMINI_BASE_URL"] = "http://localhost:4000"
+os.environ["GEMINI_API_KEY"] = "sk-1234"
+
+client = Client()
+
+response = client.models.generate_content(
+ model="gemini-flash",
+ contents=[
+ {
+ "parts": [{"text": "Write a short story about AI"}],
+ "role": "user"
+ }
+ ],
+ config={"max_output_tokens": 100}
+)
+```
+
+
+
+
+
+
+#### Generate Content
+
+```bash showLineNumbers title="generateContent via LiteLLM Proxy"
+curl -L -X POST 'http://localhost:4000/v1beta/models/gemini-flash:generateContent' \
+-H 'content-type: application/json' \
+-H 'authorization: Bearer sk-1234' \
+-d '{
+ "contents": [
+ {
+ "parts": [
+ {
+ "text": "Write a short story about AI"
+ }
+ ],
+ "role": "user"
+ }
+ ],
+ "generationConfig": {
+ "maxOutputTokens": 100
+ }
+}'
+```
+
+#### Stream Generate Content
+
+```bash showLineNumbers title="streamGenerateContent via LiteLLM Proxy"
+curl -L -X POST 'http://localhost:4000/v1beta/models/gemini-flash:streamGenerateContent' \
+-H 'content-type: application/json' \
+-H 'authorization: Bearer sk-1234' \
+-d '{
+ "contents": [
+ {
+ "parts": [
+ {
+ "text": "Write a long story about space exploration"
+ }
+ ],
+ "role": "user"
+ }
+ ],
+ "generationConfig": {
+ "maxOutputTokens": 500
+ }
+}'
+```
+
+
+
+
+
+## Related
+
+- [Use LiteLLM with gemini-cli](../docs/tutorials/litellm_gemini_cli)
\ No newline at end of file
diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md
index 15ee00a7273..6b2c1fd531e 100644
--- a/docs/my-website/docs/getting_started.md
+++ b/docs/my-website/docs/getting_started.md
@@ -32,7 +32,8 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
More details 👉
- [Completion() function details](./completion/)
-- [All supported models / providers on LiteLLM](./providers/)
+- [Overview of supported models / providers on LiteLLM](./providers/)
+- [Search all models / providers](https://models.litellm.ai/)
- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
## streaming
diff --git a/docs/my-website/docs/guides/security_settings.md b/docs/my-website/docs/guides/security_settings.md
index 4dfeda2d70b..7995f6c3c9c 100644
--- a/docs/my-website/docs/guides/security_settings.md
+++ b/docs/my-website/docs/guides/security_settings.md
@@ -1,14 +1,45 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# SSL Security Settings
+# SSL, HTTP Proxy Security Settings
-If you're in an environment using an older TTS bundle, with an older encryption, follow this guide.
+If you're in an environment using an older TTS bundle, with an older encryption, follow this guide. By default
+LiteLLM uses the certifi CA bundle for SSL verification, which is compatible with most modern servers.
+ However, if you need to disable SSL verification or use a custom CA bundle, you can do so by following the steps below.
+Be aware that environmental variables take precedence over the settings in the SDK.
-LiteLLM uses HTTPX for network requests, unless otherwise specified.
+LiteLLM uses HTTPX for network requests, unless otherwise specified.
-1. Disable SSL verification
+## 1. Custom CA Bundle
+
+You can set a custom CA bundle file path using the `SSL_CERT_FILE` environmental variable or passing a string to the the ssl_verify setting.
+
+
+
+
+```python
+import litellm
+litellm.ssl_verify = "client.pem"
+```
+
+
+
+```yaml
+litellm_settings:
+ ssl_verify: "client.pem"
+```
+
+
+
+
+```bash
+export SSL_CERT_FILE="client.pem"
+```
+
+
+
+## 2. Disable SSL verification
@@ -35,14 +66,42 @@ export SSL_VERIFY="False"
-2. Lower security settings
+## 3. Lower security settings
+
+The `ssl_security_level` allows setting a lower security level for SSL connections.
+
+
+
+
+```python
+import litellm
+litellm.ssl_security_level = "DEFAULT@SECLEVEL=1"
+```
+
+
+
+```yaml
+litellm_settings:
+ ssl_security_level: "DEFAULT@SECLEVEL=1"
+```
+
+
+
+```bash
+export SSL_SECURITY_LEVEL="DEFAULT@SECLEVEL=1"
+```
+
+
+
+## 4. Certificate authentication
+
+The `SSL_CERTIFICATE` environmental variable or `ssl_certificate` attribute allows setting a client side certificate to authenticate the client to the server.
```python
import litellm
-litellm.ssl_security_level = 1
litellm.ssl_certificate = "/path/to/certificate.pem"
```
@@ -50,17 +109,40 @@ litellm.ssl_certificate = "/path/to/certificate.pem"
```yaml
litellm_settings:
- ssl_security_level: 1
ssl_certificate: "/path/to/certificate.pem"
```
```bash
-export SSL_SECURITY_LEVEL="1"
export SSL_CERTIFICATE="/path/to/certificate.pem"
```
+## 5. Use HTTP_PROXY environment variable
+
+Both httpx and aiohttp libraries use `urllib.request.getproxies` from environment variables. Before client initialization, you may set proxy (and optional SSL_CERT_FILE) by setting the environment variables:
+
+
+
+
+```python
+import litellm
+litellm.aiohttp_trust_env = True
+```
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+```
+
+
+
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+export AIOHTTP_TRUST_ENV='True'
+```
+
+
diff --git a/docs/my-website/docs/hosted.md b/docs/my-website/docs/hosted.md
deleted file mode 100644
index 99bfe990315..00000000000
--- a/docs/my-website/docs/hosted.md
+++ /dev/null
@@ -1,66 +0,0 @@
-import Image from '@theme/IdealImage';
-
-# Hosted LiteLLM Proxy
-
-LiteLLM maintains the proxy, so you can focus on your core products.
-
-## [**Get Onboarded**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-This is in alpha. Schedule a call with us, and we'll give you a hosted proxy within 30 minutes.
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-### **Status**: Alpha
-
-Our proxy is already used in production by customers.
-
-See our status page for [**live reliability**](https://status.litellm.ai/)
-
-### **Benefits**
-- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects.
-- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
-- **Secure**: LiteLLM is currently undergoing SOC-2 compliance, to make sure your data is as secure as possible.
-
-## Data Privacy & Security
-
-You can find our [data privacy & security policy for cloud litellm here](../docs/data_security#litellm-cloud)
-
-## Supported data regions for LiteLLM Cloud
-
-You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
-
-### Pricing
-
-Pricing is based on usage. We can figure out a price that works for your team, on the call.
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## **Screenshots**
-
-### 1. Create keys
-
-
-
-### 2. Add Models
-
-
-
-### 3. Track spend
-
-
-
-
-### 4. Configure load balancing
-
-
-
-#### [**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## Feature List
-
-- Easy way to add/remove models
-- 100% uptime even when models are added/removed
-- custom callback webhooks
-- your domain name with HTTPS
-- Ability to create/delete User API keys
-- Reasonable set monthly cost
\ No newline at end of file
diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md
index f0254032964..84dddd5e4ad 100644
--- a/docs/my-website/docs/image_edits.md
+++ b/docs/my-website/docs/image_edits.md
@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# /images/edits
-LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
+LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
| Feature | Supported | Notes |
|---------|-----------|--------|
@@ -13,11 +13,14 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
-| Supported operations | Create image edits | |
+| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | |
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
+ #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
+
## Usage
### LiteLLM Python SDK
@@ -41,6 +44,26 @@ response = litellm.image_edit(
print(response)
```
+#### Multiple Images Edit
+```python showLineNumbers title="OpenAI Multiple Images Edit"
+import litellm
+
+# Edit multiple images with a prompt
+response = litellm.image_edit(
+ model="gpt-image-1",
+ image=[
+ open("image1.png", "rb"),
+ open("image2.png", "rb"),
+ open("image3.png", "rb")
+ ],
+ prompt="Apply vintage filter to all images",
+ n=1,
+ size="1024x1024"
+)
+
+print(response)
+```
+
#### Image Edit with Mask
```python showLineNumbers title="OpenAI Image Edit with Mask"
import litellm
@@ -80,6 +103,30 @@ response = asyncio.run(edit_image())
print(response)
```
+#### Async Multiple Images Edit
+```python showLineNumbers title="Async OpenAI Multiple Images Edit"
+import litellm
+import asyncio
+
+async def edit_multiple_images():
+ response = await litellm.aimage_edit(
+ model="gpt-image-1",
+ image=[
+ open("portrait1.png", "rb"),
+ open("portrait2.png", "rb")
+ ],
+ prompt="Add professional lighting to the portraits",
+ n=1,
+ size="1024x1024",
+ response_format="url"
+ )
+ return response
+
+# Run the async function
+response = asyncio.run(edit_multiple_images())
+print(response)
+```
+
#### Image Edit with Custom Parameters
```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
import litellm
@@ -163,6 +210,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
-F "response_format=url"
```
+#### cURL Multiple Images Example
+```bash showLineNumbers title="cURL Multiple Images Edit Request"
+curl -X POST "http://localhost:4000/v1/images/edits" \
+ -H "Authorization: Bearer your-api-key" \
+ -F "model=gpt-image-1" \
+ -F "image=@image1.png" \
+ -F "image=@image2.png" \
+ -F "image=@image3.png" \
+ -F "prompt=Apply artistic filter to all images" \
+ -F "n=1" \
+ -F "size=1024x1024" \
+ -F "response_format=url"
+```
+
diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md
index 5af3e10e0ca..8cd5803aa6c 100644
--- a/docs/my-website/docs/image_generation.md
+++ b/docs/my-website/docs/image_generation.md
@@ -52,7 +52,7 @@ litellm --config /path/to/config.yaml
curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
--D '{
+-d '{
"model": "gpt-image-1",
"prompt": "A cute baby sea otter",
"n": 1,
@@ -154,7 +154,7 @@ Any non-openai params, will be treated as provider-specific params, and sent in
## OpenAI Image Generation Models
### Usage
-```python
+```python showLineNumbers
from litellm import image_generation
import os
os.environ['OPENAI_API_KEY'] = ""
@@ -171,7 +171,7 @@ response = image_generation(model='gpt-image-1', prompt="cute baby otter")
### API keys
This can be set as env variables or passed as **params to litellm.image_generation()**
-```python
+```python showLineNumbers
import os
os.environ['AZURE_API_KEY'] =
os.environ['AZURE_API_BASE'] =
@@ -179,7 +179,7 @@ os.environ['AZURE_API_VERSION'] =
```
### Usage
-```python
+```python showLineNumbers
from litellm import embedding
response = embedding(
model="azure/",
@@ -197,6 +197,34 @@ print(response)
| dall-e-3 | `image_generation(model="azure/", prompt="cute baby otter")` |
| dall-e-2 | `image_generation(model="azure/", prompt="cute baby otter")` |
+## Xinference Image Generation Models
+
+Use this for Stable Diffusion models hosted on Xinference
+
+#### Usage
+
+See Xinference usage with LiteLLM [here](./providers/xinference.md#image-generation)
+
+## Recraft Image Generation Models
+
+Use this for AI-powered design and image generation with Recraft
+
+#### Usage
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+response = image_generation(
+ model="recraft/recraftv3",
+ prompt="A beautiful sunset over a calm ocean",
+)
+print(response)
+```
+
+See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
## OpenAI Compatible Image Generation Models
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
@@ -204,7 +232,7 @@ Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers,
**Note add `openai/` prefix to model so litellm knows to route to OpenAI**
### Usage
-```python
+```python showLineNumbers
from litellm import image_generation
response = image_generation(
model = "openai/", # add `openai/` prefix to model so litellm knows to route to OpenAI
@@ -218,7 +246,7 @@ Use this for stable diffusion on bedrock
### Usage
-```python
+```python showLineNumbers
import os
from litellm import image_generation
@@ -239,7 +267,7 @@ print(f"response: {response}")
Use this for image generation models on VertexAI
-```python
+```python showLineNumbers
response = litellm.image_generation(
prompt="An olympic size swimming pool",
model="vertex_ai/imagegeneration@006",
@@ -248,3 +276,18 @@ response = litellm.image_generation(
)
print(f"response: {response}")
```
+
+## Supported Providers
+
+#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
+| Provider | Documentation Link |
+|----------|-------------------|
+| OpenAI | [OpenAI Image Generation →](./providers/openai) |
+| Azure OpenAI | [Azure OpenAI Image Generation →](./providers/azure/azure) |
+| Google AI Studio | [Google AI Studio Image Generation →](./providers/google_ai_studio/image_gen) |
+| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
+| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
+| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
+| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
+| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) |
\ No newline at end of file
diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md
index 58cabc81b48..11d2963b7a3 100644
--- a/docs/my-website/docs/index.md
+++ b/docs/my-website/docs/index.md
@@ -226,6 +226,23 @@ response = completion(
+
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+ model="vercel_ai_gateway/openai/gpt-4o",
+ messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+
+
### Response Format (OpenAI Format)
@@ -234,7 +251,7 @@ response = completion(
{
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
"created": 1734366691,
- "model": "claude-3-sonnet-20240229",
+ "model": "gpt-4o-2024-08-06",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
@@ -446,6 +463,24 @@ response = completion(
+
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+ model="vercel_ai_gateway/openai/gpt-4o",
+ messages = [{ "content": "Hello, how are you?","role": "user"}],
+ stream=True,
+)
+```
+
+
+
### Streaming Response Format (OpenAI Format)
@@ -489,6 +524,15 @@ try:
except OpenAIError as e:
print(e)
```
+### See How LiteLLM Transforms Your Requests
+
+Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
+
+You can try it out now directly on our Demo App!
+Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
+
+LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
+
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack
diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md
new file mode 100644
index 00000000000..95c922cce89
--- /dev/null
+++ b/docs/my-website/docs/integrations/index.md
@@ -0,0 +1,18 @@
+# Integrations
+
+This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
+
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
+Click into each section to learn more about the integrations.
\ No newline at end of file
diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md
new file mode 100644
index 00000000000..2afb82542f2
--- /dev/null
+++ b/docs/my-website/docs/integrations/letta.md
@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+
+
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+
+ - model_name: claude-3-sonnet
+ litellm_params:
+ model: anthropic/claude-3-sonnet-20240229
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: azure/gpt-35-turbo
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+ model="gpt-4", # This should match a model from your LiteLLM config
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000", # Your LiteLLM proxy URL
+ context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+ embedding_endpoint_type="openai",
+ embedding_endpoint="http://localhost:4000",
+ embedding_model="text-embedding-ada-002"
+)
+```
+
+
+
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+ def __init__(self, model: str):
+ self.model = model
+
+ def chat_completions_create(self, messages: List[Dict], **kwargs):
+ # Use LiteLLM SDK for completion
+ response = litellm.completion(
+ model=self.model,
+ messages=messages,
+ **kwargs
+ )
+ return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+ model="gpt-4", # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+
+
+
+### 3. Create and Use a Letta Agent
+
+
+
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+
+
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint_type="openai", # Using OpenAI-compatible endpoint
+ model_endpoint="http://localhost:4000",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+ model="openai/gpt-4", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+### Function Calling with Tools
+
+
+
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4", # Direct model specification
+ model_endpoint_type="openai",
+ context_window=8192
+ ),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+## Authentication
+
+
+
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ model_wrapper="openai",
+ context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+ master_key: "your-master-key"
+
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192,
+ api_key="your-master-key" # Proxy master key
+)
+```
+
+
+
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key"
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY") # Default key
+litellm.set_verbose = True # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+ model="openai/gpt-4", # Will use OPENAI_API_KEY automatically
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+ model="azure/gpt-35-turbo",
+ model_endpoint_type="openai",
+ context_window=4096
+)
+```
+
+
+
+
+## Load Balancing and Fallbacks
+
+
+
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ tpm: 40000
+ rpm: 500
+
+ - model_name: gpt-4 # Same model name for fallback
+ litellm_params:
+ model: azure/gpt-4
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+ tpm: 80000
+ rpm: 800
+
+router_settings:
+ routing_strategy: "usage-based-routing"
+ fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+
+
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+ model_list=[
+ {
+ "model_name": "gpt-4",
+ "litellm_params": {
+ "model": "openai/gpt-4",
+ "api_key": os.environ["OPENAI_API_KEY"]
+ },
+ "tpm": 40000,
+ "rpm": 500
+ },
+ {
+ "model_name": "gpt-4", # Same name for fallback
+ "litellm_params": {
+ "model": "azure/gpt-4",
+ "api_key": os.environ["AZURE_API_KEY"],
+ "api_base": os.environ["AZURE_API_BASE"],
+ "api_version": "2023-07-01-preview"
+ },
+ "tpm": 80000,
+ "rpm": 800
+ }
+ ],
+ fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+ routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+ return router.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+
+
+
+## Monitoring and Observability
+
+
+
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+ # ... your models
+
+litellm_settings:
+ success_callback: ["langfuse"] # or other observability tools
+
+environment_variables:
+ LANGFUSE_PUBLIC_KEY: "your-key"
+ LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+
+
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+ # Custom logging logic here if needed
+ return response
+ except Exception as e:
+ # Custom error handling
+ print(f"LLM call failed: {e}")
+ raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+
+
+
+## Example: Multi-Agent System
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="gpt-4",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4",
+ model_endpoint_type="openai"
+ )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+ name="reviewer",
+ system="You are an editor. Review and improve content quality.",
+ llm_config=LLMConfig(
+ model="openai/gpt-3.5-turbo",
+ model_endpoint_type="openai"
+ )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ draft_article = write_response.messages[-1].text
+
+ # Review phase
+ review_response = client.user_message(
+ agent_id=agents['reviewer'].id,
+ message=f"Please review and improve this article:\n\n{draft_article}"
+ )
+
+ return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+## Best Practices
+
+
+
+
+1. **Model Selection**: Use appropriate models for different tasks:
+ - Claude for analysis and reasoning
+ - GPT-4 for creative tasks
+ - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+ - Set appropriate rate limits and timeouts
+ - Use fallbacks for reliability
+ - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**:
+ - Use the proxy's budgeting features to control costs
+ - Set up rate limiting per user/team
+ - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+
+
+
+1. **Model Selection**: Choose models based on task requirements:
+ - Use `openai/gpt-4` for complex reasoning
+ - Use `anthropic/claude-3-sonnet-20240229` for analysis
+ - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+ ```python
+ import litellm
+ from litellm import completion
+
+ # Set up retry logic
+ litellm.num_retries = 3
+ litellm.request_timeout = 60
+
+ # Custom error handling
+ def safe_completion(**kwargs):
+ try:
+ return completion(**kwargs)
+ except Exception as e:
+ print(f"LLM call failed: {e}")
+ # Implement fallback logic
+ return completion(model="openai/gpt-3.5-turbo", **kwargs)
+ ```
+
+3. **Cost Management**:
+ - Use cheaper models for non-critical tasks
+ - Implement token counting and budgets
+ - Cache responses when appropriate
+
+4. **Performance**:
+ - Use async operations for concurrent requests
+ - Implement connection pooling
+ - Monitor response times
+
+5. **Security**:
+ - Store API keys securely (environment variables)
+ - Rotate keys regularly
+ - Implement rate limiting
+
+
+
+
+## Troubleshooting
+
+
+
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}]
+ }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+
+
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+ response = litellm.completion(
+ model="openai/gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+ print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+ print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=[{"role": "user", "content": "Test"}],
+ max_tokens=10
+ )
+ print(f"✓ {model} working")
+ except Exception as e:
+ print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+
+
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)
\ No newline at end of file
diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md
index 78425a73b99..c67375ce1be 100644
--- a/docs/my-website/docs/langchain/langchain.md
+++ b/docs/my-website/docs/langchain/langchain.md
@@ -162,3 +162,321 @@ Get more details [here](../observability/lunary_integration.md)
## Use LangChain ChatLiteLLM + Langfuse
Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM.
+
+## Using Tags with LangChain and LiteLLM
+
+Tags are a powerful feature in LiteLLM that allow you to categorize, filter, and track your LLM requests. When using LangChain with LiteLLM, you can pass tags through the `extra_body` parameter in the metadata.
+
+### Basic Tag Usage
+
+
+
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['OPENAI_API_KEY'] = "sk-your-key-here"
+
+chat = ChatOpenAI(
+ model="gpt-4o",
+ temperature=0.7,
+ extra_body={
+ "metadata": {
+ "tags": ["production", "customer-support", "high-priority"]
+ }
+ }
+)
+
+messages = [
+ SystemMessage(content="You are a helpful customer support assistant."),
+ HumanMessage(content="How do I reset my password?")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+
+
+
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['ANTHROPIC_API_KEY'] = "sk-ant-your-key-here"
+
+chat = ChatOpenAI(
+ model="claude-3-sonnet-20240229",
+ temperature=0.7,
+ extra_body={
+ "metadata": {
+ "tags": ["research", "analysis", "claude-model"]
+ }
+ }
+)
+
+messages = [
+ SystemMessage(content="You are a research analyst."),
+ HumanMessage(content="Analyze this market trend...")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+
+
+
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# No API key needed when using proxy
+chat = ChatOpenAI(
+ openai_api_base="http://localhost:4000", # Your proxy URL
+ model="gpt-4o",
+ temperature=0.7,
+ extra_body={
+ "metadata": {
+ "tags": ["proxy", "team-alpha", "feature-flagged"],
+ "generation_name": "customer-onboarding",
+ "trace_user_id": "user-12345"
+ }
+ }
+)
+
+messages = [
+ SystemMessage(content="You are an onboarding assistant."),
+ HumanMessage(content="Welcome our new customer!")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+
+
+
+### Advanced Tag Patterns
+
+#### Dynamic Tags Based on Context
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+def create_chat_with_tags(user_type: str, feature: str):
+ """Create a chat instance with dynamic tags based on context"""
+
+ # Build tags dynamically
+ tags = ["langchain-integration"]
+
+ if user_type == "premium":
+ tags.extend(["premium-user", "high-priority"])
+ elif user_type == "enterprise":
+ tags.extend(["enterprise", "custom-sla"])
+ else:
+ tags.append("standard-user")
+
+ # Add feature-specific tags
+ if feature == "code-review":
+ tags.extend(["development", "code-analysis"])
+ elif feature == "content-gen":
+ tags.extend(["marketing", "content-creation"])
+
+ return ChatOpenAI(
+ openai_api_base="http://localhost:4000",
+ model="gpt-4o",
+ temperature=0.7,
+ extra_body={
+ "metadata": {
+ "tags": tags,
+ "user_type": user_type,
+ "feature": feature,
+ "trace_user_id": f"user-{user_type}-{feature}"
+ }
+ }
+ )
+
+# Usage examples
+premium_chat = create_chat_with_tags("premium", "code-review")
+enterprise_chat = create_chat_with_tags("enterprise", "content-gen")
+
+messages = [HumanMessage(content="Help me with this task")]
+response = premium_chat.invoke(messages)
+```
+
+#### Tags for Cost Tracking and Analytics
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# Tags for cost tracking
+cost_tracking_chat = ChatOpenAI(
+ openai_api_base="http://localhost:4000",
+ model="gpt-4o",
+ temperature=0.7,
+ extra_body={
+ "metadata": {
+ "tags": [
+ "cost-center-marketing",
+ "budget-q4-2024",
+ "project-launch-campaign",
+ "high-cost-model" # Flag for expensive models
+ ],
+ "department": "marketing",
+ "project_id": "campaign-2024-q4",
+ "cost_threshold": "high"
+ }
+ }
+)
+
+messages = [
+ SystemMessage(content="You are a marketing copywriter."),
+ HumanMessage(content="Create compelling ad copy for our new product launch.")
+]
+
+response = cost_tracking_chat.invoke(messages)
+```
+
+#### Tags for A/B Testing
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+import random
+
+def create_ab_test_chat(test_variant: str = None):
+ """Create chat instance for A/B testing with appropriate tags"""
+
+ if test_variant is None:
+ test_variant = random.choice(["variant-a", "variant-b"])
+
+ return ChatOpenAI(
+ openai_api_base="http://localhost:4000",
+ model="gpt-4o",
+ temperature=0.7 if test_variant == "variant-a" else 0.9, # Different temp for variants
+ extra_body={
+ "metadata": {
+ "tags": [
+ "ab-test-experiment-1",
+ f"variant-{test_variant}",
+ "temperature-test",
+ "user-experience"
+ ],
+ "experiment_id": "ab-test-001",
+ "variant": test_variant,
+ "test_group": "temperature-optimization"
+ }
+ }
+ )
+
+# Run A/B test
+variant_a_chat = create_ab_test_chat("variant-a")
+variant_b_chat = create_ab_test_chat("variant-b")
+
+test_message = [HumanMessage(content="Explain quantum computing in simple terms")]
+
+response_a = variant_a_chat.invoke(test_message)
+response_b = variant_b_chat.invoke(test_message)
+```
+
+### Tag Best Practices
+
+#### 1. **Consistent Naming Convention**
+```python
+# ✅ Good: Consistent, descriptive tags
+tags = ["production", "api-v2", "customer-support", "urgent"]
+
+# ❌ Avoid: Inconsistent or unclear tags
+tags = ["prod", "v2", "support", "urgent123"]
+```
+
+#### 2. **Hierarchical Tags**
+```python
+# ✅ Good: Hierarchical structure
+tags = ["env:production", "team:backend", "service:api", "priority:high"]
+
+# This allows for easy filtering and grouping
+```
+
+#### 3. **Include Context Information**
+```python
+extra_body={
+ "metadata": {
+ "tags": ["production", "user-onboarding"],
+ "user_id": "user-12345",
+ "session_id": "session-abc123",
+ "feature_flag": "new-onboarding-flow",
+ "environment": "production"
+ }
+}
+```
+
+#### 4. **Tag Categories**
+Consider organizing tags into categories:
+- **Environment**: `production`, `staging`, `development`
+- **Team/Service**: `backend`, `frontend`, `api`, `worker`
+- **Feature**: `authentication`, `payment`, `notification`
+- **Priority**: `critical`, `high`, `medium`, `low`
+- **User Type**: `premium`, `enterprise`, `free`
+
+### Using Tags with LiteLLM Proxy
+
+When using tags with LiteLLM Proxy, you can:
+
+1. **Filter requests** based on tags
+2. **Track costs** by tags in spend reports
+3. **Apply routing rules** based on tags
+4. **Monitor usage** with tag-based analytics
+
+#### Example Proxy Configuration with Tags
+
+```yaml
+# config.yaml
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: gpt-4o
+ api_key: your-key
+
+# Tag-based routing rules
+tag_routing:
+ - tags: ["premium", "high-priority"]
+ models: ["gpt-4o", "claude-3-opus"]
+ - tags: ["standard"]
+ models: ["gpt-3.5-turbo", "claude-3-haiku"]
+```
+
+### Monitoring and Analytics
+
+Tags enable powerful analytics capabilities:
+
+```python
+# Example: Get spend reports by tags
+import requests
+
+response = requests.get(
+ "http://localhost:4000/global/spend/report",
+ headers={"Authorization": "Bearer sk-your-key"},
+ params={
+ "start_date": "2024-01-01",
+ "end_date": "2024-12-31",
+ "group_by": "tags"
+ }
+)
+
+spend_by_tags = response.json()
+```
+
+This documentation covers the essential patterns for using tags effectively with LangChain and LiteLLM, enabling better organization, tracking, and analytics of your LLM requests.
diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md
index 0b3d38f3fcc..3171bc33594 100644
--- a/docs/my-website/docs/load_test_advanced.md
+++ b/docs/my-website/docs/load_test_advanced.md
@@ -27,13 +27,13 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
**Use this config for testing:**
-**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
+**Note:** we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `openai/` provider for load testing.
```yaml
model_list:
- model_name: "fake-openai-endpoint"
litellm_params:
- model: aiohttp_openai/any
+ model: openai/any
api_base: https://your-fake-openai-endpoint.com/chat/completions
api_key: "test"
```
@@ -58,7 +58,7 @@ litellm provides a hosted `fake-openai-endpoint` you can load test against
model_list:
- model_name: fake-openai-endpoint
litellm_params:
- model: aiohttp_openai/fake
+ model: openai/fake
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
diff --git a/docs/my-website/docs/load_test_rpm.md b/docs/my-website/docs/load_test_rpm.md
index 0954ffcdfac..b7621a76468 100644
--- a/docs/my-website/docs/load_test_rpm.md
+++ b/docs/my-website/docs/load_test_rpm.md
@@ -53,8 +53,8 @@ model_list = [
},
]
-router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
-router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="simple-shuffle", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="simple-shuffle", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
@@ -142,7 +142,7 @@ router_settings:
redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
redis_password: os.environ/REDIS_PASSWORD
redis_port: os.environ/REDIS_PORT
- routing_strategy: usage-based-routing-v2
+ routing_strategy: simple-shuffle # recommended for best performance
```
### 2. Start proxy 2 instances
diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md
index ad16cd17d1e..9365b0a5542 100644
--- a/docs/my-website/docs/mcp.md
+++ b/docs/my-website/docs/mcp.md
@@ -2,11 +2,9 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
-# /mcp [BETA] - Model Context Protocol
+# MCP Overview
-## Expose MCP tools on LiteLLM Proxy Server
-
-This allows you to define tools that can be called by any MCP compatible client. Define your `mcp_servers` with LiteLLM and all your clients can list and call available tools.
+LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint for all MCP tools and control MCP access by Key, Team.
-#### How it works
+## Overview
+| Feature | Description |
+|---------|-------------|
+| MCP Operations | • List Tools • Call Tools |
+| Supported MCP Transports | • Streamable HTTP • SSE • Standard Input/Output (stdio) |
+| LiteLLM Permission Management | • By Key • By Team • By Organization |
-1. Allow proxy admin users to perform create, update, and delete operations on MCP servers stored in the db.
-2. Allows users to view and call tools to the MCP servers they have access to.
+## Adding your MCP
-LiteLLM exposes the following MCP endpoints:
+### Prerequisites
-- GET `/mcp/enabled` - Returns if MCP is enabled (python>=3.10 requirements are met)
-- GET `/mcp/tools/list` - List all available tools
-- POST `/mcp/tools/call` - Call a specific tool with the provided arguments
-- GET `/v1/mcp/server` - Returns all of the configured mcp servers in the db filtered by requestor's access
-- GET `/v1/mcp/server/{server_id}` - Returns the the specific mcp server in the db given `server_id` filtered by requestor's access
-- PUT `/v1/mcp/server` - Updates an existing external mcp server.
-- POST `/v1/mcp/server` - Add a new external mcp server.
-- DELETE `/v1/mcp/server/{server_id}` - Deletes the mcp server given `server_id`.
+To store MCP servers in the database, you need to enable database storage:
-When MCP clients connect to LiteLLM they can follow this workflow:
+**Environment Variable:**
+```bash
+export STORE_MODEL_IN_DB=True
+```
-1. Connect to the LiteLLM MCP server
-2. List all available tools on LiteLLM
-3. Client makes LLM API request with tool call(s)
-4. LLM API returns which tools to call and with what arguments
-5. MCP client makes MCP tool calls to LiteLLM
-6. LiteLLM makes the tool calls to the appropriate MCP server
-7. LiteLLM returns the tool call results to the MCP client
+**OR in config.yaml:**
+```yaml
+general_settings:
+ store_model_in_db: true
+```
-#### Usage
+#### Fine-grained Database Storage Control
-#### 1. Define your tools on under `mcp_servers` in your config.yaml file.
+By default, when `store_model_in_db` is `true`, all object types (models, MCPs, guardrails, vector stores, etc.) are stored in the database. If you want to store only specific object types, use the `supported_db_objects` setting.
-LiteLLM allows you to define your tools on the `mcp_servers` section in your config.yaml file. All tools listed here will be available to MCP clients (when they connect to LiteLLM and call `list_tools`).
+**Example: Store only MCP servers in the database**
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+ store_model_in_db: true
+ supported_db_objects: ["mcp"] # Only store MCP servers in DB
+
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: sk-xxxxxxx
+```
+
+**See all available object types:** [Config Settings - supported_db_objects](./proxy/config_settings.md#general_settings---reference)
+
+If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
+
+
+
+
+On the LiteLLM UI, Navigate to "MCP Servers" and click "Add New MCP Server".
+
+On this form, you should enter your MCP Server URL and the transport you want to use.
+
+LiteLLM supports the following MCP transports:
+- Streamable HTTP
+- SSE (Server-Sent Events)
+- Standard Input/Output (stdio)
+
+
+
+
+
+
+### Add HTTP MCP Server
+
+This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE.
+
+
+
+
+
+
+### Add SSE MCP Server
+
+This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE.
+
+
+
+
+
+
+### Add STDIO MCP Server
+
+For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format:
+
+
+
+
+
+
+
+Add your MCP servers directly in your `config.yaml` file:
```yaml title="config.yaml" showLineNumbers
model_list:
@@ -55,128 +120,1180 @@ model_list:
model: openai/gpt-4o
api_key: sk-xxxxxxx
+litellm_settings:
+ # MCP Aliases - Map aliases to server names for easier tool access
+ mcp_aliases:
+ "github": "github_mcp_server"
+ "zapier": "zapier_mcp_server"
+ "deepwiki": "deepwiki_mcp_server"
+
mcp_servers:
+ # HTTP Streamable Server
+ deepwiki_mcp:
+ url: "https://mcp.deepwiki.com/mcp"
+ # SSE Server
zapier_mcp:
url: "https://actions.zapier.com/mcp/sk-akxxxxx/sse"
- fetch:
- url: "http://localhost:8000/sse"
+
+ # Standard Input/Output (stdio) Server - CircleCI Example
+ circleci_mcp:
+ transport: "stdio"
+ command: "npx"
+ args: ["-y", "@circleci/mcp-server-circleci"]
+ env:
+ CIRCLECI_TOKEN: "your-circleci-token"
+ CIRCLECI_BASE_URL: "https://circleci.com"
+
+ # Full configuration with all optional fields
+ my_http_server:
+ url: "https://my-mcp-server.com/mcp"
+ transport: "http"
+ description: "My custom MCP server"
+ auth_type: "api_key"
+ auth_value: "abc123"
+```
+
+**Configuration Options:**
+- **Server Name**: Use any descriptive name for your MCP server (e.g., `zapier_mcp`, `deepwiki_mcp`, `circleci_mcp`)
+- **Alias**: This name will be prefilled with the server name with "_" replacing spaces, else edit it to be the prefix in tool names
+- **URL**: The endpoint URL for your MCP server (required for HTTP/SSE transports)
+- **Transport**: Optional transport type (defaults to `sse`)
+ - `sse` - SSE (Server-Sent Events) transport
+ - `http` - Streamable HTTP transport
+ - `stdio` - Standard Input/Output transport
+- **Command**: The command to execute for stdio transport (required for stdio)
+- **Args**: Array of arguments to pass to the command (optional for stdio)
+- **Env**: Environment variables to set for the stdio process (optional for stdio)
+- **Description**: Optional description for the server
+- **Auth Type**: Optional authentication type. Supported values:
+
+ | Value | Header sent |
+ |-------|-------------|
+ | `api_key` | `X-API-Key: ` |
+ | `bearer_token` | `Authorization: Bearer ` |
+ | `basic` | `Authorization: Basic ` |
+ | `authorization` | `Authorization: ` |
+
+- **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
+- **Spec Version**: Optional MCP specification version (defaults to `2025-06-18`)
+
+Examples for each auth type:
+
+```yaml title="MCP auth examples (config.yaml)" showLineNumbers
+mcp_servers:
+ api_key_example:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: "api_key"
+ auth_value: "abc123" # headers={"X-API-Key": "abc123"}
+
+ # NEW – OAuth 2.0 Client Credentials (v1.77.5)
+ oauth2_example:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: "oauth2" # 👈 KEY CHANGE
+ authorization_url: "https://my-mcp-server.com/oauth/authorize" # optional for client-credentials
+ token_url: "https://my-mcp-server.com/oauth/token" # required
+ client_id: os.environ/OAUTH_CLIENT_ID
+ client_secret: os.environ/OAUTH_CLIENT_SECRET
+ scopes: ["tool.read", "tool.write"] # optional
+
+ bearer_example:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: "bearer_token"
+ auth_value: "abc123" # headers={"Authorization": "Bearer abc123"}
+
+ basic_example:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: "basic"
+ auth_value: "dXNlcjpwYXNz" # headers={"Authorization": "Basic dXNlcjpwYXNz"}
+
+ custom_auth_example:
+ url: "https://my-mcp-server.com/mcp"
+ auth_type: "authorization"
+ auth_value: "Token example123" # headers={"Authorization": "Token example123"}
+
+ # Example with extra headers forwarding
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: "bearer_token"
+ auth_value: "ghp_example_token"
+ extra_headers: ["custom_key", "x-custom-header"] # These headers will be forwarded from client
```
-#### 2. Start LiteLLM Gateway
+### MCP Aliases
+
+You can define aliases for your MCP servers in the `litellm_settings` section. This allows you to:
+
+1. **Map friendly names to server names**: Use shorter, more memorable aliases
+2. **Override server aliases**: If a server doesn't have an alias defined, the system will use the first matching alias from `mcp_aliases`
+3. **Ensure uniqueness**: Only the first alias for each server is used, preventing conflicts
+
+**Example:**
+```yaml
+litellm_settings:
+ mcp_aliases:
+ "github": "github_mcp_server" # Maps "github" alias to "github_mcp_server"
+ "zapier": "zapier_mcp_server" # Maps "zapier" alias to "zapier_mcp_server"
+ "docs": "deepwiki_mcp_server" # Maps "docs" alias to "deepwiki_mcp_server"
+ "github_alt": "github_mcp_server" # This will be ignored since "github" already maps to this server
+```
+
+**Benefits:**
+- **Simplified tool access**: Use `github_create_issue` instead of `github_mcp_server_create_issue`
+- **Consistent naming**: Standardize alias patterns across your organization
+- **Easy migration**: Change server names without breaking existing tool references
+
+
+
+
+## MCP Tool Filtering
+
+Control which tools are available from your MCP servers. You can either allow only specific tools or block dangerous ones.
-
+
-```shell title="Docker Run" showLineNumbers
-docker run -d \
- -p 4000:4000 \
- -e OPENAI_API_KEY=$OPENAI_API_KEY \
- --name my-app \
- -v $(pwd)/my_config.yaml:/app/config.yaml \
- my-app:latest \
- --config /app/config.yaml \
- --port 4000 \
- --detailed_debug \
+Use `allowed_tools` to specify exactly which tools users can access. All other tools will be blocked.
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: oauth2
+ authorization_url: https://github.com/login/oauth/authorize
+ token_url: https://github.com/login/oauth/access_token
+ client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+ client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+ scopes: ["public_repo", "user:email"]
+ allowed_tools: ["list_tools"]
+ # only list_tools will be available
```
+**Use this when:**
+- You want strict control over which tools are available
+- You're in a high-security environment
+- You're testing a new MCP server with limited tools
+
+
+
+
+Use `disallowed_tools` to block specific tools. All other tools will be available.
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: oauth2
+ authorization_url: https://github.com/login/oauth/authorize
+ token_url: https://github.com/login/oauth/access_token
+ client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+ client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+ scopes: ["public_repo", "user:email"]
+ disallowed_tools: ["repo_delete"]
+ # only repo_delete will be blocked
+```
+
+**Use this when:**
+- Most tools are safe, but you want to block a few dangerous ones
+- You want to prevent expensive API calls
+- You're gradually adding restrictions to an existing server
+
+
+
+
+### Important Notes
+
+- If you specify both `allowed_tools` and `disallowed_tools`, the allowed list takes priority
+- Tool names are case-sensitive
+
+## MCP Server Access Control
+
+LiteLLM Proxy provides two methods for controlling access to specific MCP servers:
+
+1. **URL-based Namespacing** - Use URL paths to directly access specific servers or access groups
+2. **Header-based Namespacing** - Use the `x-mcp-servers` header to specify which servers to access
+
+---
+
+### Method 1: URL-based Namespacing
+
+LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/mcp/`. This allows you to:
+
+- **Direct URL Access**: Point MCP clients directly to specific servers or access groups via URL
+- **Simplified Configuration**: Use URLs instead of headers for server selection
+- **Access Group Support**: Use access group names in URLs for grouped server access
+
+#### URL Format
+
+```
+/mcp/
+```
+
+**Examples:**
+- `/mcp/github` - Access tools from the "github" MCP server
+- `/mcp/zapier` - Access tools from the "zapier" MCP server
+- `/mcp/dev_group` - Access tools from all servers in the "dev_group" access group
+- `/mcp/github,zapier` - Access tools from multiple specific servers
+
+#### Usage Examples
+
+
+
+
+```bash title="cURL Example with URL Namespacing" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "/mcp/github",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+This example uses URL namespacing to access only the "github" MCP server.
+
-
+
-```shell title="litellm pip" showLineNumbers
-litellm --config config.yaml --detailed_debug
+```bash title="cURL Example with URL Namespacing" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "/mcp/dev_group",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
```
+This example uses URL namespacing to access all servers in the "dev_group" access group.
+
+
+
+
+
+```json title="Cursor MCP Configuration with URL Namespacing" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "/mcp/github,zapier",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+ }
+ }
+ }
+}
+```
+
+This configuration uses URL namespacing to access tools from both "github" and "zapier" MCP servers.
+
+
+
+
+#### Benefits of URL Namespacing
+
+- **Direct Access**: No need for additional headers to specify servers
+- **Clean URLs**: Self-documenting URLs that clearly indicate which servers are accessible
+- **Access Group Support**: Use access group names for grouped server access
+- **Multiple Servers**: Specify multiple servers in a single URL with comma separation
+- **Simplified Configuration**: Easier setup for MCP clients that prefer URL-based configuration
+
+---
+
+### Method 2: Header-based Namespacing
+
+You can choose to access specific MCP servers and only list their tools using the `x-mcp-servers` header. This header allows you to:
+- Limit tool access to one or more specific MCP servers
+- Control which tools are available in different environments or use cases
+
+The header accepts a comma-separated list of server aliases: `"alias_1,Server2,Server3"`
+
+**Notes:**
+- If the header is not provided, tools from all available MCP servers will be accessible
+- This method works with the standard LiteLLM MCP endpoint
+
+
+
+
+```bash title="cURL Example with Header Namespacing" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "/mcp/",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+In this example, the request will only have access to tools from the "alias_1" MCP server.
+
+
+
+
+
+```bash title="cURL Example with Header Namespacing" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "/mcp/",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1,Server2"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+This configuration restricts the request to only use tools from the specified MCP servers.
+
+
+
+
+
+```json title="Cursor MCP Configuration with Header Namespacing" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "/mcp/",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1,Server2"
+ }
+ }
+ }
+}
+```
+
+This configuration in Cursor IDE settings will limit tool access to only the specified MCP servers.
+
+
+
+
+---
+
+### Comparison: Header vs URL Namespacing
+
+| Feature | Header Namespacing | URL Namespacing |
+|---------|-------------------|-----------------|
+| **Method** | Uses `x-mcp-servers` header | Uses URL path `/mcp/` |
+| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `/mcp/` endpoint |
+| **Configuration** | Requires additional header | Self-contained in URL |
+| **Multiple Servers** | Comma-separated in header | Comma-separated in URL path |
+| **Access Groups** | Supported via header | Supported via URL path |
+| **Client Support** | Works with all MCP clients | Works with URL-aware MCP clients |
+| **Use Case** | Dynamic server selection | Fixed server configuration |
+
+
+
+
+```bash title="cURL Example with Server Segregation" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "/mcp/",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+In this example, the request will only have access to tools from the "alias_1" MCP server.
+
+
+
+
+
+```bash title="cURL Example with Server Segregation" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1,Server2"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+This configuration restricts the request to only use tools from the specified MCP servers.
+
+
+
+
+
+```json title="Cursor MCP Configuration with Server Segregation" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "litellm_proxy",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "x-mcp-servers": "alias_1,Server2"
+ }
+ }
+ }
+}
+```
+
+This configuration in Cursor IDE settings will limit tool access to only the specified MCP server.
+
+
+
+
+### Grouping MCPs (Access Groups)
+
+MCP Access Groups allow you to group multiple MCP servers together for easier management.
+
+#### 1. Create an Access Group
+
+##### A. Creating Access Groups using Config:
+
+```yaml title="Creating access groups for MCP using the config" showLineNumbers
+mcp_servers:
+ "deepwiki_mcp":
+ url: https://mcp.deepwiki.com/mcp
+ transport: "http"
+ auth_type: "none"
+ access_groups: ["dev_group"]
+```
+
+While adding `mcp_servers` using the config:
+- Pass in a list of strings inside `access_groups`
+- These groups can then be used for segregating access using keys, teams and MCP clients using headers
+
+##### B. Creating Access Groups using UI
+
+To create an access group:
+- Go to MCP Servers in the LiteLLM UI
+- Click "Add a New MCP Server"
+- Under "MCP Access Groups", create a new group (e.g., "dev_group") by typing it
+- Add the same group name to other servers to group them together
+
+
+
+#### 2. Use Access Group in Cursor
+
+Include the access group name in the `x-mcp-servers` header:
+
+```json title="Cursor Configuration with Access Groups" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "litellm_proxy",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "x-mcp-servers": "dev_group"
+ }
+ }
+ }
+}
+```
+
+This gives you access to all servers in the "dev_group" access group.
+- Which means that if deepwiki server (and any other servers) which have the access group `dev_group` assigned to them will be available for tool calling
+
+#### Advanced: Connecting Access Groups to API Keys
+
+When creating API keys, you can assign them to specific access groups for permission management:
+
+- Go to "Keys" in the LiteLLM UI and click "Create Key"
+- Select the desired MCP access groups from the dropdown
+- The key will have access to all MCP servers in those groups
+- This is reflected in the Test Key page
+
+
+
+
+## Forwarding Custom Headers to MCP Servers
+
+LiteLLM supports forwarding additional custom headers from MCP clients to backend MCP servers using the `extra_headers` configuration parameter. This allows you to pass custom authentication tokens, API keys, or other headers that your MCP server requires.
+
+### Configuration
+
+
+
+
+Configure `extra_headers` in your MCP server configuration to specify which header names should be forwarded:
+
+```yaml title="config.yaml with extra_headers" showLineNumbers
+mcp_servers:
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: "bearer_token"
+ auth_value: "ghp_default_token"
+ extra_headers: ["custom_key", "x-custom-header", "Authorization"]
+ description: "GitHub MCP server with custom header forwarding"
+```
+
+
+
+Use this when giving users access to a [group of MCP servers](#grouping-mcps-access-groups).
+
+**Format:** `x-mcp-{server_alias}-{header_name}: value`
+
+This allows you to use different authentication for different MCP servers.
+
+
+**Examples:**
+- `x-mcp-github-authorization: Bearer ghp_xxxxxxxxx` - GitHub MCP server with Bearer token
+- `x-mcp-zapier-x-api-key: sk-xxxxxxxxx` - Zapier MCP server with API key
+- `x-mcp-deepwiki-authorization: Basic base64_encoded_creds` - DeepWiki MCP server with Basic auth
+
+```python title="Python Client with Server-Specific Auth" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+# Standard MCP configuration with multiple servers
+config = {
+ "mcpServers": {
+ "mcp_group": {
+ "url": "http://localhost:4000/mcp",
+ "headers": {
+ "x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
+ "x-litellm-api-key": "Bearer sk-1234",
+ "x-mcp-github-authorization": "Bearer gho_token",
+ "x-mcp-zapier-x-api-key": "sk-xxxxxxxxx",
+ "x-mcp-deepwiki-authorization": "Basic base64_encoded_creds",
+ "custom_key": "value"
+ }
+ }
+ }
+}
+
+# Create a client that connects to all servers
+client = Client(config)
+
+
+async def main():
+ async with client:
+ tools = await client.list_tools()
+ print(f"Available tools: {tools}")
+
+ # call mcp
+ await client.call_tool(
+ name="github_mcp-search_issues",
+ arguments={'query': 'created:>2024-01-01', 'sort': 'created', 'order': 'desc', 'perPage': 30}
+ )
+
+if __name__ == "__main__":
+ asyncio.run(main())
+
+```
+
+
+
+**Benefits:**
+- **Server-specific authentication**: Each MCP server can use different auth methods
+- **Better security**: No need to share the same auth token across all servers
+- **Flexible header names**: Support for different auth header types (authorization, x-api-key, etc.)
+- **Clean separation**: Each server's auth is clearly identified
+
+
+
-#### 3. Make an LLM API request
+### Client Usage
-In this example we will do the following:
+When connecting from MCP clients, include the custom headers that match the `extra_headers` configuration:
-1. Use MCP client to list MCP tools on LiteLLM Proxy
-2. Use `transform_mcp_tool_to_openai_tool` to convert MCP tools to OpenAI tools
-3. Provide the MCP tools to `gpt-4o`
-4. Handle tool call from `gpt-4o`
-5. Convert OpenAI tool call to MCP tool call
-6. Execute tool call on MCP server
+
+
-```python title="MCP Client List Tools" showLineNumbers
+```python title="FastMCP Client with Custom Headers" showLineNumbers
+from fastmcp import Client
import asyncio
-from openai import AsyncOpenAI
-from openai.types.chat import ChatCompletionUserMessageParam
-from mcp import ClientSession
-from mcp.client.sse import sse_client
-from litellm.experimental_mcp_client.tools import (
- transform_mcp_tool_to_openai_tool,
- transform_openai_tool_call_request_to_mcp_tool_call_request,
+
+# MCP client configuration with custom headers
+config = {
+ "mcpServers": {
+ "github": {
+ "url": "http://localhost:4000/github_mcp/mcp",
+ "headers": {
+ "x-litellm-api-key": "Bearer sk-1234",
+ "Authorization": "Bearer gho_token",
+ "custom_key": "custom_value",
+ "x-custom-header": "additional_data"
+ }
+ }
+ }
+}
+
+# Create a client that connects to the server
+client = Client(config)
+
+async def main():
+ async with client:
+ # List available tools
+ tools = await client.list_tools()
+ print(f"Available tools: {tools}")
+
+ # Call a tool if available
+ if tools:
+ result = await client.call_tool(tools[0].name, {})
+ print(f"Tool result: {result}")
+
+# Run the client
+asyncio.run(main())
+```
+
+
+
+
+
+```json title="Cursor MCP Configuration with Custom Headers" showLineNumbers
+{
+ "mcpServers": {
+ "GitHub": {
+ "url": "http://localhost:4000/github_mcp/mcp",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "Authorization": "Bearer $GITHUB_TOKEN",
+ "custom_key": "custom_value",
+ "x-custom-header": "additional_data"
+ }
+ }
+ }
+}
+```
+
+
+
+
+
+```bash title="cURL with Custom Headers" showLineNumbers
+curl --location 'http://localhost:4000/github_mcp/mcp' \
+--header 'Content-Type: application/json' \
+--header 'x-litellm-api-key: Bearer sk-1234' \
+--header 'Authorization: Bearer gho_token' \
+--header 'custom_key: custom_value' \
+--header 'x-custom-header: additional_data' \
+--data '{
+ "jsonrpc": "2.0",
+ "id": 1,
+ "method": "tools/list"
+}'
+```
+
+
+
+
+### How It Works
+
+1. **Configuration**: Define `extra_headers` in your MCP server config with the header names you want to forward
+2. **Client Headers**: Include the corresponding headers in your MCP client requests
+3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server
+4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers
+
+### Use Cases
+
+- **Custom Authentication**: Forward custom API keys or tokens required by specific MCP servers
+- **Request Context**: Pass user identification, session data, or request tracking headers
+- **Third-party Integration**: Include headers required by external services that your MCP server integrates with
+- **Multi-tenant Systems**: Forward tenant-specific headers for proper request routing
+
+### Security Considerations
+
+- Only headers listed in `extra_headers` are forwarded to maintain security
+- Sensitive headers should be passed through environment variables when possible
+- Consider using server-specific auth headers for better security isolation
+
+---
+
+## MCP Oauth
+
+LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
+
+
+This configuration is currently available on the config.yaml, with UI support coming soon.
+
+```yaml
+mcp_servers:
+ github_mcp:
+ url: "https://api.githubcopilot.com/mcp"
+ auth_type: oauth2
+ authorization_url: https://github.com/login/oauth/authorize
+ token_url: https://github.com/login/oauth/access_token
+ client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+ client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+ scopes: ["public_repo", "user:email"]
+```
+
+## Using your MCP with client side credentials
+
+Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
+
+
+### New Server-Specific Auth Headers (Recommended)
+
+You can specify MCP auth tokens using server-specific headers in the format `x-mcp-{server_alias}-{header_name}`. This allows you to use different authentication for different MCP servers.
+
+**Benefits:**
+- **Server-specific authentication**: Each MCP server can use different auth methods
+- **Better security**: No need to share the same auth token across all servers
+- **Flexible header names**: Support for different auth header types (authorization, x-api-key, etc.)
+- **Clean separation**: Each server's auth is clearly identified
+
+### Legacy Auth Header (Deprecated)
+
+You can also specify your MCP auth token using the header `x-mcp-auth`. This will be forwarded to all MCP servers and is deprecated in favor of server-specific headers.
+
+
+
+
+#### Connect via OpenAI Responses API with Server-Specific Auth
+
+Use the OpenAI Responses API and include server-specific auth headers:
+
+```bash title="cURL Example with Server-Specific Auth" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-github-authorization": "Bearer YOUR_GITHUB_TOKEN",
+ "x-mcp-zapier-x-api-key": "YOUR_ZAPIER_API_KEY"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+#### Connect via OpenAI Responses API with Legacy Auth
+
+Use the OpenAI Responses API and include the `x-mcp-auth` header for your MCP server authentication:
+
+```bash title="cURL Example with Legacy MCP Auth" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-auth": YOUR_MCP_AUTH_TOKEN
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+
+
+
+
+#### Connect via LiteLLM Proxy Responses API with Server-Specific Auth
+
+Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint with server-specific authentication:
+
+```bash title="cURL Example with Server-Specific Auth" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-github-authorization": "Bearer YOUR_GITHUB_TOKEN",
+ "x-mcp-zapier-x-api-key": "YOUR_ZAPIER_API_KEY"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+#### Connect via LiteLLM Proxy Responses API with Legacy Auth
+
+Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint with MCP authentication:
+
+```bash title="cURL Example with Legacy MCP Auth" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-auth": "YOUR_MCP_AUTH_TOKEN"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+
+
+
+
+#### Connect via Cursor IDE with Server-Specific Auth
+
+Use tools directly from Cursor IDE with LiteLLM MCP and include server-specific authentication:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Cursor MCP Configuration with Server-Specific Auth" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "litellm_proxy",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "x-mcp-github-authorization": "Bearer $GITHUB_TOKEN",
+ "x-mcp-zapier-x-api-key": "$ZAPIER_API_KEY"
+ }
+ }
+ }
+}
+```
+
+#### Connect via Cursor IDE with Legacy Auth
+
+Use tools directly from Cursor IDE with LiteLLM MCP and include your MCP authentication token:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Cursor MCP Configuration with Legacy Auth" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "litellm_proxy",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+ "x-mcp-auth": "$MCP_AUTH_TOKEN"
+ }
+ }
+ }
+}
+```
+
+
+
+
+
+#### Connect via Streamable HTTP Transport with Server-Specific Auth
+
+Connect to LiteLLM MCP using HTTP transport with server-specific authentication:
+
+**Server URL:**
+```text showLineNumbers
+litellm_proxy
+```
+
+**Headers:**
+```text showLineNumbers
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
+x-mcp-github-authorization: Bearer YOUR_GITHUB_TOKEN
+x-mcp-zapier-x-api-key: YOUR_ZAPIER_API_KEY
+```
+
+#### Connect via Streamable HTTP Transport with Legacy Auth
+
+Connect to LiteLLM MCP using HTTP transport with MCP authentication:
+
+**Server URL:**
+```text showLineNumbers
+litellm_proxy
+```
+
+**Headers:**
+```text showLineNumbers
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
+x-mcp-auth: Bearer YOUR_MCP_AUTH_TOKEN
+```
+
+This URL can be used with any MCP client that supports HTTP transport. The `x-mcp-auth` header will be forwarded to your MCP server for authentication.
+
+
+
+
+
+#### Connect via Python FastMCP Client with Server-Specific Auth
+
+Use the Python FastMCP client to connect to your LiteLLM MCP server with server-specific authentication:
+
+```python title="Python FastMCP Example with Server-Specific Auth" showLineNumbers
+import asyncio
+import json
+
+from fastmcp import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
+# Create the transport with your LiteLLM MCP server URL and server-specific auth headers
+server_url = "litellm_proxy"
+transport = StreamableHttpTransport(
+ server_url,
+ headers={
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-github-authorization": "Bearer YOUR_GITHUB_TOKEN",
+ "x-mcp-zapier-x-api-key": "YOUR_ZAPIER_API_KEY"
+ }
)
+# Initialize the client with the transport
+client = Client(transport=transport)
+
async def main():
- # Initialize clients
-
- # point OpenAI client to LiteLLM Proxy
- client = AsyncOpenAI(api_key="sk-1234", base_url="http://localhost:4000")
+ # Connection is established here
+ print("Connecting to LiteLLM MCP server with server-specific authentication...")
+ async with client:
+ print(f"Client connected: {client.is_connected()}")
- # Point MCP client to LiteLLM Proxy
- async with sse_client("http://localhost:4000/mcp/") as (read, write):
- async with ClientSession(read, write) as session:
- await session.initialize()
+ # Make MCP calls within the context
+ print("Fetching available tools...")
+ tools = await client.list_tools()
- # 1. List MCP tools on LiteLLM Proxy
- mcp_tools = await session.list_tools()
- print("List of MCP tools for MCP server:", mcp_tools.tools)
-
- # Create message
- messages = [
- ChatCompletionUserMessageParam(
- content="Send an email about LiteLLM supporting MCP", role="user"
- )
- ]
-
- # 2. Use `transform_mcp_tool_to_openai_tool` to convert MCP tools to OpenAI tools
- # Since OpenAI only supports tools in the OpenAI format, we need to convert the MCP tools to the OpenAI format.
- openai_tools = [
- transform_mcp_tool_to_openai_tool(tool) for tool in mcp_tools.tools
- ]
-
- # 3. Provide the MCP tools to `gpt-4o`
- response = await client.chat.completions.create(
- model="gpt-4o",
- messages=messages,
- tools=openai_tools,
- tool_choice="auto",
- )
-
- # 4. Handle tool call from `gpt-4o`
- if response.choices[0].message.tool_calls:
- tool_call = response.choices[0].message.tool_calls[0]
- if tool_call:
-
- # 5. Convert OpenAI tool call to MCP tool call
- # Since MCP servers expect tools in the MCP format, we need to convert the OpenAI tool call to the MCP format.
- # This is done using litellm.experimental_mcp_client.tools.transform_openai_tool_call_request_to_mcp_tool_call_request
- mcp_call = (
- transform_openai_tool_call_request_to_mcp_tool_call_request(
- openai_tool=tool_call.model_dump()
- )
- )
-
- # 6. Execute tool call on MCP server
- result = await session.call_tool(
- name=mcp_call.name, arguments=mcp_call.arguments
- )
-
- print("Result:", result)
+ print(f"Available tools: {json.dumps([t.name for t in tools], indent=2)}")
+
+ # Example: Call a tool (replace 'tool_name' with an actual tool name)
+ if tools:
+ tool_name = tools[0].name
+ print(f"Calling tool: {tool_name}")
+
+ # Call the tool with appropriate arguments
+ result = await client.call_tool(tool_name, arguments={})
+ print(f"Tool result: {result}")
-# Run it
-asyncio.run(main())
+# Run the example
+if __name__ == "__main__":
+ asyncio.run(main())
```
+#### Connect via Python FastMCP Client with Legacy Auth
+
+Use the Python FastMCP client to connect to your LiteLLM MCP server with MCP authentication:
+
+```python title="Python FastMCP Example with Legacy MCP Auth" showLineNumbers
+import asyncio
+import json
+
+from fastmcp import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
+# Create the transport with your LiteLLM MCP server URL and auth headers
+server_url = "litellm_proxy"
+transport = StreamableHttpTransport(
+ server_url,
+ headers={
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "x-mcp-auth": "Bearer YOUR_MCP_AUTH_TOKEN"
+ }
+)
+
+# Initialize the client with the transport
+client = Client(transport=transport)
+
+
+async def main():
+ # Connection is established here
+ print("Connecting to LiteLLM MCP server with authentication...")
+ async with client:
+ print(f"Client connected: {client.is_connected()}")
+
+ # Make MCP calls within the context
+ print("Fetching available tools...")
+ tools = await client.list_tools()
+
+ print(f"Available tools: {json.dumps([t.name for t in tools], indent=2)}")
+
+ # Example: Call a tool (replace 'tool_name' with an actual tool name)
+ if tools:
+ tool_name = tools[0].name
+ print(f"Calling tool: {tool_name}")
+
+ # Call the tool with appropriate arguments
+ result = await client.call_tool(tool_name, arguments={})
+ print(f"Tool result: {result}")
+
+
+# Run the example
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+
+
+
+### Customize the MCP Auth Header Name
+
+By default, LiteLLM uses `x-mcp-auth` to pass your credentials to MCP servers. You can change this header name in one of the following ways:
+1. Set the `LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME` environment variable
+
+```bash title="Environment Variable" showLineNumbers
+export LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME="authorization"
+```
+
+
+2. Set the `mcp_client_side_auth_header_name` in the general settings on the config.yaml file
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: sk-xxxxxxx
+
+general_settings:
+ mcp_client_side_auth_header_name: "authorization"
+```
+
+#### Using the authorization header
+
+In this example the `authorization` header will be passed to the MCP server for authentication.
+
+```bash title="cURL with authorization header" showLineNumbers
+curl --location '/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+ "model": "gpt-4o",
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never",
+ "headers": {
+ "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+ "authorization": "Bearer sk-zapier-token-123"
+ }
+ }
+ ],
+ "input": "Run available tools",
+ "tool_choice": "required"
+}'
+```
+
+## LiteLLM Proxy - Walk through MCP Gateway
+LiteLLM exposes an MCP Gateway for admins to add all their MCP servers to LiteLLM. The key benefits of using LiteLLM Proxy with MCP are:
+
+1. Use a fixed endpoint for all MCP tools
+2. MCP Permission management by Key, Team, or User
+
+This video demonstrates how you can onboard an MCP server to LiteLLM Proxy, use it and set access controls.
+
+
+
## LiteLLM Python SDK MCP Bridge
LiteLLM Python SDK acts as a MCP bridge to utilize MCP tools with all LiteLLM supported models. LiteLLM offers the following features for using MCP
@@ -429,10 +1546,4 @@ async with stdio_client(server_params) as (read, write):
```
-
-
-### Permission Management
-
-Currently, all Virtual Keys are able to access the MCP endpoints. We are working on a feature to allow restricting MCP access by keys/teams/users/orgs.
-
-Join the discussion [here](https://github.com/BerriAI/litellm/discussions/9891)
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/my-website/docs/mcp_control.md b/docs/my-website/docs/mcp_control.md
new file mode 100644
index 00000000000..484cb13708c
--- /dev/null
+++ b/docs/my-website/docs/mcp_control.md
@@ -0,0 +1,45 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Permission Management
+
+Control which MCP servers and tools can be accessed by specific keys, teams, or organizations in LiteLLM. When a client attempts to list or call tools, LiteLLM enforces access controls based on configured permissions.
+
+## Overview
+
+LiteLLM provides fine-grained permission management for MCP servers, allowing you to:
+
+- **Restrict MCP access by entity**: Control which keys, teams, or organizations can access specific MCP servers
+- **Tool-level filtering**: Automatically filter available tools based on entity permissions
+- **Centralized control**: Manage all MCP permissions from the LiteLLM Admin UI or API
+
+This ensures that only authorized entities can discover and use MCP tools, providing an additional security layer for your MCP infrastructure.
+
+:::info Related Documentation
+- [MCP Overview](./mcp.md) - Learn about MCP in LiteLLM
+- [MCP Cost Tracking](./mcp_cost.md) - Track costs for MCP tool calls
+- [MCP Guardrails](./mcp_guardrail.md) - Apply security guardrails to MCP calls
+- [Using MCP](./mcp_usage.md) - How to use MCP with LiteLLM
+:::
+
+## How It Works
+
+LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
+
+When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
+
+
+
+
+## Set Allowed Tools for a Key, Team, or Organization
+
+Control which tools different teams can access from the same MCP server. For example, give your Engineering team access to `list_repositories`, `create_issue`, and `search_code`, while Sales only gets `search_code` and `close_issue`.
+
+
+This video shows how to set allowed tools for a Key, Team, or Organization.
+
+
diff --git a/docs/my-website/docs/mcp_cost.md b/docs/my-website/docs/mcp_cost.md
new file mode 100644
index 00000000000..4f5d65fe019
--- /dev/null
+++ b/docs/my-website/docs/mcp_cost.md
@@ -0,0 +1,121 @@
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Cost Tracking
+
+LiteLLM provides two ways to track costs for MCP tool calls:
+
+| Method | When to Use | What It Does |
+|--------|-------------|--------------|
+| **Config-based Cost Tracking** | Simple cost tracking with fixed costs per tool/server | Automatically tracks costs based on configuration |
+| **Custom Post-MCP Hook** | Dynamic cost tracking with custom logic | Allows custom cost calculations and response modifications |
+
+### Config-based Cost Tracking
+
+Configure fixed costs for MCP servers directly in your config.yaml:
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: sk-xxxxxxx
+
+mcp_servers:
+ zapier_server:
+ url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
+ mcp_info:
+ mcp_server_cost_info:
+ # Default cost for all tools in this server
+ default_cost_per_query: 0.01
+ # Custom cost for specific tools
+ tool_name_to_cost_per_query:
+ send_email: 0.05
+ create_document: 0.03
+
+ expensive_api_server:
+ url: "https://api.expensive-service.com/mcp"
+ mcp_info:
+ mcp_server_cost_info:
+ default_cost_per_query: 1.50
+```
+
+### Custom Post-MCP Hook
+
+Use this when you need dynamic cost calculation or want to modify the MCP response before it's returned to the user.
+
+#### 1. Create a custom MCP hook file
+
+```python title="custom_mcp_hook.py" showLineNumbers
+from typing import Optional
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.mcp import MCPPostCallResponseObject
+
+
+class CustomMCPCostTracker(CustomLogger):
+ """
+ Custom handler for MCP cost tracking and response modification
+ """
+
+ async def async_post_mcp_tool_call_hook(
+ self,
+ kwargs,
+ response_obj: MCPPostCallResponseObject,
+ start_time,
+ end_time
+ ) -> Optional[MCPPostCallResponseObject]:
+ """
+ Called after each MCP tool call.
+ Modify costs and response before returning to user.
+ """
+
+ # Extract tool information from kwargs
+ tool_name = kwargs.get("name", "")
+ server_name = kwargs.get("server_name", "")
+
+ # Calculate custom cost based on your logic
+ custom_cost = 42.00
+
+ # Set the response cost
+ response_obj.hidden_params.response_cost = custom_cost
+
+
+
+ return response_obj
+
+
+# Create instance for LiteLLM to use
+custom_mcp_cost_tracker = CustomMCPCostTracker()
+```
+
+#### 2. Configure in config.yaml
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: sk-xxxxxxx
+
+# Add your custom MCP hook
+callbacks:
+ - custom_mcp_hook.custom_mcp_cost_tracker
+
+mcp_servers:
+ zapier_server:
+ url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
+```
+
+#### 3. Start the proxy
+
+```shell
+$ litellm --config /path/to/config.yaml
+```
+
+When MCP tools are called, your custom hook will:
+1. Calculate costs based on your custom logic
+2. Modify the response if needed
+3. Track costs in LiteLLM's logging system
+
diff --git a/docs/my-website/docs/mcp_guardrail.md b/docs/my-website/docs/mcp_guardrail.md
new file mode 100644
index 00000000000..f71ea2fe5ef
--- /dev/null
+++ b/docs/my-website/docs/mcp_guardrail.md
@@ -0,0 +1,88 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Guardrails
+
+LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
+
+### Supported MCP Guardrail Modes
+
+MCP guardrails support the following modes:
+
+- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
+- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
+
+### Configuration Examples
+
+Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
+
+```yaml title="config.yaml" showLineNumbers
+guardrails:
+ - guardrail_name: "mcp-input-validation"
+ litellm_params:
+ guardrail: presidio # or other supported guardrails
+ mode: "pre_mcp_call" # or during_mcp_call
+ pii_entities_config:
+ CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers
+ EMAIL_ADDRESS: "MASK" # Will mask email addresses
+ PHONE_NUMBER: "MASK" # Will mask phone numbers
+ default_on: true
+```
+
+
+### Usage Examples
+
+#### Testing Pre-MCP Call Guardrails
+
+Test your MCP guardrails with a request that includes sensitive information:
+
+```bash title="Test MCP Guardrail" showLineNumbers
+curl http://localhost:4000/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
+ ],
+ "guardrails": ["mcp-input-validation"]
+ }'
+```
+
+The request will be processed as follows:
+1. Credit card number will be blocked (request rejected)
+2. Email address will be masked (e.g., replaced with ``)
+
+#### Using with MCP Tools
+
+When using MCP tools, guardrails will be applied to the tool inputs:
+
+```python title="Python Example with MCP Guardrails" showLineNumbers
+import openai
+
+client = openai.OpenAI(
+ api_key="your-api-key",
+ base_url="http://localhost:4000"
+)
+
+# This request will trigger MCP guardrails
+response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
+ ],
+ tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
+ guardrails=["mcp-input-validation"]
+)
+```
+
+### Supported Guardrail Providers
+
+MCP guardrails work with all LiteLLM-supported guardrail providers:
+
+- **Presidio**: PII detection and masking
+- **Bedrock**: AWS Bedrock guardrails
+- **Lakera**: Content moderation
+- **Aporia**: Custom guardrails
+- **Custom**: Your own guardrail implementations
\ No newline at end of file
diff --git a/docs/my-website/docs/mcp_usage.md b/docs/my-website/docs/mcp_usage.md
new file mode 100644
index 00000000000..ef9d8a5ed1b
--- /dev/null
+++ b/docs/my-website/docs/mcp_usage.md
@@ -0,0 +1,209 @@
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# Using your MCP
+
+This document covers how to use LiteLLM as an MCP Gateway. You can see how to use it with Responses API, Cursor IDE, and OpenAI SDK.
+
+### Use on LiteLLM UI
+
+Follow this walkthrough to use your MCP on LiteLLM UI
+
+
+
+### Use with Responses API
+
+Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
+
+Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
+
+
+
+
+
+```bash title="cURL Example" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+ "model": "gpt-5",
+ "input": [
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never"
+ }
+ ],
+ "stream": true,
+ "tool_choice": "required"
+}'
+```
+
+
+
+
+```python title="Python SDK Example" showLineNumbers
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234", # paste your litellm proxy api key here
+ base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+ model="gpt-5",
+ input=[
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ tools=[
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy",
+ "require_approval": "never"
+ }
+ ],
+ stream=True,
+ tool_choice="required"
+)
+
+for chunk in response:
+ print("response chunk: ", chunk)
+```
+
+
+
+
+#### Specifying MCP Tools
+
+You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
+
+To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
+
+
+
+
+```bash title="cURL Example with allowed_tools" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+ "model": "gpt-5",
+ "input": [
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ "tools": [
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy/mcp",
+ "require_approval": "never",
+ "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+ }
+ ],
+ "stream": true,
+ "tool_choice": "required"
+}'
+```
+
+
+
+
+```python title="Python SDK Example with allowed_tools" showLineNumbers
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234",
+ base_url="http://localhost:4000"
+)
+
+response = client.responses.create(
+ model="gpt-5",
+ input=[
+ {
+ "role": "user",
+ "content": "give me TLDR of what BerriAI/litellm repo is about",
+ "type": "message"
+ }
+ ],
+ tools=[
+ {
+ "type": "mcp",
+ "server_label": "litellm",
+ "server_url": "litellm_proxy/mcp",
+ "require_approval": "never",
+ "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+ }
+ ],
+ stream=True,
+ tool_choice="required"
+)
+
+print(response)
+```
+
+
+
+
+### Use with Cursor IDE
+
+Use tools directly from Cursor IDE with LiteLLM MCP:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Basic Cursor MCP Configuration" showLineNumbers
+{
+ "mcpServers": {
+ "LiteLLM": {
+ "url": "litellm_proxy",
+ "headers": {
+ "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+ }
+ }
+ }
+}
+```
+
+#### How it works when server_url="litellm_proxy"
+
+When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
+
+- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions
+- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call
+- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results
+- Response Integration: Tool results are sent back to LLM for final response generation
+- Output: Complete response combining LLM reasoning with tool execution results
+
+This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support.
+
+#### Auto-execution for require_approval: "never"
+
+Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction.
diff --git a/docs/my-website/docs/moderation.md b/docs/my-website/docs/moderation.md
index 95fe8b2856d..f9c2810bc8a 100644
--- a/docs/my-website/docs/moderation.md
+++ b/docs/my-website/docs/moderation.md
@@ -130,6 +130,8 @@ Here's the exact json output and type you can expect from all moderation calls:
## **Supported Providers**
+#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
| Provider |
|-------------|
| OpenAI |
diff --git a/docs/my-website/docs/observability/argilla.md b/docs/my-website/docs/observability/argilla.md
index dad28ce90c8..f59e8b49a68 100644
--- a/docs/my-website/docs/observability/argilla.md
+++ b/docs/my-website/docs/observability/argilla.md
@@ -50,7 +50,7 @@ For further configuration, please refer to the [Argilla documentation](https://d
## Usage
-
+
```python
import os
@@ -78,9 +78,9 @@ response = completion(
)
```
-
+
-
+
```yaml
litellm_settings:
@@ -90,7 +90,7 @@ litellm_settings:
llm_output: "response"
```
-
+
## Example Output
diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md
index 5a88964069d..e6b4fe769bc 100644
--- a/docs/my-website/docs/observability/braintrust.md
+++ b/docs/my-website/docs/observability/braintrust.md
@@ -2,25 +2,25 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# Braintrust - Evals + Logging
+# Braintrust - Evals + Logging
[Braintrust](https://www.braintrust.dev/) manages evaluations, logging, prompt playground, to data management for AI products.
-
## Quick Start
```python
-# pip install langfuse
+# pip install braintrust
import litellm
import os
-# set env
-os.environ["BRAINTRUST_API_KEY"] = ""
+# set env
+os.environ["BRAINTRUST_API_KEY"] = ""
+os.environ["BRAINTRUST_API_BASE"] = "https://api.braintrustdata.com/v1"
os.environ['OPENAI_API_KEY']=""
# set braintrust as a callback, litellm will send the data to braintrust
-litellm.callbacks = ["braintrust"]
-
+litellm.callbacks = ["braintrust"]
+
# openai call
response = litellm.completion(
model="gpt-3.5-turbo",
@@ -30,16 +30,17 @@ response = litellm.completion(
)
```
-
-
## OpenAI Proxy Usage
-1. Add keys to env
+1. Add keys to env
+
```env
-BRAINTRUST_API_KEY=""
+BRAINTRUST_API_KEY=""
+BRAINTRUST_API_BASE="https://api.braintrustdata.com/v1"
```
-2. Add braintrust to callbacks
+2. Add braintrust to callbacks
+
```yaml
model_list:
- model_name: gpt-3.5-turbo
@@ -47,12 +48,11 @@ model_list:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
-
litellm_settings:
callbacks: ["braintrust"]
```
-3. Test it!
+3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
@@ -69,6 +69,12 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
## Advanced - pass Project ID or name
+It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
+
+### Custom Span Names
+
+You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
+
@@ -77,12 +83,31 @@ response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
- ],
+ ],
metadata={
"project_id": "1234",
# passing project_name will try to find a project with that name, or create one if it doesn't exist
# if both project_id and project_name are passed, project_id will be used
- # "project_name": "my-special-project"
+ # "project_name": "my-special-project",
+ # custom span name for this operation (default: "Chat Completion")
+ "span_name": "User Greeting Handler"
+ }
+)
+```
+
+Note: Other `metadata` can be included here as well when using the SDK.
+
+```python
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "Hi 👋 - i'm openai"}
+ ],
+ metadata={
+ "project_id": "1234",
+ "span_name": "Custom Operation",
+ "item1": "an item",
+ "item2": "another item"
}
)
```
@@ -103,7 +128,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
{ "role": "user", "content": "What time is it now? Use your tool"}
],
"metadata": {
- "project_id": "my-special-project"
+ "project_id": "my-special-project",
+ "span_name": "Tool Usage Request"
}
}'
```
@@ -127,8 +153,9 @@ response = client.chat.completions.create(
}
],
extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
- "metadata": { # 👈 use for logging additional params (e.g. to langfuse)
- "project_id": "my-special-project"
+ "metadata": { # 👈 use for logging additional params (e.g. to braintrust)
+ "project_id": "my-special-project",
+ "span_name": "Poetry Generation"
}
}
)
@@ -141,10 +168,16 @@ For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions)
-## Full API Spec
+You can use `BRAINTRUST_API_BASE` to point to your self-hosted Braintrust data plane. Read more about this [here](https://www.braintrust.dev/docs/guides/self-hosting).
-Here's everything you can pass in metadata for a braintrust request
+## Full API Spec
-`braintrust_*` - any metadata field starting with `braintrust_` will be passed as metadata to the logging request
+Here's everything you can pass in metadata for a braintrust request
-`project_id` - set the project id for a braintrust call. Default is `litellm`.
\ No newline at end of file
+`braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
+
+`project_id` - Set the project id for a braintrust call. Default is `litellm`.
+
+`project_name` - Set the project name for a braintrust call. Will try to find a project with that name, or create one if it doesn't exist. If both `project_id` and `project_name` are passed, `project_id` will be used.
+
+`span_name` - Set a custom span name for the operation. Default is `"Chat Completion"`. Use this to provide more descriptive names for different types of operations in your application (e.g., "User Query", "Document Summary", "Code Generation").
diff --git a/docs/my-website/docs/observability/callbacks.md b/docs/my-website/docs/observability/callbacks.md
index 69cb0d053ee..b752bdc2764 100644
--- a/docs/my-website/docs/observability/callbacks.md
+++ b/docs/my-website/docs/observability/callbacks.md
@@ -4,9 +4,16 @@
liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
-liteLLM supports:
+:::tip
+**New to LiteLLM Callbacks?**
+
+- For proxy/server logging and observability, see the [Proxy Logging Guide](https://docs.litellm.ai/docs/proxy/logging).
+- To write your own callback logic, see the [Custom Callbacks Guide](https://docs.litellm.ai/docs/observability/custom_callback).
+:::
+
+
+### Supported Callback Integrations
-- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [LangSmith](https://www.langchain.com/langsmith)
@@ -16,9 +23,20 @@ liteLLM supports:
- [Sentry](https://docs.sentry.io/platforms/python/)
- [PostHog](https://posthog.com/docs/libraries/python)
- [Slack](https://slack.dev/bolt-python/concepts)
+- [Arize](https://docs.arize.com/)
+- [PromptLayer](https://docs.promptlayer.com/)
This is **not** an extensive list. Please check the dropdown for all logging integrations.
+### Related Cookbooks
+Try out our cookbooks for code snippets and interactive demos:
+
+- [Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Langfuse.ipynb)
+- [Lunary Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Lunary.ipynb)
+- [Arize Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Arize.ipynb)
+- [Proxy + Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb)
+- [PromptLayer Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_PromptLayer.ipynb)
+
### Quick Start
```python
diff --git a/docs/my-website/docs/observability/cloudzero.md b/docs/my-website/docs/observability/cloudzero.md
new file mode 100644
index 00000000000..f213ef64e13
--- /dev/null
+++ b/docs/my-website/docs/observability/cloudzero.md
@@ -0,0 +1,209 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CloudZero Integration
+
+LiteLLM provides an integration with CloudZero's AnyCost API, allowing you to export your LLM usage data to CloudZero for cost tracking analysis.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Export LiteLLM usage data to CloudZero AnyCost API for cost tracking and analysis |
+| callback name | `cloudzero`|
+| Supported Operations | • Automatic hourly data export • Manual data export • Dry run testing • Cost and token usage tracking |
+| Data Format | CloudZero Billing Format (CBF) with proper resource tagging |
+| Export Frequency | Hourly (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`) |
+
+## Environment Variables
+
+| Variable | Required | Description | Example |
+|----------|----------|-------------|---------|
+| `CLOUDZERO_API_KEY` | Yes | Your CloudZero API key | `cz_api_xxxxxxxxxx` |
+| `CLOUDZERO_CONNECTION_ID` | Yes | CloudZero connection ID for data submission | `conn_xxxxxxxxxx` |
+| `CLOUDZERO_TIMEZONE` | No | Timezone for date handling (default: UTC) | `America/New_York` |
+| `CLOUDZERO_EXPORT_INTERVAL_MINUTES` | No | Export frequency in minutes (default: 60) | `60` |
+
+## Setup
+
+### End to End Video Walkthrough
+This video walks through the entire process of setting up LiteLLM with CloudZero integration and viewing LiteLLM exported usage data in CloudZero.
+
+
+
+### Step 1: Configure Environment Variables
+
+Set your CloudZero credentials in your environment:
+
+```bash
+export CLOUDZERO_API_KEY="cz_api_xxxxxxxxxx"
+export CLOUDZERO_CONNECTION_ID="conn_xxxxxxxxxx"
+export CLOUDZERO_TIMEZONE="UTC" # Optional, defaults to UTC
+```
+
+### Step 2: Enable CloudZero Integration
+
+Add the CloudZero callback to your LiteLLM configuration YAML file:
+
+
+```yaml
+model_list:
+ - model_name: gpt-4o
+ litellm_params:
+ model: openai/gpt-4o
+ api_key: sk-xxxxxxx
+
+litellm_settings:
+ callbacks: ["cloudzero"] # Enable CloudZero integration
+```
+
+### Step 3: Start LiteLLM Proxy
+
+Start your LiteLLM proxy with the configuration:
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Testing Your Setup
+
+### Dry Run Export
+
+Call the dry run endpoint to test your CloudZero configuration without sending data to CloudZero. This endpoint will not send any data to CloudZero, but will return the data that would be exported.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/dry-run" \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "limit": 10
+ }' | jq
+```
+
+**Expected Response:**
+```json
+{
+ "message": "CloudZero dry run export completed successfully.",
+ "status": "success",
+ "dry_run_data": {
+ "usage_data": [...],
+ "cbf_data": [...],
+ "summary": {
+ "total_cost": 0.05,
+ "total_tokens": 1250,
+ "total_records": 10
+ }
+ }
+}
+```
+
+### Manual Export
+
+Call the export endpoint to send data immediately to CloudZero. We suggest setting a small `limit` to test the export. This will only export the last 10 records to CloudZero. Note: Cloudzero can take up to 15 minutes to process the exported data.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "limit": 10
+ }' | jq
+```
+
+**Expected Response:**
+```json
+{
+ "message": "CloudZero export completed successfully",
+ "status": "success"
+}
+```
+
+## Data Export Details
+
+### Automatic Export Schedule
+
+- **Frequency**: Every 60 minutes (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`)
+- **Data Processing**: LiteLLM automatically processes and exports usage data hourly
+- **CloudZero Processing**: CloudZero typically takes 10-15 minutes to process data from LiteLLM
+
+### Data Format
+
+LiteLLM exports data in CloudZero Billing Format (CBF) with the following structure:
+
+```json
+{
+ "time/usage_start": "2024-01-15T14:00:00Z",
+ "cost/cost": 0.002,
+ "usage/amount": 150,
+ "usage/units": "tokens",
+ "resource/id": "czrn:litellm:openai:cross-region:team-123:llm-usage:gpt-4o",
+ "resource/service": "litellm",
+ "resource/account": "team-123",
+ "resource/region": "cross-region",
+ "resource/usage_family": "llm-usage",
+ "resource/tag:provider": "openai",
+ "resource/tag:model": "gpt-4o",
+ "resource/tag:prompt_tokens": "100",
+ "resource/tag:completion_tokens": "50"
+}
+```
+
+### Resource Tagging
+
+LiteLLM automatically creates comprehensive resource tags for cost attribution:
+
+- **Provider Tags**: `openai`, `anthropic`, `azure`, etc.
+- **Model Tags**: Specific model names like `gpt-4o`, `claude-3-sonnet`
+- **Team/User Tags**: Team IDs and user IDs for cost allocation
+- **Token Breakdown**: Separate tracking of prompt and completion tokens
+- **Usage Metrics**: Total tokens consumed per request
+
+## Advanced Configuration
+
+### Custom Export Frequency
+
+Change the export frequency (not recommended to go below 60 minutes):
+
+```bash
+export CLOUDZERO_EXPORT_INTERVAL_MINUTES=120 # Export every 2 hours
+```
+
+### Custom Time Range Export
+
+Export data for a specific time range:
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "start_time_utc": "2024-01-15T00:00:00Z",
+ "end_time_utc": "2024-01-15T23:59:59Z",
+ "operation": "replace_hourly"
+ }' | jq
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing Credentials Error**
+ ```
+ CloudZero configuration missing. Please set CLOUDZERO_API_KEY and CLOUDZERO_CONNECTION_ID environment variables.
+ ```
+ **Solution**: Ensure both environment variables are set with valid values.
+
+2. **Connection Issues**
+ - Verify your CloudZero API key is valid
+ - Check that the connection ID exists in your CloudZero account
+ - Ensure your proxy has internet access to reach CloudZero's API
+
+3. **No Data in CloudZero**
+ - CloudZero can take 10-15 minutes to process data
+ - Check that your LiteLLM proxy is generating usage data
+ - Use the dry-run endpoint to verify data is being formatted correctly
+
+## Related Links
+
+- [CloudZero Documentation](https://docs.cloudzero.com/)
+- [CloudZero AnyCost API](https://docs.cloudzero.com/reference/anycost-api)
diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md
index cc586b2e5d9..cfe97ca42c0 100644
--- a/docs/my-website/docs/observability/custom_callback.md
+++ b/docs/my-website/docs/observability/custom_callback.md
@@ -4,7 +4,6 @@
**For PROXY** [Go Here](../proxy/logging.md#custom-callback-class-async)
:::
-
## Callback Class
You can create a custom callback class to precisely log events as they occur in litellm.
@@ -57,6 +56,34 @@ def async completion():
asyncio.run(completion())
```
+## Common Hooks
+
+- `async_log_success_event` - Log successful API calls
+- `async_log_failure_event` - Log failed API calls
+- `log_pre_api_call` - Log before API call
+- `log_post_api_call` - Log after API call
+
+**Proxy-only hooks** (only work with LiteLLM Proxy):
+- `async_post_call_success_hook` - Access user data + modify responses
+- `async_pre_call_hook` - Modify requests before sending
+
+### Example: Modifying the Response in async_post_call_success_hook
+
+You can use `async_post_call_success_hook` to add custom headers or metadata to the response before it is returned to the client. For example:
+
+```python
+async def async_post_call_success_hook(data, user_api_key_dict, response):
+ # Add a custom header to the response
+ additional_headers = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {}
+ additional_headers["x-litellm-custom-header"] = "my-value"
+ if not hasattr(response, "_hidden_params"):
+ response._hidden_params = {}
+ response._hidden_params["additional_headers"] = additional_headers
+ return response
+```
+
+This allows you to inject custom metadata or headers into the response for downstream consumers. You can use this pattern to pass information to clients, proxies, or observability tools.
+
## Callback Functions
If you just want to log on a specific event (e.g. on input) - you can use callback functions.
@@ -174,260 +201,87 @@ async def test_chat_openai():
asyncio.run(test_chat_openai())
```
-:::info
+## What's Available in kwargs?
-We're actively trying to expand this to other event types. [Tell us if you need this!](https://github.com/BerriAI/litellm/issues/1007)
-:::
-
-## What's in kwargs?
-
-Notice we pass in a kwargs argument to custom callback.
-```python
-def custom_callback(
- kwargs, # kwargs to completion
- completion_response, # response from completion
- start_time, end_time # start/end time
-):
- # Your custom code here
- print("LITELLM: in custom callback function")
- print("kwargs", kwargs)
- print("completion_response", completion_response)
- print("start_time", start_time)
- print("end_time", end_time)
-```
-
-This is a dictionary containing all the model-call details (the params we receive, the values we send to the http endpoint, the response we receive, stacktrace in case of errors, etc.).
-
-This is all logged in the [model_call_details via our Logger](https://github.com/BerriAI/litellm/blob/fc757dc1b47d2eb9d0ea47d6ad224955b705059d/litellm/utils.py#L246).
-
-Here's exactly what you can expect in the kwargs dictionary:
-```shell
-### DEFAULT PARAMS ###
-"model": self.model,
-"messages": self.messages,
-"optional_params": self.optional_params, # model-specific params passed in
-"litellm_params": self.litellm_params, # litellm-specific params passed in (e.g. metadata passed to completion call)
-"start_time": self.start_time, # datetime object of when call was started
-
-### PRE-API CALL PARAMS ### (check via kwargs["log_event_type"]="pre_api_call")
-"input" = input # the exact prompt sent to the LLM API
-"api_key" = api_key # the api key used for that LLM API
-"additional_args" = additional_args # any additional details for that API call (e.g. contains optional params sent)
-
-### POST-API CALL PARAMS ### (check via kwargs["log_event_type"]="post_api_call")
-"original_response" = original_response # the original http response received (saved via response.text)
-
-### ON-SUCCESS PARAMS ### (check via kwargs["log_event_type"]="successful_api_call")
-"complete_streaming_response" = complete_streaming_response # the complete streamed response (only set if `completion(..stream=True)`)
-"end_time" = end_time # datetime object of when call was completed
-
-### ON-FAILURE PARAMS ### (check via kwargs["log_event_type"]="failed_api_call")
-"exception" = exception # the Exception raised
-"traceback_exception" = traceback_exception # the traceback generated via `traceback.format_exc()`
-"end_time" = end_time # datetime object of when call was completed
-```
-
-
-### Cache hits
-
-Cache hits are logged in success events as `kwarg["cache_hit"]`.
-
-Here's an example of accessing it:
-
- ```python
- import litellm
-from litellm.integrations.custom_logger import CustomLogger
-from litellm import completion, acompletion, Cache
-
-class MyCustomHandler(CustomLogger):
- async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
- print(f"On Success")
- print(f"Value of Cache hit: {kwargs['cache_hit']"})
-
-async def test_async_completion_azure_caching():
- customHandler_caching = MyCustomHandler()
- litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
- litellm.callbacks = [customHandler_caching]
- unique_time = time.time()
- response1 = await litellm.acompletion(model="azure/chatgpt-v-2",
- messages=[{
- "role": "user",
- "content": f"Hi 👋 - i'm async azure {unique_time}"
- }],
- caching=True)
- await asyncio.sleep(1)
- print(f"customHandler_caching.states pre-cache hit: {customHandler_caching.states}")
- response2 = await litellm.acompletion(model="azure/chatgpt-v-2",
- messages=[{
- "role": "user",
- "content": f"Hi 👋 - i'm async azure {unique_time}"
- }],
- caching=True)
- await asyncio.sleep(1) # success callbacks are done in parallel
- print(f"customHandler_caching.states post-cache hit: {customHandler_caching.states}")
- assert len(customHandler_caching.errors) == 0
- assert len(customHandler_caching.states) == 4 # pre, post, success, success
- ```
-
-### Get complete streaming response
-
-LiteLLM will pass you the complete streaming response in the final streaming chunk as part of the kwargs for your custom callback function.
+The kwargs dictionary contains all the details about your API call:
```python
-# litellm.set_verbose = False
- def custom_callback(
- kwargs, # kwargs to completion
- completion_response, # response from completion
- start_time, end_time # start/end time
- ):
- # print(f"streaming response: {completion_response}")
- if "complete_streaming_response" in kwargs:
- print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}")
-
- # Assign the custom callback function
- litellm.success_callback = [custom_callback]
-
- response = completion(model="claude-instant-1", messages=messages, stream=True)
- for idx, chunk in enumerate(response):
- pass
-```
-
-
-### Log additional metadata
-
-LiteLLM accepts a metadata dictionary in the completion call. You can pass additional metadata into your completion call via `completion(..., metadata={"key": "value"})`.
-
-Since this is a [litellm-specific param](https://github.com/BerriAI/litellm/blob/b6a015404eed8a0fa701e98f4581604629300ee3/litellm/main.py#L235), it's accessible via kwargs["litellm_params"]
-
-```python
-from litellm import completion
-import os, litellm
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-def custom_callback(
- kwargs, # kwargs to completion
- completion_response, # response from completion
- start_time, end_time # start/end time
-):
- print(kwargs["litellm_params"]["metadata"])
+def custom_callback(kwargs, completion_response, start_time, end_time):
+ # Access common data
+ model = kwargs.get("model")
+ messages = kwargs.get("messages", [])
+ cost = kwargs.get("response_cost", 0)
+ cache_hit = kwargs.get("cache_hit", False)
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = litellm.completion(model="gpt-3.5-turbo", messages=messages, metadata={"hello": "world"})
+ # Access metadata you passed in
+ metadata = kwargs.get("litellm_params", {}).get("metadata", {})
```
-## Examples
+**Key fields in kwargs:**
+- `model` - The model name
+- `messages` - Input messages
+- `response_cost` - Calculated cost
+- `cache_hit` - Whether response was cached
+- `litellm_params.metadata` - Your custom metadata
-### Custom Callback to track costs for Streaming + Non-Streaming
-By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async)
+## Practical Examples
+
+### Track API Costs
```python
+def track_cost_callback(kwargs, completion_response, start_time, end_time):
+ cost = kwargs["response_cost"] # litellm calculates this for you
+ print(f"Request cost: ${cost}")
-# Step 1. Write your custom callback function
-def track_cost_callback(
- kwargs, # kwargs to completion
- completion_response, # response from completion
- start_time, end_time # start/end time
-):
- try:
- response_cost = kwargs["response_cost"] # litellm calculates response cost for you
- print("regular response_cost", response_cost)
- except:
- pass
-
-# Step 2. Assign the custom callback function
litellm.success_callback = [track_cost_callback]
-# Step 3. Make litellm.completion call
-response = completion(
- model="gpt-3.5-turbo",
- messages=[
- {
- "role": "user",
- "content": "Hi 👋 - i'm openai"
- }
- ]
-)
-
-print(response)
+response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello"}])
```
-### Custom Callback to log transformed Input to LLMs
+### Log Inputs to LLMs
```python
-def get_transformed_inputs(
- kwargs,
-):
+def get_transformed_inputs(kwargs):
params_to_model = kwargs["additional_args"]["complete_input_dict"]
print("params to model", params_to_model)
litellm.input_callback = [get_transformed_inputs]
-def test_chat_openai():
- try:
- response = completion(model="claude-2",
- messages=[{
- "role": "user",
- "content": "Hi 👋 - i'm openai"
- }])
-
- print(response)
-
- except Exception as e:
- print(e)
- pass
+response = completion(model="claude-2", messages=[{"role": "user", "content": "Hello"}])
```
-#### Output
-```shell
-params to model {'model': 'claude-2', 'prompt': "\n\nHuman: Hi 👋 - i'm openai\n\nAssistant: ", 'max_tokens_to_sample': 256}
+### Send to External Service
+```python
+import requests
+
+def send_to_analytics(kwargs, completion_response, start_time, end_time):
+ data = {
+ "model": kwargs.get("model"),
+ "cost": kwargs.get("response_cost", 0),
+ "duration": (end_time - start_time).total_seconds()
+ }
+ requests.post("https://your-analytics.com/api", json=data)
+
+litellm.success_callback = [send_to_analytics]
```
-### Custom Callback to write to Mixpanel
+## Common Issues
+
+### Callback Not Called
+Make sure you:
+1. Register callbacks correctly: `litellm.callbacks = [MyHandler()]`
+2. Use the right hook names (check spelling)
+3. Don't use proxy-only hooks in library mode
+
+### Performance Issues
+- Use async hooks for I/O operations
+- Don't block in callback functions
+- Handle exceptions properly:
```python
-import mixpanel
-import litellm
-from litellm import completion
-
-def custom_callback(
- kwargs, # kwargs to completion
- completion_response, # response from completion
- start_time, end_time # start/end time
-):
- # Your custom code here
- mixpanel.track("LLM Response", {"llm_response": completion_response})
-
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = completion(
- model="gpt-3.5-turbo",
- messages=[
- {
- "role": "user",
- "content": "Hi 👋 - i'm openai"
- }
- ]
-)
-
-print(response)
-
+class SafeHandler(CustomLogger):
+ async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+ try:
+ await external_service(response_obj)
+ except Exception as e:
+ print(f"Callback error: {e}") # Log but don't break the flow
```
-
-
-
-
-
-
-
-
-
-
-
diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md
new file mode 100644
index 00000000000..08ebf8b28ce
--- /dev/null
+++ b/docs/my-website/docs/observability/datadog.md
@@ -0,0 +1,180 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# DataDog
+
+LiteLLM Supports logging to the following Datdog Integrations:
+- `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
+- `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
+- `ddtrace-run` [Datadog Tracing](#datadog-tracing)
+
+## Datadog Logs
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | [StandardLoggingPayload](../proxy/logging_spec) |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog Logs](https://docs.datadoghq.com/logs/) |
+
+
+We will use the `--config` to set `litellm.callbacks = ["datadog"]` this will log all successful LLM calls to DataDog
+
+**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: gpt-3.5-turbo
+litellm_settings:
+ callbacks: ["datadog"] # logs llm success + failure logs on datadog
+ service_callback: ["datadog"] # logs redis, postgres failures on datadog
+```
+
+
+## Datadog LLM Observability
+
+**Overview**
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | [StandardLoggingPayload](../proxy/logging_spec) |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) |
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: gpt-3.5-turbo
+litellm_settings:
+ callbacks: ["datadog_llm_observability"] # logs llm success logs on datadog
+```
+
+
+
+**Step 2**: Set Required env variables for datadog
+
+```shell
+DD_API_KEY="5f2d0f310***********" # your datadog API Key
+DD_SITE="us5.datadoghq.com" # your datadog base url
+DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to differentiate dev vs. prod deployments
+```
+
+**Step 3**: Start the proxy, make a test request
+
+Start proxy
+
+```shell
+litellm --config config.yaml --debug
+```
+
+Test Request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ "metadata": {
+ "your-custom-metadata": "custom-field",
+ }
+}'
+```
+
+Expected output on Datadog
+
+
+
+### Redacting Messages and Responses
+
+This section covers how to redact sensitive data from messages and responses in the logged payload on Datadog LLM Observability.
+
+
+When redaction is enabled, the actual message content and response text will be excluded from Datadog logs while preserving metadata like token counts, latency, and model information.
+
+**Step 1**: Configure redaction in your `config.yaml`
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: gpt-3.5-turbo
+litellm_settings:
+ callbacks: ["datadog_llm_observability"] # logs llm success logs on datadog
+
+ # Params to apply only for "datadog_llm_observability" callback
+ datadog_llm_observability_params:
+ turn_off_message_logging: true # redacts input messages and output responses
+```
+
+**Step 2**: Send a chat completion request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+**Step 3**: Verify redaction in Datadog LLM Observability
+
+On the Datadog LLM Observability page, you should see that both input messages and output responses are redacted, while metadata (token counts, timing, model info) remains visible.
+
+
+
+
+
+### Datadog Tracing
+
+Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
+
+**DD Tracer**
+Pass `USE_DDTRACE=true` to the docker run command. When `USE_DDTRACE=true`, the proxy will run `ddtrace-run litellm` as the `ENTRYPOINT` instead of just `litellm`
+
+**DD Profiler**
+
+Pass `USE_DDPROFILER=true` to the docker run command. When `USE_DDPROFILER=true`, the proxy will activate the [Datadog Profiler](https://docs.datadoghq.com/profiler/enabling/python/). This is useful for debugging CPU% and memory usage.
+
+We don't recommend using `USE_DDPROFILER` in production. It is only recommended for debugging CPU% and memory usage.
+
+
+```bash
+docker run \
+ -v $(pwd)/litellm_config.yaml:/app/config.yaml \
+ -e USE_DDTRACE=true \
+ -e USE_DDPROFILER=true \
+ -p 4000:4000 \
+ ghcr.io/berriai/litellm:main-latest \
+ --config /app/config.yaml --detailed_debug
+```
+
+## Set DD variables (`DD_SERVICE` etc)
+
+LiteLLM supports customizing the following Datadog environment variables
+
+| Environment Variable | Description | Default Value | Required |
+|---------------------|-------------|---------------|----------|
+| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes |
+| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes |
+| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No |
+| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No |
+| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No |
+| `DD_VERSION` | Version tag for your logs | "unknown" | ❌ No |
+| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No |
+| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
+
diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md
index 9b807b8d0f6..22ea051f7cd 100644
--- a/docs/my-website/docs/observability/helicone_integration.md
+++ b/docs/my-website/docs/observability/helicone_integration.md
@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
# Helicone - OSS LLM Observability Platform
:::tip
@@ -9,9 +12,68 @@ https://github.com/BerriAI/litellm
[Helicone](https://helicone.ai/) is an open source observability platform that proxies your LLM requests and provides key insights into your usage, spend, latency and more.
-## Using Helicone with LiteLLM
+## Quick Start
-LiteLLM provides `success_callbacks` and `failure_callbacks`, allowing you to easily log data to Helicone based on the status of your responses.
+
+
+
+Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
+
+```python
+import os
+from litellm import completion
+
+## Set env variables
+os.environ["HELICONE_API_KEY"] = "your-helicone-key"
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Set callbacks
+litellm.success_callback = ["helicone"]
+
+# OpenAI call
+response = completion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hi 👋 - I'm OpenAI"}],
+)
+
+print(response)
+```
+
+
+
+
+Add Helicone to your LiteLLM proxy configuration:
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+
+# Add Helicone callback
+litellm_settings:
+ success_callback: ["helicone"]
+
+# Set Helicone API key
+environment_variables:
+ HELICONE_API_KEY: "your-helicone-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+
+
+
+## Integration Methods
+
+There are two main approaches to integrate Helicone with LiteLLM:
+
+1. **Callbacks**: Log to Helicone while using any provider
+2. **Proxy Mode**: Use Helicone as a proxy for advanced features
### Supported LLM Providers
@@ -26,27 +88,16 @@ Helicone can log requests across [various LLM providers](https://docs.helicone.a
- Replicate
- And more
-### Integration Methods
+## Method 1: Using Callbacks
-There are two main approaches to integrate Helicone with LiteLLM:
+Log requests to Helicone while using any LLM provider directly.
-1. Using callbacks
-2. Using Helicone as a proxy
-
-Let's explore each method in detail.
-
-### Approach 1: Use Callbacks
-
-Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
-
-```python
-litellm.success_callback = ["helicone"]
-```
-
-Complete Code
+
+
```python
import os
+import litellm
from litellm import completion
## Set env variables
@@ -66,28 +117,78 @@ response = completion(
print(response)
```
-### Approach 2: Use Helicone as a proxy
+
+
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ - model_name: claude-3
+ litellm_params:
+ model: anthropic/claude-3-sonnet-20240229
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+# Add Helicone logging
+litellm_settings:
+ success_callback: ["helicone"]
+
+# Environment variables
+environment_variables:
+ HELICONE_API_KEY: "your-helicone-key"
+ OPENAI_API_KEY: "your-openai-key"
+ ANTHROPIC_API_KEY: "your-anthropic-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+Make requests to your proxy:
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="anything", # proxy doesn't require real API key
+ base_url="http://localhost:4000"
+)
+
+response = client.chat.completions.create(
+ model="gpt-4", # This gets logged to Helicone
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+
+
+
+## Method 2: Using Helicone as a Proxy
Helicone's proxy provides [advanced functionality](https://docs.helicone.ai/getting-started/proxy-vs-async) like caching, rate limiting, LLM security through [PromptArmor](https://promptarmor.com/) and more.
-To use Helicone as a proxy for your LLM requests:
+
+
-1. Set Helicone as your base URL via: litellm.api_base
-2. Pass in Helicone request headers via: litellm.metadata
-
-Complete Code:
+Set Helicone as your base URL and pass authentication headers:
```python
import os
import litellm
from litellm import completion
+# Configure LiteLLM to use Helicone proxy
litellm.api_base = "https://oai.hconeai.com/v1"
litellm.headers = {
- "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
+ "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
}
-response = litellm.completion(
+# Set your OpenAI API key
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+response = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "How does a court case get to the Supreme Court?"}]
)
@@ -136,36 +237,119 @@ litellm.metadata = {
}
```
-### Session Tracking and Tracing
+
+
+
+## Session Tracking and Tracing
Track multi-step and agentic LLM interactions using session IDs and paths:
-```python
-litellm.metadata = {
- "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
- "Helicone-Session-Id": "session-abc-123", # The session ID you want to track
- "Helicone-Session-Path": "parent-trace/child-trace", # The path of the session
-}
-```
-
-- `Helicone-Session-Id`: Use this to specify the unique identifier for the session you want to track. This allows you to group related requests together.
-- `Helicone-Session-Path`: This header defines the path of the session, allowing you to represent parent and child traces. For example, "parent/child" represents a child trace of a parent trace.
-
-By using these two headers, you can effectively group and visualize multi-step LLM interactions, gaining insights into complex AI workflows.
-
-### Retry and Fallback Mechanisms
-
-Set up retry mechanisms and fallback options:
+
+
```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
litellm.metadata = {
- "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
- "Helicone-Retry-Enabled": "true", # Enable retry mechanism
- "helicone-retry-num": "3", # Set number of retries
- "helicone-retry-factor": "2", # Set exponential backoff factor
- "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]', # Set fallback models
+ "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+ "Helicone-Session-Id": "session-abc-123",
+ "Helicone-Session-Path": "parent-trace/child-trace",
}
+
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Start a conversation"}]
+)
```
+
+
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://localhost:4000"
+)
+
+# First request in session
+response1 = client.chat.completions.create(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello"}],
+ extra_headers={
+ "Helicone-Session-Id": "session-abc-123",
+ "Helicone-Session-Path": "conversation/greeting"
+ }
+)
+
+# Follow-up request in same session
+response2 = client.chat.completions.create(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Tell me more"}],
+ extra_headers={
+ "Helicone-Session-Id": "session-abc-123",
+ "Helicone-Session-Path": "conversation/follow-up"
+ }
+)
+```
+
+
+
+
+- `Helicone-Session-Id`: Unique identifier for the session to group related requests
+- `Helicone-Session-Path`: Hierarchical path to represent parent/child traces (e.g., "parent/child")
+
+## Retry and Fallback Mechanisms
+
+
+
+
+```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
+litellm.metadata = {
+ "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+ "Helicone-Retry-Enabled": "true",
+ "helicone-retry-num": "3",
+ "helicone-retry-factor": "2", # Exponential backoff
+ "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]',
+}
+
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello"}]
+)
+```
+
+
+
+
+```yaml title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ api_base: "https://oai.hconeai.com/v1"
+
+default_litellm_params:
+ headers:
+ Helicone-Auth: "Bearer ${HELICONE_API_KEY}"
+ Helicone-Retry-Enabled: "true"
+ helicone-retry-num: "3"
+ helicone-retry-factor: "2"
+ Helicone-Fallbacks: '["gpt-3.5-turbo", "gpt-4"]'
+
+environment_variables:
+ HELICONE_API_KEY: "your-helicone-key"
+ OPENAI_API_KEY: "your-openai-key"
+```
+
+
+
+
> **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/getting-started/quick-start).
> By utilizing these headers and metadata options, you can gain deeper insights into your LLM usage, optimize performance, and better manage your AI workflows with Helicone and LiteLLM.
diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md
index 576135ba67c..a81336c5bc6 100644
--- a/docs/my-website/docs/observability/langfuse_integration.md
+++ b/docs/my-website/docs/observability/langfuse_integration.md
@@ -11,6 +11,13 @@ Example trace in Langfuse using multiple models via LiteLLM:
+:::info
+
+For Langfuse v3, we recommend using the [Langfuse OTEL](./langfuse_otel_integration) integration.
+
+:::
+
+
## Usage with LiteLLM Proxy (LLM Gateway)
👉 [**Follow this link to start sending logs to langfuse with LiteLLM Proxy server**](../proxy/logging)
@@ -21,7 +28,7 @@ Example trace in Langfuse using multiple models via LiteLLM:
### Pre-Requisites
Ensure you have run `pip install langfuse` for this integration
```shell
-pip install langfuse>=2.0.0 litellm
+pip install langfuse==2.59.7 litellm
```
### Quick Start
@@ -205,6 +212,7 @@ The following parameters can be updated on a continuation of a trace by passing
* `parent_observation_id` - Identifier for the parent observation, defaults to `None`
* `prompt` - Langfuse prompt object used for the generation, defaults to `None`
+
Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation.
#### Disable Logging - Specific Calls
diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md
new file mode 100644
index 00000000000..b4c9a2bd1ad
--- /dev/null
+++ b/docs/my-website/docs/observability/langfuse_otel_integration.md
@@ -0,0 +1,250 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+import Image from '@theme/IdealImage';
+
+# 🪢 Langfuse OpenTelemetry Integration
+
+The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and observability data to Langfuse using the OpenTelemetry protocol. This provides a standardized way to collect and analyze your LLM usage data.
+
+
+
+## Features
+
+- Automatic trace collection for all LiteLLM requests
+- Support for Langfuse Cloud (EU and US regions)
+- Support for self-hosted Langfuse instances
+- Custom endpoint configuration
+- Secure authentication using Basic Auth
+- Consistent attribute mapping with other OTEL integrations
+
+## Prerequisites
+
+1. **Langfuse Account**: Sign up at [Langfuse Cloud](https://cloud.langfuse.com) or set up a self-hosted instance
+2. **API Keys**: Get your public and secret keys from your Langfuse project settings
+3. **Dependencies**: Install required packages:
+ ```bash
+ pip install litellm opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp
+ ```
+
+## Configuration
+
+### Environment Variables
+
+| Variable | Required | Description | Example |
+|----------|----------|-------------|---------|
+| `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
+| `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
+| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
+
+### Endpoint Resolution
+
+The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
+- **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
+- **EU Region**: `https://cloud.langfuse.com/api/public/otel`
+- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
+
+## Usage
+
+### Basic Setup
+
+```python
+import os
+import litellm
+
+# Set your Langfuse credentials
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
+
+# Enable Langfuse OTEL integration
+litellm.callbacks = ["langfuse_otel"]
+
+# Make LLM requests as usual
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Advanced Configuration
+
+```python
+import os
+import litellm
+
+# Set your Langfuse credentials
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
+
+# Use EU region
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
+
+# Or use self-hosted instance
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
+
+litellm.callbacks = ["langfuse_otel"]
+```
+
+### Manual OTEL Configuration
+
+If you need direct control over the OpenTelemetry configuration:
+
+```python
+import os
+import base64
+import litellm
+
+# Get keys for your project from the project settings page: https://cloud.langfuse.com
+os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
+os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
+
+LANGFUSE_AUTH = base64.b64encode(
+ f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
+).decode()
+
+host = os.environ.get("LANGFUSE_OTEL_HOST")
+os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
+os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
+
+litellm.callbacks = ["langfuse_otel"]
+```
+
+### With LiteLLM Proxy
+
+Add the integration to your proxy configuration:
+
+1. Add the credentials to your environment variables
+
+```bash
+export LANGFUSE_PUBLIC_KEY="pk-lf-..."
+export LANGFUSE_SECRET_KEY="sk-lf-..."
+export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region
+# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint
+```
+
+2. Setup config.yaml
+
+```yaml
+# config.yaml
+litellm_settings:
+ callbacks: ["langfuse_otel"]
+```
+
+3. Run the proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Data Collected
+
+The integration automatically collects the following data:
+
+- **Request Details**: Model, messages, parameters (temperature, max_tokens, etc.)
+- **Response Details**: Generated content, token usage, finish reason
+- **Timing Information**: Request duration, time to first token
+- **Metadata**: User ID, session ID, custom tags (if provided)
+- **Error Information**: Exception details and stack traces (if errors occur)
+
+## Metadata Support
+
+All metadata fields available in the vanilla Langfuse integration are now **fully supported** when you use the OTEL integration.
+
+- Any key you pass in the `metadata` dictionary (`generation_name`, `trace_id`, `session_id`, `tags`, and the rest) is exported as an OpenTelemetry span attribute.
+- Attribute names are prefixed with `langfuse.` so you can filter or search for them easily in your observability backend.
+ Examples: `langfuse.generation.name`, `langfuse.trace.id`, `langfuse.trace.session_id`.
+
+### Passing Metadata – Example
+
+```python
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello!"}],
+ metadata={
+ "generation_name": "welcome-message",
+ "trace_id": "trace-123",
+ "session_id": "sess-42",
+ "tags": ["prod", "beta-user"]
+ }
+)
+```
+
+The resulting span will contain attributes similar to:
+
+```
+langfuse.generation.name = "welcome-message"
+langfuse.trace.id = "trace-123"
+langfuse.trace.session_id = "sess-42"
+langfuse.trace.tags = ["prod", "beta-user"]
+```
+
+Use the **Langfuse UI** (Traces tab) to search, filter and analyse spans that contain the `langfuse.*` attributes.
+The OTEL exporter in this integration sends data directly to Langfuse’s OTLP HTTP endpoint; it is **not** intended for Grafana, Honeycomb, Datadog, or other generic OTEL back-ends.
+
+## Authentication
+
+The integration uses HTTP Basic Authentication with your Langfuse public and secret keys:
+
+```
+Authorization: Basic
+```
+
+This is automatically handled by the integration - you just need to provide the keys via environment variables.
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing Credentials Error**
+ ```
+ ValueError: LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set
+ ```
+ **Solution**: Ensure both environment variables are set with valid keys.
+
+2. **Connection Issues**
+ - Check your internet connection
+ - Verify the endpoint URL is correct
+ - For self-hosted instances, ensure the `/api/public/otel` endpoint is accessible
+
+3. **Authentication Errors**
+ - Verify your public and secret keys are correct
+ - Check that the keys belong to the same Langfuse project
+ - Ensure the keys have the necessary permissions
+
+### Debug Mode
+
+Enable verbose logging to see detailed information:
+
+
+
+
+```python
+import litellm
+litellm._turn_on_debug()
+```
+
+
+
+
+```bash
+export LITELLM_LOG="DEBUG"
+```
+
+
+
+
+This will show:
+- Endpoint resolution logic
+- Authentication header creation
+- OTEL trace submission details
+
+## Related Links
+
+- [Langfuse Documentation](https://langfuse.com/docs)
+- [Langfuse OpenTelemetry Guide](https://langfuse.com/docs/integrations/opentelemetry)
+- [OpenTelemetry Python SDK](https://opentelemetry.io/docs/languages/python/)
+- [LiteLLM Observability](https://docs.litellm.ai/docs/observability/)
\ No newline at end of file
diff --git a/docs/my-website/docs/observability/mlflow.md b/docs/my-website/docs/observability/mlflow.md
index 39746b2cad7..5fa46bdfdac 100644
--- a/docs/my-website/docs/observability/mlflow.md
+++ b/docs/my-website/docs/observability/mlflow.md
@@ -17,7 +17,7 @@ MLflow’s integration with LiteLLM supports advanced observability compatible w
Install MLflow:
```shell
-pip install mlflow
+pip install "litellm[mlflow]"
```
To enable MLflow auto tracing for LiteLLM:
@@ -160,6 +160,102 @@ class CustomAgent:
This approach generates a unified trace, combining your custom Python code with LiteLLM calls.
+## LiteLLM Proxy Server
+
+### Dependencies
+
+For using `mlflow` on LiteLLM Proxy Server, you need to install the `mlflow` package on your docker container.
+
+```shell
+pip install "mlflow>=3.1.4"
+```
+
+### Configuration
+
+Configure MLflow in your LiteLLM proxy configuration file:
+
+```yaml
+model_list:
+ - model_name: openai/*
+ litellm_params:
+ model: openai/*
+
+litellm_settings:
+ success_callback: ["mlflow"]
+ failure_callback: ["mlflow"]
+```
+
+### Environment Variables
+
+For MLflow with Databricks service, set these required environment variables:
+
+```shell
+DATABRICKS_TOKEN="dapixxxxx"
+DATABRICKS_HOST="https://dbc-xxxx.cloud.databricks.com"
+MLFLOW_TRACKING_URI="databricks"
+MLFLOW_REGISTRY_URI="databricks-uc"
+MLFLOW_EXPERIMENT_ID="xxxx"
+```
+
+### Adding Tags for Better Tracing
+
+You can add custom tags to your requests for improved trace organization and filtering in MLflow. Tags help you categorize and search your traces by job ID, task name, or any custom metadata.
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --header 'Authorization: Bearer sk-1234' \
+ --data '{
+ "model": "gemini-2.5-flash",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ "litellm_metadata": {
+ "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
+ }
+}'
+```
+
+
+
+
+```python
+from openai import OpenAI
+
+# Initialize the OpenAI client pointing to your LiteLLM proxy
+client = OpenAI(
+ api_key="sk-1234", # Your LiteLLM proxy API key
+ base_url="http://0.0.0.0:4000" # Your LiteLLM proxy URL
+)
+
+# Make a request with tags in metadata
+response = client.chat.completions.create(
+ model="gemini-2.5-flash",
+ messages=[
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ extra_body={
+ "litellm_metadata": {
+ "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
+ }
+ }
+)
+```
+
+
+
## Support
diff --git a/docs/my-website/docs/observability/opentelemetry_integration.md b/docs/my-website/docs/observability/opentelemetry_integration.md
index 958c33f18e6..23532ab6e80 100644
--- a/docs/my-website/docs/observability/opentelemetry_integration.md
+++ b/docs/my-website/docs/observability/opentelemetry_integration.md
@@ -104,4 +104,14 @@ for successful + failed requests
click under `litellm_request` in the trace
-
\ No newline at end of file
+
+
+### Not seeing traces land on Integration
+
+If you don't see traces landing on your integration, set `OTEL_DEBUG="True"` in your LiteLLM environment and try again.
+
+```shell
+export OTEL_DEBUG="True"
+```
+
+This will emit any logging issues to the console.
\ No newline at end of file
diff --git a/docs/my-website/docs/observability/opik_integration.md b/docs/my-website/docs/observability/opik_integration.md
index b4bcef53937..1ba1c2de210 100644
--- a/docs/my-website/docs/observability/opik_integration.md
+++ b/docs/my-website/docs/observability/opik_integration.md
@@ -140,6 +140,7 @@ These can be passed inside metadata with the `opik` key.
- `project_name` - Name of the Opik project to send data to.
- `current_span_data` - The current span data to be used for tracing.
- `tags` - Tags to be used for tracing.
+- `thread_id` - The thread id to group together multiple related traces.
### Usage
@@ -159,8 +160,10 @@ response = litellm.completion(
messages=messages,
metadata = {
"opik": {
+ "project_name": "your-opik-project-name",
"current_span_data": get_current_span_data(),
"tags": ["streaming-test"],
+ "thread_id": "your-thread-id"
},
}
)
@@ -174,7 +177,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
- "model": "gpt-3.5-turbo-testing",
+ "model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
@@ -183,8 +186,10 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
],
"metadata": {
"opik": {
+ "project_name": "your-opik-project-name",
"current_span_data": "...",
"tags": ["streaming-test"],
+ "thread_id": "your-thread-id"
},
}
}'
@@ -195,12 +200,25 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+You can also pass the fields as part of the request header with a `opik_*` prefix:
-
-
-
-
-
+```shell
+curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'opik_project_name: your-opik-project-name' \
+ --header 'opik_thread_id: your-thread-id' \
+ --header 'opik_tags: ["streaming-test"]' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What's the weather like in Boston today?"
+ }
+ ]
+}'
+```
diff --git a/docs/my-website/docs/observability/posthog_integration.md b/docs/my-website/docs/observability/posthog_integration.md
new file mode 100644
index 00000000000..7e6a0e1076b
--- /dev/null
+++ b/docs/my-website/docs/observability/posthog_integration.md
@@ -0,0 +1,216 @@
+# PostHog - Tracking LLM Usage Analytics
+
+## What is PostHog?
+
+PostHog is an open-source product analytics platform that helps you track and analyze how users interact with your product. For LLM applications, PostHog provides specialized AI features to track model usage, performance, and user interactions with your AI features.
+
+## Usage with LiteLLM Proxy (LLM Gateway)
+
+**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: gpt-3.5-turbo
+
+litellm_settings:
+ success_callback: ["posthog"]
+ failure_callback: ["posthog"]
+```
+
+**Step 2**: Set required environment variables
+
+```shell
+export POSTHOG_API_KEY="your-posthog-api-key"
+# Optional, defaults to https://app.posthog.com
+export POSTHOG_API_URL="https://app.posthog.com" # optional
+```
+
+**Step 3**: Start the proxy, make a test request
+
+Start proxy
+
+```shell
+litellm --config config.yaml --debug
+```
+
+Test Request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ "metadata": {
+ "user_id": "user-123",
+ "custom_field": "custom_value"
+ }
+}'
+```
+
+## Usage with LiteLLM Python SDK
+
+### Quick Start
+
+Use just 2 lines of code, to instantly log your responses **across all providers** with PostHog:
+
+```python
+litellm.success_callback = ["posthog"]
+litellm.failure_callback = ["posthog"] # logs errors to posthog
+```
+```python
+import litellm
+import os
+
+# from PostHog
+os.environ["POSTHOG_API_KEY"] = ""
+# Optional, defaults to https://app.posthog.com
+os.environ["POSTHOG_API_URL"] = "" # optional
+
+# LLM API Keys
+os.environ['OPENAI_API_KEY']=""
+
+# set posthog as a callback, litellm will send the data to posthog
+litellm.success_callback = ["posthog"]
+
+# openai call
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "Hi - i'm openai"}
+ ],
+ metadata = {
+ "user_id": "user-123", # set posthog user ID
+ }
+)
+```
+
+### Advanced
+
+#### Set User ID and Custom Metadata
+
+Pass `user_id` in `metadata` to associate events with specific users in PostHog:
+
+**With LiteLLM Python SDK:**
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "Hello world"}
+ ],
+ metadata={
+ "user_id": "user-123", # Add user ID for PostHog tracking
+ "custom_field": "custom_value" # Add custom metadata
+ }
+)
+```
+
+**With LiteLLM Proxy using OpenAI Python SDK:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234", # Your LiteLLM Proxy API key
+ base_url="http://0.0.0.0:4000" # Your LiteLLM Proxy URL
+)
+
+response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "Hello world"}
+ ],
+ extra_body={
+ "metadata": {
+ "user_id": "user-123", # Add user ID for PostHog tracking
+ "project_name": "my-project", # Add custom metadata
+ "environment": "production"
+ }
+ }
+)
+```
+
+#### Disable Logging for Specific Calls
+
+Use the `no-log` flag to prevent logging for specific calls:
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[
+ {"role": "user", "content": "This won't be logged"}
+ ],
+ metadata={"no-log": True}
+)
+```
+
+## What's Logged to PostHog?
+
+When LiteLLM logs to PostHog, it captures detailed information about your LLM usage:
+
+### For Completion Calls
+- **Model Information**: Provider, model name, model parameters
+- **Usage Metrics**: Input tokens, output tokens, total cost
+- **Performance**: Latency, completion time
+- **Content**: Input messages, model responses (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Embedding Calls
+- **Model Information**: Provider, model name
+- **Usage Metrics**: Input tokens, total cost
+- **Performance**: Latency
+- **Content**: Input text (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Errors
+- **Error Details**: Error type, error message, stack trace
+- **Context**: Model, provider, input that caused the error
+- **Timing**: When the error occurred, request duration
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `POSTHOG_API_KEY` | Yes | Your PostHog project API key |
+| `POSTHOG_API_URL` | No | PostHog API URL (defaults to https://app.posthog.com) |
+
+## Troubleshooting
+
+### 1. Missing API Key
+```
+Error: POSTHOG_API_KEY is not set
+```
+
+Set your PostHog API key:
+```python
+import os
+os.environ["POSTHOG_API_KEY"] = "your-api-key"
+```
+
+### 2. Custom PostHog Instance
+If you're using a self-hosted PostHog instance:
+```python
+import os
+os.environ["POSTHOG_API_URL"] = "https://your-posthog-instance.com"
+```
+
+### 3. Events Not Appearing
+- Check that your API key is correct
+- Verify network connectivity to PostHog
+- Events may take a few minutes to appear in PostHog dashboard
\ No newline at end of file
diff --git a/docs/my-website/docs/old_guardrails.md b/docs/my-website/docs/old_guardrails.md
index 451ca8ab508..73448666c43 100644
--- a/docs/my-website/docs/old_guardrails.md
+++ b/docs/my-website/docs/old_guardrails.md
@@ -212,7 +212,7 @@ If you need to switch `pii_masking` off for an API Key set `"permissions": {"pii
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
- -D '{
+ -d '{
"permissions": {"pii_masking": true}
}'
```
diff --git a/docs/my-website/docs/pass_through/azure_passthrough.md b/docs/my-website/docs/pass_through/azure_passthrough.md
new file mode 100644
index 00000000000..cac06333589
--- /dev/null
+++ b/docs/my-website/docs/pass_through/azure_passthrough.md
@@ -0,0 +1,89 @@
+# Azure Passthrough
+
+Pass-through endpoints for `/azure`
+
+## Overview
+
+| Feature | Supported | Notes |
+|-------|-------|-------|
+| Cost Tracking | ❌ | Not supported |
+| Logging | ✅ | Works across all integrations |
+| Streaming | ✅ | Fully supported |
+
+### When to use this?
+
+- For most use cases, you should use the [native LiteLLM Azure OpenAI Integration](../providers/azure/azure) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, etc.)
+- Use this passthrough to call newer or less common Azure OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
+
+Simply replace your Azure endpoint (e.g. `https://.openai.azure.com`) with `LITELLM_PROXY_BASE_URL/azure`
+
+## Usage Examples
+
+### Assistants API
+
+#### Create Azure OpenAI Client
+
+Make sure you do the following:
+- Point `azure_endpoint` to your `LITELLM_PROXY_BASE_URL/azure`
+- Use your `LITELLM_API_KEY` as the `api_key`
+
+```python
+import openai
+
+client = openai.AzureOpenAI(
+ azure_endpoint="http://0.0.0.0:4000/azure", # /azure
+ api_key="sk-anything", #
+ api_version="2024-05-01-preview" # required Azure API version
+)
+```
+
+#### Create an Assistant
+
+```python
+assistant = client.beta.assistants.create(
+ name="Math Tutor",
+ instructions="You are a math tutor. Help solve equations.",
+ model="gpt-4o",
+)
+```
+
+#### Create a Thread
+```python
+thread = client.beta.threads.create()
+```
+
+#### Add a Message to the Thread
+```python
+message = client.beta.threads.messages.create(
+ thread_id=thread.id,
+ role="user",
+ content="Solve 3x + 11 = 14",
+)
+```
+
+#### Run the Assistant
+```python
+run = client.beta.threads.runs.create(
+ thread_id=thread.id,
+ assistant_id=assistant.id,
+)
+
+# Check run status
+run_status = client.beta.threads.runs.retrieve(
+ thread_id=thread.id,
+ run_id=run.id
+)
+```
+
+#### Retrieve Messages
+```python
+messages = client.beta.threads.messages.list(
+ thread_id=thread.id
+)
+```
+
+#### Delete the Assistant
+
+```python
+client.beta.assistants.delete(assistant.id)
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md
index 5c90f3c5d1c..48502864d78 100644
--- a/docs/my-website/docs/pass_through/bedrock.md
+++ b/docs/my-website/docs/pass_through/bedrock.md
@@ -4,7 +4,7 @@ Pass-through endpoints for Bedrock - call provider-specific endpoint, in native
| Feature | Supported | Notes |
|-------|-------|-------|
-| Cost Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
+| Cost Tracking | ✅ | For `/invoke` and `/converse` endpoints |
| Logging | ✅ | works across all integrations |
| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
| Streaming | ✅ | |
@@ -33,7 +33,7 @@ Supports **ALL** Bedrock Endpoints (including streaming).
Let's call the Bedrock [`/converse` endpoint](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html)
-1. Add AWS Keyss to your environment
+1. Add AWS Keys to your environment
```bash
export AWS_ACCESS_KEY_ID="" # Access key
@@ -295,4 +295,4 @@ for event in response.get("completion"):
print(completion)
-```
\ No newline at end of file
+```
diff --git a/docs/my-website/docs/pass_through/google_ai_studio.md b/docs/my-website/docs/pass_through/google_ai_studio.md
index c3671f58d36..3de7c54aa7a 100644
--- a/docs/my-website/docs/pass_through/google_ai_studio.md
+++ b/docs/my-website/docs/pass_through/google_ai_studio.md
@@ -230,6 +230,13 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
```
+## **Example 4: Video Generation with Veo**
+
+Generate videos using Google's Veo model through LiteLLM pass-through routes.
+
+[**→ Complete Veo Video Generation Guide**](../proxy/veo_video_generation.md)
+
+
## Advanced
Pre-requisites
diff --git a/docs/my-website/docs/pass_through/intro.md b/docs/my-website/docs/pass_through/intro.md
index 3d6286afcc5..38218224f11 100644
--- a/docs/my-website/docs/pass_through/intro.md
+++ b/docs/my-website/docs/pass_through/intro.md
@@ -11,3 +11,43 @@ These endpoints are useful for 2 scenarios:
## How is your request handled?
The request is passed through to the provider's endpoint. The response is then passed back to the client. **No translation is done.**
+
+### Request Forwarding Process
+
+1. **Request Reception**: LiteLLM receives your request at `/provider/endpoint`
+2. **Authentication**: Your LiteLLM API key is validated and mapped to the provider's API key
+3. **Request Transformation**: Request is reformatted for the target provider's API
+4. **Forwarding**: Request is sent to the actual provider endpoint
+5. **Response Handling**: Provider response is returned directly to you
+
+### Authentication Flow
+
+```mermaid
+graph LR
+ A[Client Request] --> B[LiteLLM Proxy]
+ B --> C[Validate LiteLLM API Key]
+ C --> D[Map to Provider API Key]
+ D --> E[Forward to Provider]
+ E --> F[Return Response]
+```
+
+**Key Points:**
+- Use your **LiteLLM API key** in requests, not the provider's key
+- LiteLLM handles the provider authentication internally
+- Same authentication works across all passthrough endpoints
+
+### Error Handling
+
+**Provider Errors**: Forwarded directly to you with original error codes and messages
+
+**LiteLLM Errors**:
+- `401`: Invalid LiteLLM API key
+- `404`: Provider or endpoint not supported
+- `500`: Internal routing/forwarding errors
+
+### Benefits
+
+- **Unified Authentication**: One API key for all providers
+- **Centralized Logging**: All requests logged through LiteLLM
+- **Cost Tracking**: Usage tracked across all endpoints
+- **Access Control**: Same permissions apply to passthrough endpoints
diff --git a/docs/my-website/docs/pass_through/vertex_ai.md b/docs/my-website/docs/pass_through/vertex_ai.md
index d3f4e75e31d..77095667113 100644
--- a/docs/my-website/docs/pass_through/vertex_ai.md
+++ b/docs/my-website/docs/pass_through/vertex_ai.md
@@ -15,10 +15,11 @@ Pass-through endpoints for Vertex AI - call provider-specific endpoint, in nativ
## Supported Endpoints
-LiteLLM supports 2 vertex ai passthrough routes:
+LiteLLM supports 3 vertex ai passthrough routes:
1. `/vertex_ai` → routes to `https://{vertex_location}-aiplatform.googleapis.com/`
2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/)
+3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`)
## How to use
@@ -170,6 +171,50 @@ generateContent();
+## Vertex AI Live API WebSocket
+
+LiteLLM can now proxy the Vertex AI Live API to help you experiment with streaming audio/text from Gemini Live models without exposing Google credentials to clients.
+
+- Configure default Vertex credentials via `default_vertex_config` or environment variables (see examples above).
+- Connect to `wss:///vertex_ai/live`. LiteLLM will exchange your saved credentials for a short-lived access token and forward messages bidirectionally.
+- Optional query params `vertex_project`, `vertex_location`, and `model` let you override defaults for multi-project setups or global-only models.
+
+```python title="client.py"
+import asyncio
+import json
+
+from websockets.asyncio.client import connect
+
+
+async def main() -> None:
+ headers = {
+ "x-litellm-api-key": "Bearer sk-your-litellm-key",
+ "Content-Type": "application/json",
+ }
+ async with connect(
+ "ws://localhost:4000/vertex_ai/live",
+ additional_headers=headers,
+ ) as ws:
+ await ws.send(
+ json.dumps(
+ {
+ "setup": {
+ "model": "projects/your-project/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+ "generation_config": {"response_modalities": ["TEXT"]},
+ }
+ }
+ )
+ )
+
+ async for message in ws:
+ print("server:", message)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+
## Quick Start
Let's call the Vertex AI [`/generateContent` endpoint](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)
@@ -415,4 +460,4 @@ generateContent();
```
-
\ No newline at end of file
+
diff --git a/docs/my-website/docs/pass_through/vertex_ai_live_websocket.md b/docs/my-website/docs/pass_through/vertex_ai_live_websocket.md
new file mode 100644
index 00000000000..cca40d10fd8
--- /dev/null
+++ b/docs/my-website/docs/pass_through/vertex_ai_live_websocket.md
@@ -0,0 +1,284 @@
+# Vertex AI Live API WebSocket Passthrough
+
+LiteLLM now supports WebSocket passthrough for the Vertex AI Live API, enabling real-time bidirectional communication with Gemini models.
+
+## Overview
+
+The Vertex AI Live API WebSocket passthrough allows you to:
+- Connect to Vertex AI Live API through LiteLLM proxy
+- Use existing Vertex AI authentication methods
+- Pass through all WebSocket messages bidirectionally
+- Support text, audio, video, and multimodal interactions
+- Track costs automatically for all usage types
+
+## Configuration
+
+### Environment Variables
+
+Set the following environment variables for Vertex AI authentication:
+
+```bash
+# Required
+DEFAULT_VERTEXAI_PROJECT=your-project-id
+DEFAULT_VERTEXAI_LOCATION=us-central1
+
+# Optional - use one of these for authentication
+DEFAULT_GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
+# OR run: gcloud auth application-default login
+```
+
+### Configuration File
+
+Alternatively, configure in your `config.yaml`:
+
+```yaml
+litellm_settings:
+ default_vertex_config:
+ vertex_project: "your-project-id"
+ vertex_location: "us-central1"
+ vertex_credentials: "os.environ/GOOGLE_APPLICATION_CREDENTIALS"
+```
+
+## Usage
+
+### WebSocket Endpoints
+
+- `ws://your-proxy-host/v1/vertex-ai/live`
+- `ws://your-proxy-host/vertex-ai/live`
+
+### Query Parameters
+
+- `project_id` (optional): Google Cloud project ID (can be set in config)
+- `location` (optional): Vertex AI location (can be set in config, default: us-central1)
+
+### Example Connection
+
+```javascript
+// If project_id and location are set in config, you can connect without query params
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live');
+
+// Or specify them explicitly
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id&location=us-central1');
+```
+
+## Cost Tracking
+
+The WebSocket passthrough automatically tracks costs for all usage types based on the [Vertex AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing#model-optimizer-pricing):
+
+### Supported Cost Tracking
+
+- **Text**: Character-based or token-based pricing depending on model
+- **Audio**: Per-second pricing for audio input/output
+- **Video**: Per-second pricing for video input
+- **Images**: Per-image pricing for image input
+
+### Cost Calculation
+
+Costs are calculated using the same methods as other Vertex AI models in LiteLLM:
+- Uses `cost_per_character` for Gemini models
+- Uses `cost_per_token` for partner models (Claude, Llama, etc.)
+- Includes audio, video, and image costs when applicable
+
+### Cost Logging
+
+Costs are automatically logged to:
+- LiteLLM proxy logs
+- Database (if configured)
+- Spend tracking system
+- Admin dashboard
+
+Example log output:
+```
+Vertex AI Live WebSocket session cost: $0.001234 (input: $0.000800, output: $0.000434) tokens: 150, characters: 1200, duration: 45.2s
+```
+
+## API Reference
+
+### Setup Message
+
+Send this message first to initialize the session:
+
+```json
+{
+ "setup": {
+ "model": "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+ "generation_config": {
+ "response_modalities": ["TEXT"]
+ }
+ }
+}
+```
+
+### Text Input
+
+```json
+{
+ "client_content": {
+ "turns": [
+ {
+ "role": "user",
+ "parts": [{"text": "Hello! How are you?"}]
+ }
+ ],
+ "turn_complete": true
+ }
+}
+```
+
+### Audio Input
+
+```json
+{
+ "realtime_input": {
+ "media_chunks": [
+ {
+ "data": "base64-encoded-audio-data",
+ "mime_type": "audio/pcm"
+ }
+ ]
+ }
+}
+```
+
+## Supported Features
+
+### Response Modalities
+
+- **TEXT**: Text responses
+- **AUDIO**: Audio responses with voice synthesis
+
+### Tools
+
+- **Function Calling**: Define and use custom functions
+- **Code Execution**: Execute Python code
+- **Google Search**: Search the web
+- **Voice Activity Detection**: Detect when user is speaking
+
+### Advanced Features
+
+- **Audio Transcription**: Transcribe input and output audio
+- **Proactive Audio**: Model responds only when relevant
+- **Affective Dialog**: Understand emotional expressions
+
+## Examples
+
+### Python Client
+
+```python
+import asyncio
+import json
+import websockets
+
+async def chat_with_gemini():
+ uri = "ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id"
+
+ async with websockets.connect(uri) as websocket:
+ # Setup
+ setup = {
+ "setup": {
+ "model": "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+ "generation_config": {"response_modalities": ["TEXT"]}
+ }
+ }
+ await websocket.send(json.dumps(setup))
+
+ # Wait for setup response
+ response = await websocket.recv()
+ print(f"Setup: {response}")
+
+ # Send message
+ message = {
+ "client_content": {
+ "turns": [{"role": "user", "parts": [{"text": "Hello!"}]}],
+ "turn_complete": True
+ }
+ }
+ await websocket.send(json.dumps(message))
+
+ # Receive response
+ async for response in websocket:
+ print(f"Response: {response}")
+ # Check if turn is complete
+ data = json.loads(response)
+ if data.get("serverContent", {}).get("turnComplete"):
+ break
+
+asyncio.run(chat_with_gemini())
+```
+
+### JavaScript Client
+
+```javascript
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id');
+
+ws.onopen = function() {
+ // Send setup
+ const setup = {
+ setup: {
+ model: "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+ generation_config: { response_modalities: ["TEXT"] }
+ }
+ };
+ ws.send(JSON.stringify(setup));
+};
+
+ws.onmessage = function(event) {
+ const data = JSON.parse(event.data);
+ console.log('Received:', data);
+
+ // Check if setup is complete
+ if (data.setupComplete) {
+ // Send a message
+ const message = {
+ client_content: {
+ turns: [{ role: "user", parts: [{ text: "Hello!" }] }],
+ turn_complete: true
+ }
+ };
+ ws.send(JSON.stringify(message));
+ }
+};
+```
+
+## Error Handling
+
+The WebSocket connection may close with these codes:
+
+- `4001`: Vertex AI credentials not configured
+- `4002`: Project ID not provided
+- `1011`: Internal server error
+
+## Authentication
+
+The WebSocket passthrough uses the same authentication as other LiteLLM endpoints:
+
+1. **API Key**: Pass `Authorization: Bearer your-api-key` header
+2. **Vertex AI Credentials**: Set environment variables or config file
+
+## Limitations
+
+- Requires valid Google Cloud project with Vertex AI API enabled
+- WebSocket connections are not persistent across server restarts
+- Rate limits apply based on your Google Cloud quotas
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Authentication Error**: Ensure Vertex AI credentials are properly configured
+2. **Project Not Found**: Verify the project ID exists and has Vertex AI enabled
+3. **Connection Refused**: Check that the LiteLLM proxy server is running
+
+### Debug Mode
+
+Enable debug logging to see detailed connection information:
+
+```bash
+export LITELLM_LOG=DEBUG
+```
+
+## Related Documentation
+
+- [Vertex AI Live API Reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-live)
+- [LiteLLM Proxy Configuration](../proxy/)
+- [Vertex AI Passthrough Endpoints](./vertex_ai.md)
diff --git a/docs/my-website/docs/pass_through/vllm.md b/docs/my-website/docs/pass_through/vllm.md
index b267622948b..eba10536f8e 100644
--- a/docs/my-website/docs/pass_through/vllm.md
+++ b/docs/my-website/docs/pass_through/vllm.md
@@ -23,12 +23,22 @@ Supports **ALL** VLLM Endpoints (including streaming).
## Quick Start
-Let's call the VLLM [`/metrics` endpoint](https://vllm.readthedocs.io/en/latest/api_reference/api_reference.html)
+Let's call the VLLM [`/score` endpoint](https://vllm.readthedocs.io/en/latest/api_reference/api_reference.html)
-1. Add HOSTED VLLM API BASE to your environment
+1. Add a VLLM hosted model to your LiteLLM Proxy
-```bash
-export HOSTED_VLLM_API_BASE="https://my-vllm-server.com"
+:::info
+
+Works with LiteLLM v1.72.0+.
+
+:::
+
+```yaml
+model_list:
+ - model_name: "my-vllm-model"
+ litellm_params:
+ model: hosted_vllm/vllm-1.72
+ api_base: https://my-vllm-server.com
```
2. Start LiteLLM Proxy
@@ -41,12 +51,19 @@ litellm
3. Test it!
-Let's call the VLLM `/metrics` endpoint
+Let's call the VLLM `/score` endpoint
```bash
-curl -L -X GET 'http://0.0.0.0:4000/vllm/metrics' \
--H 'Content-Type: application/json' \
--H 'Authorization: Bearer sk-1234' \
+curl -X 'POST' \
+ 'http://0.0.0.0:4000/vllm/score' \
+ -H 'accept: application/json' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "my-vllm-model",
+ "encoding_format": "float",
+ "text_1": "What is the capital of France?",
+ "text_2": "The capital of France is Paris."
+}'
```
diff --git a/docs/my-website/docs/projects/HolmesGPT.md b/docs/my-website/docs/projects/HolmesGPT.md
new file mode 100644
index 00000000000..608d526368f
--- /dev/null
+++ b/docs/my-website/docs/projects/HolmesGPT.md
@@ -0,0 +1,7 @@
+# HolmesGPT
+
+[HolmesGPT](https://github.com/robusta-dev/holmesgpt) is an AI-powered observability tool designed to enhance incident response and troubleshooting processes. It's like your 24/7 on-call assistant, helps you solve alerts faster with Automatic Correlations, Investigations, and More.
+
+LiteLLM helps HolmesGPT integrate with multiple LLM providers or bring their own model and self-host it.
+
+🔗 Try HolmesGPT → [https://github.com/robusta-dev/holmesgpt](https://github.com/robusta-dev/holmesgpt)
\ No newline at end of file
diff --git a/docs/my-website/docs/projects/Railtracks.md b/docs/my-website/docs/projects/Railtracks.md
new file mode 100644
index 00000000000..3b94ec8df43
--- /dev/null
+++ b/docs/my-website/docs/projects/Railtracks.md
@@ -0,0 +1,7 @@
+# Railtracks
+
+`Railtracks` is an open-source agentic framework that helps developers build resilient agentic systems offering local and remote monitoring tools.
+
+- [Github](https://github.com/RailtownAI/railtracks)
+- [Docs](https://railtownai.github.io/railtracks/)
+- [Railtracks](https://railtracks.org/)
\ No newline at end of file
diff --git a/docs/my-website/docs/provider_registration/index.md b/docs/my-website/docs/provider_registration/index.md
new file mode 100644
index 00000000000..66f61554783
--- /dev/null
+++ b/docs/my-website/docs/provider_registration/index.md
@@ -0,0 +1,316 @@
+---
+title: "Integrate as a Model Provider"
+---
+
+This guide focuses on how to setup the classes and configuration necessary to act as a chat provider.
+
+Please see this guide first and look at the existing code in the codebase to understand how to act as a different provider, e.g. handling embeddings or image-generation.
+
+---
+
+### Overview
+
+The way liteLLM works from a provider's perspective is simple.
+
+liteLLM acts as a wrapper, it takes openai requests and routes them to your api. It then adapts your output into a standard output.
+
+To integrate as a provider, you need to write a module that slots in the api and acts as an adapter between the liteLLM API and your API.
+
+The module you will be writing acts as both a config and a means to adapt requests and responses.
+
+Your objective is to effectively write this module so that it adapts inputs to your api, and adapts outputs to the calling liteLLM code.
+
+It includes methods that:
+
+- Validate the request
+- Transform (adapt) the requests into requests sent to your api
+- Transform (adapt) responses from your api into responses given back to the calling liteLLM code
+- \+ a few others
+
+---
+
+### 1. Create Your Config Class
+
+Create a new directory with your provider name
+
+#### `litellm/llms/your_provider_name_here`
+
+Inside of there, you will want to add a file for your chat configuration
+
+#### `litellm/llms/your_provider_name_here/chat/transformation.py`
+
+The `transformation.py` file will contain a configuration class that dictates how your api will slot into the liteLLM api.
+
+Define your config class extending `BaseConfig`:
+
+```python
+from litellm.llms.base_llm.chat.transformation import BaseConfig
+
+class MyProviderChatConfig(BaseConfig):
+ def __init__(self):
+ ...
+```
+
+We will fill in the abstract methods at a later point.
+
+---
+
+### 2. Add Yourself To Various Places In The Code Base
+
+liteLLM is working to enhance this process, but currently, what you need to do is the following:
+
+#### `litellm/__init__.py`
+
+At the top part of the file, add your key to the list of keys as an option
+
+```py
+azure_key: Optional[str] = None
+anthropic_key: Optional[str] = None
+replicate_key: Optional[str] = None
+bytez_key: Optional[str] = None
+cohere_key: Optional[str] = None
+infinity_key: Optional[str] = None
+clarifai_key: Optional[str] = None
+```
+
+Import your config
+
+```
+from .llms.bytez.chat.transformation import BytezChatConfig
+from .llms.custom_llm import CustomLLM
+from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+from .llms.openai_like.chat.handler import OpenAILikeChatConfig
+```
+
+#### `litellm/main.py`
+
+Add yourself to `main.py` so requests can be routed to your config class
+
+```py
+from .llms.bedrock.chat import BedrockConverseLLM, BedrockLLM
+from .llms.bedrock.embed.embedding import BedrockEmbedding
+from .llms.bedrock.image.image_handler import BedrockImageGeneration
+from .llms.bytez.chat.transformation import BytezChatConfig
+from .llms.codestral.completion.handler import CodestralTextCompletion
+from .llms.cohere.embed import handler as cohere_embed
+from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+base_llm_http_handler = BaseLLMHTTPHandler()
+base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler()
+sagemaker_chat_completion = SagemakerChatHandler()
+bytez_transformation = BytezChatConfig()
+```
+
+Then much lower in the code
+
+```py
+elif custom_llm_provider == "bytez":
+ api_key = (
+ api_key
+ or litellm.bytez_key
+ or get_secret_str("BYTEZ_API_KEY")
+ or litellm.api_key
+ )
+
+ response = base_llm_http_handler.completion(
+ model=model,
+ messages=messages,
+ headers=headers,
+ model_response=model_response,
+ api_key=api_key,
+ api_base=api_base,
+ acompletion=acompletion,
+ logging_obj=logging,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ timeout=timeout, # type: ignore
+ client=client,
+ custom_llm_provider=custom_llm_provider,
+ encoding=encoding,
+ stream=stream,
+ )
+
+ pass
+```
+
+NOTE you can rely on liteLLM passing each of the args/kwargs to your config via the .completion() call
+
+#### `litellm/constants.py`
+
+Add yourself to the list of `LITELLM_CHAT_PROVIDERS`
+
+```py
+LITELLM_CHAT_PROVIDERS = [
+ "openai",
+ "openai_like",
+ "bytez",
+ "xai",
+ "custom_openai",
+ "text-completion-openai",
+```
+
+Add yourself to the if statement chain of providers here
+
+#### `litellm/litellm_core_utils/get_llm_provider_logic.py`
+
+```py
+elif model == "*":
+ custom_llm_provider = "openai"
+# bytez models
+elif model.startswith("bytez/"):
+ custom_llm_provider = "bytez"
+if not custom_llm_provider:
+ if litellm.suppress_debug_info is False:
+ print() # noqa
+```
+
+#### `litellm/litellm_core_utils/streaming_handler.py`
+
+#### If you are doing something custom with streaming, this needs to be updated, e.g.
+
+```py
+ def handle_bytez_chunk(self, chunk):
+ try:
+ is_finished = False
+ finish_reason = ""
+
+ return {
+ "text": chunk,
+ "is_finished": is_finished,
+ "finish_reason": finish_reason,
+ }
+ except Exception as e:
+ raise e
+```
+
+Then lower in the file
+
+```
+elif self.custom_llm_provider and self.custom_llm_provider == "bytez":
+ response_obj = self.handle_bytez_chunk(chunk)
+ completion_obj["content"] = response_obj["text"]
+ if response_obj["is_finished"]:
+ self.received_finish_reason = response_obj["finish_reason"]
+ pass
+```
+
+---
+
+### 3. Write a test file to iterate your code
+
+Add a test file somewhere in the project, `tests/test_litellm/llms/my_provider/chat/test.py`
+
+Write to it the following:
+
+```python
+import os
+from litellm import completion
+
+os.environ["MY_PROVIDER_KEY"] = "KEY_GOES_HERE"
+
+completion(model="my_provider/your-model", messages=[...], api_key="...")
+```
+
+If you want to run it with the vscode debugger you can do so with this config file (recommended)
+
+`.vscode/launch.json`
+
+```json
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Python Debugger: Current File",
+ "type": "debugpy",
+ "request": "launch",
+ "program": "${file}",
+ "console": "integratedTerminal",
+ "env": {
+ "PYTHONPATH": "${workspaceFolder}",
+ "MY_PROVIDER_API_KEY": "YOUR_API_KEY"
+ }
+ }
+ ]
+}
+```
+
+If you run with the debugger, after you update `"MY_PROVIDER_API_KEY": "YOUR_API_KEY"` you can remove this from the test script:
+
+`os.environ["MY_PROVIDER_KEY"] = "KEY_GOES_HERE"`
+
+---
+
+### 4. Implement Required Methods
+
+It's wise to follow `completion()` in `litellm/llms/custom_httpx/llm_http_handler.py`
+
+You will see it calls each of the methods defined in the base class.
+
+The debugger is your friend.
+
+###### `validate_environment`
+
+Setup headers, validate key/model:
+
+```python
+def validate_environment(...):
+ headers.update({
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json"
+ })
+ return headers
+```
+
+###### `get_complete_url`
+
+Return the final request URL:
+
+```python
+def get_complete_url(...):
+ return f"{api_base}/{model}"
+```
+
+###### `transform_request`
+
+Adapt OpenAI-style input into provider-specific format:
+
+```python
+def transform_request(...):
+ data = {"messages": messages, "params": optional_params}
+ return data
+```
+
+###### `transform_response`
+
+Process and map the raw provider response:
+
+```python
+def transform_response(...):
+ json = raw_response.json()
+ model_response.model = model
+ model_response.choices[0].message.content = json.get("output")
+ return model_response
+```
+
+###### `get_sync_custom_stream_wrapper` / `get_async_custom_stream_wrapper`
+
+If you need to do something these are here for you. See the `litellm/llms/sagemaker/chat/transformation.py` or the `litellm/llms/bytez/chat/transformation.py` implementation to better understand how to use these.
+
+Use `CustomStreamWrapper` + `httpx` streaming client to yield content.
+
+---
+
+### 🧪 Tests
+
+Create tests in `tests/test_litellm/llms/my_provider/chat/test.py`. Iterate until you are satisfied with the quality!
+
+---
+
+### Spare thoughts
+
+If you get stuck, see the other provider implementations, `ctrl + shift + f` and `ctrl + p` are your friends!
+
+You can also visit the [discord feedback channel](https://discord.gg/wuPM9dRgDw)
diff --git a/docs/my-website/docs/providers/aiml.md b/docs/my-website/docs/providers/aiml.md
index 1343cbf8d8e..9d763daf7d7 100644
--- a/docs/my-website/docs/providers/aiml.md
+++ b/docs/my-website/docs/providers/aiml.md
@@ -1,5 +1,23 @@
# AI/ML API
+https://aimlapi.com/
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | AI/ML API provides access to state-of-the-art AI models including flux-pro/v1.1 for high-quality image generation. |
+| Provider Route on LiteLLM | `aiml/` |
+| Link to Provider Doc | [AI/ML API ↗](https://docs.aimlapi.com/) |
+| Supported Operations | [`/chat/completions`], [`/images/generations`](#image-generation) |
+
+LiteLLM supports AI/ML API Image Generation calls.
+
+## API Base, Key
+```python
+# env variable
+os.environ['AIML_API_KEY'] = "your-api-key"
+os.environ['AIML_API_BASE'] = "https://api.aimlapi.com" # [optional]
+```
Getting started with the AI/ML API is simple. Follow these steps to set up your integration:
### 1. Get Your API Key
@@ -24,7 +42,7 @@ You can choose from LLama, Qwen, Flux, and 200+ other open and closed-source mod
import litellm
response = litellm.completion(
- model="openai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@@ -42,7 +60,7 @@ response = litellm.completion(
import litellm
response = litellm.completion(
- model="openai/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@@ -67,7 +85,7 @@ import litellm
async def main():
response = await litellm.acompletion(
- model="openai/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@@ -97,7 +115,7 @@ async def main():
try:
print("test acompletion + streaming")
response = await litellm.acompletion(
- model="openai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[{"content": "Hey, how's it going?", "role": "user"}],
@@ -125,7 +143,7 @@ import litellm
async def main():
response = await litellm.aembedding(
- model="openai/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
input="Your text string",
@@ -147,7 +165,7 @@ import litellm
async def main():
response = await litellm.aimage_generation(
- model="openai/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
+ model="aiml/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
prompt="A cute baby sea otter",
diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md
index 4ab4eb06086..1663d32ddfc 100644
--- a/docs/my-website/docs/providers/anthropic.md
+++ b/docs/my-website/docs/providers/anthropic.md
@@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
# Anthropic
LiteLLM supports all anthropic models.
+- `claude-sonnet-4-5-20250929`
+- `claude-opus-4-1-20250805`
- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`)
- `claude-3.7` (`claude-3-7-sonnet-20250219`)
- `claude-3.5` (`claude-3-5-sonnet-20240620`)
@@ -54,8 +56,29 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
# os.environ["ANTHROPIC_API_BASE"] = "" # [OPTIONAL] or 'ANTHROPIC_BASE_URL'
+# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
```
+### Custom API Base
+
+When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.
+
+If your custom endpoint already includes the full path or doesn't follow Anthropic's standard URL structure, you can disable this automatic suffix appending:
+
+```python
+import os
+
+os.environ["ANTHROPIC_API_BASE"] = "https://my-custom-endpoint.com/custom/path"
+os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # Prevents automatic suffix
+```
+
+Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
+- Base URL `https://my-proxy.com` → `https://my-proxy.com/v1/messages`
+- Base URL `https://my-proxy.com/api` → `https://my-proxy.com/api/v1/messages`
+
+With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
+- Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
+
## Usage
```python
@@ -107,7 +130,7 @@ model_list:
- model_name: claude-4 ### RECEIVED MODEL NAME ###
litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ###
- api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU")
+ api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY")
```
```bash
@@ -246,6 +269,7 @@ print(response)
| Model Name | Function Call |
|------------------|--------------------------------------------|
+| claude-sonnet-4-5 | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
@@ -606,11 +630,6 @@ response = await client.chat.completions.create(
## **Function/Tool Calling**
-:::info
-
-LiteLLM now uses Anthropic's 'tool' param 🎉 (v1.34.29+)
-:::
-
```python
from litellm import completion
@@ -669,6 +688,185 @@ response = completion(
)
```
+### Disable Tool Calling
+
+You can disable tool calling by setting the `tool_choice` to `"none"`.
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="anthropic/claude-3-opus-20240229",
+ messages=messages,
+ tools=tools,
+ tool_choice="none",
+)
+
+```
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: anthropic-claude-model
+ litellm_params:
+ model: anthropic/claude-3-opus-20240229
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+Replace `anything` with your LiteLLM Proxy Virtual Key, if [setup](../proxy/virtual_keys).
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer anything" \
+ -d '{
+ "model": "anthropic-claude-model",
+ "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp", "require_approval": "never"}],
+ "tool_choice": "none"
+ }'
+```
+
+
+
+
+
+### MCP Tool Calling
+
+Here's how to use MCP tool calling with Anthropic:
+
+
+
+
+LiteLLM supports MCP tool calling with Anthropic in the OpenAI Responses API format.
+
+
+
+
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
+
+tools=[
+ {
+ "type": "mcp",
+ "server_label": "deepwiki",
+ "server_url": "https://mcp.deepwiki.com/mcp",
+ "require_approval": "never",
+ },
+]
+
+response = completion(
+ model="anthropic/claude-sonnet-4-20250514",
+ messages=[{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ tools=tools
+)
+```
+
+
+
+
+```python
+import os
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
+
+tools = [
+ {
+ "type": "url",
+ "url": "https://mcp.deepwiki.com/mcp",
+ "name": "deepwiki-mcp",
+ }
+]
+response = completion(
+ model="anthropic/claude-sonnet-4-20250514",
+ messages=[{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ tools=tools
+)
+
+print(response)
+```
+
+
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: claude-4-sonnet
+ litellm_params:
+ model: anthropic/claude-sonnet-4-20250514
+ api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "claude-4-sonnet",
+ "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ "tools": [{"type": "mcp", "server_label": "deepwiki", "server_url": "https://mcp.deepwiki.com/mcp", "require_approval": "never"}]
+ }'
+```
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "claude-4-sonnet",
+ "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ "tools": [
+ {
+ "type": "url",
+ "url": "https://mcp.deepwiki.com/mcp",
+ "name": "deepwiki-mcp",
+ }
+ ]
+ }'
+```
+
+
+
+
+
### Parallel Function Calling
diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md
index d0b03719868..1feec52b3ec 100644
--- a/docs/my-website/docs/providers/azure/azure.md
+++ b/docs/my-website/docs/providers/azure/azure.md
@@ -9,9 +9,9 @@ import TabItem from '@theme/TabItem';
| Property | Details |
|-------|-------|
-| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
-| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) |
-| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
+| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
+| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) |
+| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
## API Keys, Params
@@ -175,6 +175,25 @@ print(response)
+### Setting API Version
+
+You can set the `api_version` for Azure OpenAI in your proxy config.yaml in the following ways
+
+#### Option 1: Per Model Configuration
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: azure/my-gpt4-deployment
+ api_base: https://your-resource.openai.azure.com/
+ api_version: "2024-08-01-preview" # Set version per model
+ api_key: os.environ/AZURE_API_KEY
+```
+
+
+
+
## Azure OpenAI Chat Completion Models
@@ -188,6 +207,7 @@ print(response)
|------------------|----------------------------------------|
| o1-mini | `response = completion(model="azure/", messages=messages)` |
| o1-preview | `response = completion(model="azure/", messages=messages)` |
+| gpt-5 | `response = completion(model="azure/", messages=messages)` |
| gpt-4o-mini | `completion('azure/', messages)` |
| gpt-4o | `completion('azure/', messages)` |
| gpt-4 | `completion('azure/', messages)` |
@@ -349,6 +369,82 @@ model_list:
+## GPT-5 Models
+
+| Property | Details |
+|-------|-------|
+| Description | Azure OpenAI GPT-5 models |
+| Provider Route on LiteLLM | `azure/gpt5_series/` or `azure/gpt-5-deployment-name` |
+
+LiteLLM supports using Azure GPT-5 models in one of the two ways:
+1. Explicit Routing: `model = azure/gpt5_series/`. In this scenario the model onboarded to litellm follows the format `model=azure/gpt5_series/`.
+2. Inferred Routing (If the azure deployment name contains `gpt-5` in the name): `model = azure/gpt-5-mini`. In this scenario the model onboarded to litellm follows the format `model=azure/gpt-5-mini`.
+
+#### Explicit Routing
+Use `azure/gpt5_series/` for explicit GPT-5 model routing.
+
+
+
+
+```python
+import litellm
+
+response = litellm.completion(
+ model="azure/gpt5_series/my-gpt-5-deployment",
+ messages=[{"role": "user", "content": "Hello, world!"}]
+)
+```
+
+
+
+```yaml
+model_list:
+ - model_name: gpt-5
+ litellm_params:
+ model: azure/gpt5_series/my-gpt-5-deployment
+ api_base: os.environ/AZURE_API_BASE
+ api_key: os.environ/AZURE_API_KEY
+```
+
+
+
+
+#### Inferred Routing (gpt-5 in the deployment name)
+If your Azure deployment name contains `gpt-5`, LiteLLM automatically recognizes it as a GPT-5 model.
+
+
+
+
+```python
+import litellm
+
+# Deployment name contains 'gpt-5' - automatically inferred
+response = litellm.completion(
+ model="azure/my-gpt-5-deployment",
+ messages=[{"role": "user", "content": "Hello, world!"}]
+)
+```
+
+
+
+
+```yaml
+model_list:
+ - model_name: gpt-5-mini
+ litellm_params:
+ model: azure/my-gpt-5-deployment # deployment name contains 'gpt-5'
+ api_base: os.environ/AZURE_API_BASE
+ api_key: os.environ/AZURE_API_KEY
+```
+
+
+
+
+
+
+
+
+
## Azure Audio Model
@@ -558,6 +654,7 @@ model_list:
tenant_id: os.environ/AZURE_TENANT_ID
client_id: os.environ/AZURE_CLIENT_ID
client_secret: os.environ/AZURE_CLIENT_SECRET
+ azure_scope: os.environ/AZURE_SCOPE # defaults to "https://cognitiveservices.azure.com/.default"
```
Test it
@@ -594,6 +691,7 @@ model_list:
client_id: os.environ/AZURE_CLIENT_ID
azure_username: os.environ/AZURE_USERNAME
azure_password: os.environ/AZURE_PASSWORD
+ azure_scope: os.environ/AZURE_SCOPE # defaults to "https://cognitiveservices.azure.com/.default"
```
Test it
@@ -616,23 +714,43 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
### Azure AD Token Refresh - `DefaultAzureCredential`
-Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests
+Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests. `DefaultAzureCredential` automatically discovers and uses available Azure credentials from multiple sources.
+**Option 1: Explicit DefaultAzureCredential (Recommended)**
```python
from litellm import completion
from azure.identity import DefaultAzureCredential, get_bearer_token_provider
+# DefaultAzureCredential automatically discovers credentials from:
+# - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID)
+# - Managed Identity (AKS, Azure VMs, etc.)
+# - Azure CLI credentials
+# - And other Azure identity sources
token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")
-
response = completion(
model = "azure/", # model = azure/
api_base = "", # azure api base
api_version = "", # azure api version
- azure_ad_token_provider=token_provider
+ azure_ad_token_provider=token_provider,
+ messages = [{"role": "user", "content": "good morning"}],
+)
+```
+
+**Option 2: LiteLLM Auto-Fallback to DefaultAzureCredential**
+```python
+import litellm
+
+# Enable automatic fallback to DefaultAzureCredential
+litellm.enable_azure_ad_token_refresh = True
+
+response = litellm.completion(
+ model = "azure/",
+ api_base = "",
+ api_version = "",
messages = [{"role": "user", "content": "good morning"}],
)
```
@@ -640,6 +758,8 @@ response = completion(
+**Scenario 1: With Environment Variables (Traditional)**
+
1. Add relevant env vars
```bash
@@ -661,12 +781,48 @@ litellm_settings:
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
```
+**Scenario 2: Managed Identity (AKS, Azure VMs) - No Hard-coded Credentials Required**
+
+Perfect for AKS clusters, Azure VMs, or other managed environments where Azure automatically injects credentials.
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: azure/your-deployment-name
+ api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
+
+litellm_settings:
+ enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
+```
+
+**Scenario 3: Azure CLI Authentication**
+
+If you're authenticated via `az login`, no additional configuration needed:
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: azure/your-deployment-name
+ api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
+
+litellm_settings:
+ enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
+```
+
3. Start proxy
```bash
litellm --config /path/to/config.yaml
```
+**How it works**:
+- LiteLLM first tries Service Principal authentication (if environment variables are available)
+- If that fails, it automatically falls back to `DefaultAzureCredential`
+- `DefaultAzureCredential` will use Managed Identity, Azure CLI credentials, or other available Azure identity sources
+- This eliminates the need for hard-coded credentials in managed environments like AKS
+
@@ -775,7 +931,7 @@ curl http://localhost:4000/v1/batches \
```python
retrieved_batch = client.batches.retrieve(
batch.id,
- extra_body={"custom_llm_provider": "azure"}
+ extra_query={"custom_llm_provider": "azure"}
)
```
@@ -822,7 +978,7 @@ curl http://localhost:4000/v1/batches/batch_abc123/cancel \
```python
-client.batches.list(extra_body={"custom_llm_provider": "azure"})
+client.batches.list(extra_query={"custom_llm_provider": "azure"})
```
@@ -1001,129 +1157,6 @@ Expected Response:
{"data":[{"id":"batch_R3V...}
```
-
-## **Azure Responses API**
-
-| Property | Details |
-|-------|-------|
-| Description | Azure OpenAI Responses API |
-| `custom_llm_provider` on LiteLLM | `azure/` |
-| Supported Operations | `/v1/responses`|
-| Azure OpenAI Responses API | [Azure OpenAI Responses API ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/responses?tabs=python-secure) |
-| Cost Tracking, Logging Support | ✅ LiteLLM will log, track cost for Responses API Requests |
-| Supported OpenAI Params | ✅ All OpenAI params are supported, [See here](https://github.com/BerriAI/litellm/blob/0717369ae6969882d149933da48eeb8ab0e691bd/litellm/llms/openai/responses/transformation.py#L23) |
-
-## Usage
-
-## Create a model response
-
-
-
-
-#### Non-streaming
-
-```python showLineNumbers title="Azure Responses API"
-import litellm
-
-# Non-streaming response
-response = litellm.responses(
- model="azure/o1-pro",
- input="Tell me a three sentence bedtime story about a unicorn.",
- max_output_tokens=100,
- api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
- api_base="https://litellm8397336933.openai.azure.com/",
- api_version="2023-03-15-preview",
-)
-
-print(response)
-```
-
-#### Streaming
-```python showLineNumbers title="Azure Responses API"
-import litellm
-
-# Streaming response
-response = litellm.responses(
- model="azure/o1-pro",
- input="Tell me a three sentence bedtime story about a unicorn.",
- stream=True,
- api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
- api_base="https://litellm8397336933.openai.azure.com/",
- api_version="2023-03-15-preview",
-)
-
-for event in response:
- print(event)
-```
-
-
-
-
-First, add this to your litellm proxy config.yaml:
-```yaml showLineNumbers title="Azure Responses API"
-model_list:
- - model_name: o1-pro
- litellm_params:
- model: azure/o1-pro
- api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
- api_base: https://litellm8397336933.openai.azure.com/
- api_version: 2023-03-15-preview
-```
-
-Start your LiteLLM proxy:
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-Then use the OpenAI SDK pointed to your proxy:
-
-#### Non-streaming
-```python showLineNumbers
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
- base_url="http://localhost:4000", # Your proxy URL
- api_key="your-api-key" # Your proxy API key
-)
-
-# Non-streaming response
-response = client.responses.create(
- model="o1-pro",
- input="Tell me a three sentence bedtime story about a unicorn."
-)
-
-print(response)
-```
-
-#### Streaming
-```python showLineNumbers
-from openai import OpenAI
-
-# Initialize client with your proxy URL
-client = OpenAI(
- base_url="http://localhost:4000", # Your proxy URL
- api_key="your-api-key" # Your proxy API key
-)
-
-# Streaming response
-response = client.responses.create(
- model="o1-pro",
- input="Tell me a three sentence bedtime story about a unicorn.",
- stream=True
-)
-
-for event in response:
- print(event)
-```
-
-
-
-
-
-
## Advanced
### Azure API Load-Balancing
diff --git a/docs/my-website/docs/providers/azure/azure_responses.md b/docs/my-website/docs/providers/azure/azure_responses.md
new file mode 100644
index 00000000000..34ec0e194f7
--- /dev/null
+++ b/docs/my-website/docs/providers/azure/azure_responses.md
@@ -0,0 +1,295 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure Responses API
+
+| Property | Details |
+|-------|-------|
+| Description | Azure OpenAI Responses API |
+| `custom_llm_provider` on LiteLLM | `azure/` |
+| Supported Operations | `/v1/responses`|
+| Azure OpenAI Responses API | [Azure OpenAI Responses API ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/responses?tabs=python-secure) |
+| Cost Tracking, Logging Support | ✅ LiteLLM will log, track cost for Responses API Requests |
+| Supported OpenAI Params | ✅ All OpenAI params are supported, [See here](https://github.com/BerriAI/litellm/blob/0717369ae6969882d149933da48eeb8ab0e691bd/litellm/llms/openai/responses/transformation.py#L23) |
+
+## Usage
+
+## Create a model response
+
+
+
+
+#### Non-streaming
+
+```python showLineNumbers title="Azure Responses API"
+import litellm
+
+# Non-streaming response
+response = litellm.responses(
+ model="azure/o1-pro",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ max_output_tokens=100,
+ api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+ api_base="https://litellm8397336933.openai.azure.com/",
+ api_version="2023-03-15-preview",
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers title="Azure Responses API"
+import litellm
+
+# Streaming response
+response = litellm.responses(
+ model="azure/o1-pro",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ stream=True,
+ api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+ api_base="https://litellm8397336933.openai.azure.com/",
+ api_version="2023-03-15-preview",
+)
+
+for event in response:
+ print(event)
+```
+
+
+
+
+First, add this to your litellm proxy config.yaml:
+```yaml showLineNumbers title="Azure Responses API"
+model_list:
+ - model_name: o1-pro
+ litellm_params:
+ model: azure/o1-pro
+ api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
+ api_base: https://litellm8397336933.openai.azure.com/
+ api_version: 2023-03-15-preview
+```
+
+Start your LiteLLM proxy:
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+Then use the OpenAI SDK pointed to your proxy:
+
+#### Non-streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.responses.create(
+ model="o1-pro",
+ input="Tell me a three sentence bedtime story about a unicorn."
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.responses.create(
+ model="o1-pro",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ stream=True
+)
+
+for event in response:
+ print(event)
+```
+
+
+
+
+## Azure Codex Models
+
+Codex models use Azure's new [/v1/preview API](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-lifecycle?tabs=key#next-generation-api) which provides ongoing access to the latest features with no need to update `api-version` each month.
+
+**LiteLLM will send your requests to the `/v1/preview` endpoint when you set `api_version="preview"`.**
+
+
+
+
+#### Non-streaming
+
+```python showLineNumbers title="Azure Codex Models"
+import litellm
+
+# Non-streaming response with Codex models
+response = litellm.responses(
+ model="azure/codex-mini",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ max_output_tokens=100,
+ api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+ api_base="https://litellm8397336933.openai.azure.com",
+ api_version="preview", # 👈 key difference
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers title="Azure Codex Models"
+import litellm
+
+# Streaming response with Codex models
+response = litellm.responses(
+ model="azure/codex-mini",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ stream=True,
+ api_key=os.getenv("AZURE_RESPONSES_OPENAI_API_KEY"),
+ api_base="https://litellm8397336933.openai.azure.com",
+ api_version="preview", # 👈 key difference
+)
+
+for event in response:
+ print(event)
+```
+
+
+
+
+First, add this to your litellm proxy config.yaml:
+```yaml showLineNumbers title="Azure Codex Models"
+model_list:
+ - model_name: codex-mini
+ litellm_params:
+ model: azure/codex-mini
+ api_key: os.environ/AZURE_RESPONSES_OPENAI_API_KEY
+ api_base: https://litellm8397336933.openai.azure.com
+ api_version: preview # 👈 key difference
+```
+
+Start your LiteLLM proxy:
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+Then use the OpenAI SDK pointed to your proxy:
+
+#### Non-streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.responses.create(
+ model="codex-mini",
+ input="Tell me a three sentence bedtime story about a unicorn."
+)
+
+print(response)
+```
+
+#### Streaming
+```python showLineNumbers
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.responses.create(
+ model="codex-mini",
+ input="Tell me a three sentence bedtime story about a unicorn.",
+ stream=True
+)
+
+for event in response:
+ print(event)
+```
+
+
+
+
+
+## Calling via `/chat/completions`
+
+You can also call the Azure Responses API via the `/chat/completions` endpoint.
+
+
+
+
+
+```python showLineNumbers
+from litellm import completion
+import os
+
+os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
+os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
+os.environ["AZURE_API_KEY"] = "my-api-key"
+
+response = completion(
+ model="azure/responses/my-custom-o1-pro",
+ messages=[{"role": "user", "content": "Hello world"}],
+)
+
+print(response)
+```
+
+
+
+1. Setup config.yaml
+
+```yaml showLineNumbers
+model_list:
+ - model_name: my-custom-o1-pro
+ litellm_params:
+ model: azure/responses/my-custom-o1-pro
+ api_key: os.environ/AZURE_API_KEY
+ api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
+ api_version: 2023-03-15-preview
+```
+
+2. Start LiteLLM proxy
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash
+curl http://localhost:4000/v1/chat/completions \
+ -X POST \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_API_KEY" \
+ -d '{
+ "model": "my-custom-o1-pro",
+ "messages": [{"role": "user", "content": "Hello world"}]
+ }'
+```
+
+
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md
index 60f7ecb2a5c..b1b5de5bb34 100644
--- a/docs/my-website/docs/providers/azure_ai.md
+++ b/docs/my-website/docs/providers/azure_ai.md
@@ -339,7 +339,7 @@ documents = [
]
response = rerank(
- model="azure_ai/rerank-english-v3.0",
+ model="azure_ai/cohere-rerank-v3.5",
query=query,
documents=documents,
top_n=3,
@@ -362,9 +362,9 @@ model_list:
litellm_params:
model: together_ai/Salesforce/Llama-Rank-V1
api_key: os.environ/TOGETHERAI_API_KEY
- - model_name: rerank-english-v3.0
+ - model_name: cohere-rerank-v3.5
litellm_params:
- model: azure_ai/rerank-english-v3.0
+ model: azure_ai/cohere-rerank-v3.5
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
```
@@ -384,7 +384,7 @@ curl http://0.0.0.0:4000/rerank \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
- "model": "rerank-english-v3.0",
+ "model": "cohere-rerank-v3.5",
"query": "What is the capital of the United States?",
"documents": [
"Carson City is the capital city of the American state of Nevada.",
diff --git a/docs/my-website/docs/providers/azure_ai_img.md b/docs/my-website/docs/providers/azure_ai_img.md
new file mode 100644
index 00000000000..8e2f5226866
--- /dev/null
+++ b/docs/my-website/docs/providers/azure_ai_img.md
@@ -0,0 +1,266 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure AI Image Generation
+
+Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
+| Provider Route on LiteLLM | `azure_ai/` |
+| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
+| Supported Operations | [`/images/generations`](#image-generation) |
+
+## Setup
+
+### API Key & Base URL
+
+```python showLineNumbers
+# Set your Azure AI API credentials
+import os
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
+```
+
+Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
+
+## Supported Models
+
+| Model Name | Description | Cost per Image |
+|------------|-------------|----------------|
+| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
+| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
+
+## Image Generation
+
+### Usage - LiteLLM Python SDK
+
+
+
+
+```python showLineNumbers title="Basic Image Generation"
+import litellm
+import os
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+
+# Generate a single image
+response = litellm.image_generation(
+ model="azure_ai/FLUX.1-Kontext-pro",
+ prompt="A cute baby sea otter swimming in crystal clear water",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"]
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```python showLineNumbers title="FLUX 1.1 Pro Image Generation"
+import litellm
+import os
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+
+# Generate image with FLUX 1.1 Pro
+response = litellm.image_generation(
+ model="azure_ai/FLUX-1.1-pro",
+ prompt="A futuristic cityscape at night with neon lights and flying cars",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"]
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```python showLineNumbers title="Async Image Generation"
+import litellm
+import asyncio
+import os
+
+async def generate_image():
+ # Set your API credentials
+ os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+ os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+
+ # Generate image asynchronously
+ response = await litellm.aimage_generation(
+ model="azure_ai/FLUX.1-Kontext-pro",
+ prompt="A beautiful sunset over mountains with vibrant colors",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"],
+ n=1,
+ )
+
+ print(response.data[0].url)
+ return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+
+
+
+
+```python showLineNumbers title="Advanced Image Generation with Parameters"
+import litellm
+import os
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+
+# Generate image with additional parameters
+response = litellm.image_generation(
+ model="azure_ai/FLUX-1.1-pro",
+ prompt="A majestic dragon soaring over a medieval castle at dawn",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"],
+ n=1,
+ size="1024x1024",
+ quality="standard"
+)
+
+for image in response.data:
+ print(f"Generated image URL: {image.url}")
+```
+
+
+
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Azure AI Image Generation Configuration"
+model_list:
+ - model_name: azure-flux-kontext
+ litellm_params:
+ model: azure_ai/FLUX.1-Kontext-pro
+ api_key: os.environ/AZURE_AI_API_KEY
+ api_base: os.environ/AZURE_AI_API_BASE
+ model_info:
+ mode: image_generation
+
+ - model_name: azure-flux-11-pro
+ litellm_params:
+ model: azure_ai/FLUX-1.1-pro
+ api_key: os.environ/AZURE_AI_API_KEY
+ api_base: os.environ/AZURE_AI_API_BASE
+ model_info:
+ mode: image_generation
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make requests with OpenAI Python SDK
+
+
+
+
+```python showLineNumbers title="Azure AI Image Generation via Proxy - OpenAI SDK"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="sk-1234" # Your proxy API key
+)
+
+# Generate image with FLUX Kontext Pro
+response = client.images.generate(
+ model="azure-flux-kontext",
+ prompt="A serene Japanese garden with cherry blossoms and a peaceful pond",
+ n=1,
+ size="1024x1024"
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```python showLineNumbers title="Azure AI Image Generation via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.image_generation(
+ model="litellm_proxy/azure-flux-11-pro",
+ prompt="A cyberpunk warrior in a neon-lit alleyway",
+ api_base="http://localhost:4000",
+ api_key="sk-1234"
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```bash showLineNumbers title="Azure AI Image Generation via Proxy - cURL"
+curl --location 'http://localhost:4000/v1/images/generations' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer sk-1234' \
+--data '{
+ "model": "azure-flux-kontext",
+ "prompt": "A cozy coffee shop interior with warm lighting and rustic wooden furniture",
+ "n": 1,
+ "size": "1024x1024"
+}'
+```
+
+
+
+
+## Supported Parameters
+
+Azure AI Image Generation supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description | Default | Example |
+|-----------|------|-------------|---------|---------|
+| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
+| `model` | string | The FLUX model to use for generation | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
+| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
+| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
+| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
+| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
+
+## Getting Started
+
+1. Create an account at [Azure AI Studio](https://ai.azure.com/)
+2. Deploy a FLUX model in your Azure AI Studio workspace
+3. Get your API key and endpoint from the deployment details
+4. Set your `AZURE_AI_API_KEY` and `AZURE_AI_API_BASE` environment variables
+5. Start generating images using LiteLLM
+
+## Additional Resources
+
+- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
+- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)
diff --git a/docs/my-website/docs/providers/azure_ai_img_edit.md b/docs/my-website/docs/providers/azure_ai_img_edit.md
new file mode 100644
index 00000000000..0d5408f0af4
--- /dev/null
+++ b/docs/my-website/docs/providers/azure_ai_img_edit.md
@@ -0,0 +1,260 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure AI Image Editing
+
+Azure AI provides powerful image editing capabilities using FLUX models from Black Forest Labs to modify existing images based on text descriptions.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Azure AI Image Editing uses FLUX models to modify existing images based on text prompts. |
+| Provider Route on LiteLLM | `azure_ai/` |
+| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
+| Supported Operations | [`/images/edits`](#image-editing) |
+
+## Setup
+
+### API Key & Base URL & API Version
+
+```python showLineNumbers
+# Set your Azure AI API credentials
+import os
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
+os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview" # Example API version
+```
+
+Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
+
+## Supported Models
+
+| Model Name | Description | Cost per Image |
+|------------|-------------|----------------|
+| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding for editing | $0.04 |
+
+## Image Editing
+
+### Usage - LiteLLM Python SDK
+
+
+
+
+```python showLineNumbers title="Basic Image Editing"
+import os
+import base64
+from pathlib import Path
+
+import litellm
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
+
+# Edit an image with a prompt
+response = litellm.image_edit(
+ model="azure_ai/FLUX.1-Kontext-pro",
+ image=open("path/to/your/image.png", "rb"),
+ prompt="Add a winter theme with snow and cold colors",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"],
+ api_version=os.environ["AZURE_AI_API_VERSION"]
+)
+
+img_base64 = response.data[0].get("b64_json")
+img_bytes = base64.b64decode(img_base64)
+path = Path("edited_image.png")
+path.write_bytes(img_bytes)
+```
+
+
+
+
+
+```python showLineNumbers title="Async Image Editing"
+import os
+import base64
+from pathlib import Path
+
+import litellm
+import asyncio
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
+
+async def edit_image():
+ # Edit image asynchronously
+ response = await litellm.aimage_edit(
+ model="azure_ai/FLUX.1-Kontext-pro",
+ image=open("path/to/your/image.png", "rb"),
+ prompt="Make this image look like a watercolor painting",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"],
+ api_version=os.environ["AZURE_AI_API_VERSION"]
+ )
+ img_base64 = response.data[0].get("b64_json")
+ img_bytes = base64.b64decode(img_base64)
+ path = Path("async_edited_image.png")
+ path.write_bytes(img_bytes)
+
+# Run the async function
+asyncio.run(edit_image())
+```
+
+
+
+
+
+```python showLineNumbers title="Advanced Image Editing with Parameters"
+import os
+import base64
+from pathlib import Path
+
+import litellm
+
+# Set your API credentials
+os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
+os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
+os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
+
+# Edit image with additional parameters
+response = litellm.image_edit(
+ model="azure_ai/FLUX.1-Kontext-pro",
+ image=open("path/to/your/image.png", "rb"),
+ prompt="Add magical elements like floating crystals and mystical lighting",
+ api_base=os.environ["AZURE_AI_API_BASE"],
+ api_key=os.environ["AZURE_AI_API_KEY"],
+ api_version=os.environ["AZURE_AI_API_VERSION"],
+ n=1
+)
+img_base64 = response.data[0].get("b64_json")
+img_bytes = base64.b64decode(img_base64)
+path = Path("advanced_edited_image.png")
+path.write_bytes(img_bytes)
+```
+
+
+
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Azure AI Image Editing Configuration"
+model_list:
+ - model_name: azure-flux-kontext-edit
+ litellm_params:
+ model: azure_ai/FLUX.1-Kontext-pro
+ api_key: os.environ/AZURE_AI_API_KEY
+ api_base: os.environ/AZURE_AI_API_BASE
+ api_version: os.environ/AZURE_AI_API_VERSION
+ model_info:
+ mode: image_edit
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make image editing requests with OpenAI Python SDK
+
+
+
+
+```python showLineNumbers title="Azure AI Image Editing via Proxy - OpenAI SDK"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="sk-1234" # Your proxy API key
+)
+
+# Edit image with FLUX Kontext Pro
+response = client.images.edit(
+ model="azure-flux-kontext-edit",
+ image=open("path/to/your/image.png", "rb"),
+ prompt="Transform this image into a beautiful oil painting style",
+)
+
+img_base64 = response.data[0].b64_json
+img_bytes = base64.b64decode(img_base64)
+path = Path("proxy_edited_image.png")
+path.write_bytes(img_bytes)
+```
+
+
+
+
+
+```python showLineNumbers title="Azure AI Image Editing via Proxy - LiteLLM SDK"
+import litellm
+
+# Edit image through proxy
+response = litellm.image_edit(
+ model="litellm_proxy/azure-flux-kontext-edit",
+ image=open("path/to/your/image.png", "rb"),
+ prompt="Add a mystical forest background with magical creatures",
+ api_base="http://localhost:4000",
+ api_key="sk-1234"
+)
+
+img_base64 = response.data[0].b64_json
+img_bytes = base64.b64decode(img_base64)
+path = Path("proxy_edited_image.png")
+path.write_bytes(img_bytes)
+```
+
+
+
+
+
+```bash showLineNumbers title="Azure AI Image Editing via Proxy - cURL"
+curl --location 'http://localhost:4000/v1/images/edits' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'model="azure-flux-kontext-edit"' \
+--form 'prompt="Convert this image to a vintage sepia tone with old-fashioned effects"' \
+--form 'image=@"path/to/your/image.png"'
+```
+
+
+
+
+## Supported Parameters
+
+Azure AI Image Editing supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description | Default | Example |
+|-----------|------|-------------|---------|---------|
+| `image` | file | The image file to edit | Required | File object or binary data |
+| `prompt` | string | Text description of the desired changes | Required | `"Add snow and winter elements"` |
+| `model` | string | The FLUX model to use for editing | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
+| `n` | integer | Number of edited images to generate (You can specify only 1) | `1` | `1` |
+| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
+| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
+| `api_version` | string | API version for Azure AI | Required | `"2025-04-01-preview"` |
+
+## Getting Started
+
+1. Create an account at [Azure AI Studio](https://ai.azure.com/)
+2. Deploy a FLUX model in your Azure AI Studio workspace
+3. Get your API key and endpoint from the deployment details
+4. Set your `AZURE_AI_API_KEY`, `AZURE_AI_API_BASE` and `AZURE_AI_API_VERSION` environment variables
+5. Prepare your source image
+6. Use `litellm.image_edit()` to modify your images with text instructions
+
+## Additional Resources
+
+- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
+- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/baseten.md b/docs/my-website/docs/providers/baseten.md
index 902b1548faa..4e42cdf0447 100644
--- a/docs/my-website/docs/providers/baseten.md
+++ b/docs/my-website/docs/providers/baseten.md
@@ -1,23 +1,106 @@
-# Baseten
-LiteLLM supports any Text-Gen-Interface models on Baseten.
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
-[Here's a tutorial on deploying a huggingface TGI model (Llama2, CodeLlama, WizardCoder, Falcon, etc.) on Baseten](https://truss.baseten.co/examples/performance/tgi-server)
+# Baseten
+
+LiteLLM supports both Baseten Model APIs and dedicated deployments with automatic routing.
+
+## API Types
+
+### Model API (Default)
+- **URL**: `https://inference.baseten.co/v1`
+- **Format**: `baseten/` (e.g., `baseten/openai/gpt-oss-120b`)
+- **Best for**: Quick access to popular models
+
+### Dedicated Deployments
+- **URL**: `https://model-{id}.api.baseten.co/environments/production/sync/v1`
+- **Format**: `baseten/{8-digit-alphanumeric-code}` (e.g., `baseten/abcd1234`)
+- **Best for**: Custom models, latency SLAs
+
+:::tip
+**Automatic Routing**: LiteLLM detects the type based on model format:
+- 8-digit alphanumeric codes → Dedicated deployment
+- All other formats → Model API
+:::
+
+
+## Quick Start
-### API KEYS
```python
-import os
-os.environ["BASETEN_API_KEY"] = ""
+import os
+from litellm import completion
+
+os.environ['BASETEN_API_KEY'] = "your-api-key"
+
+# Model API (default)
+response = completion(
+ model="baseten/openai/gpt-oss-120b",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+
+# Dedicated deployment (8-digit ID)
+response = completion(
+ model="baseten/abcd1234",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
```
-### Baseten Models
-Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten.
+## Examples
-Example Baseten Usage - Note: liteLLM supports all models deployed on Baseten
+### Basic Usage
+```python
+# Model API
+response = completion(
+ model="baseten/openai/gpt-oss-120b",
+ messages=[{"role": "user", "content": "Explain quantum computing"}],
+ max_tokens=500,
+ temperature=0.7
+)
-Usage: Pass `model=baseten/`
+# Dedicated deployment
+response = completion(
+ model="baseten/abcd1234",
+ messages=[{"role": "user", "content": "Explain quantum computing"}],
+ max_tokens=500,
+ temperature=0.7
+)
+```
-| Model Name | Function Call | Required OS Variables |
-|------------------|--------------------------------------------|------------------------------------|
-| Falcon 7B | `completion(model='baseten/qvv0xeq', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
-| Wizard LM | `completion(model='baseten/q841o8w', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
-| MPT 7B Base | `completion(model='baseten/31dxrj3', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
+### Streaming (Model API only)
+```python
+response = completion(
+ model="baseten/openai/gpt-oss-120b",
+ messages=[{"role": "user", "content": "Write a poem"}],
+ stream=True,
+ stream_options={"include_usage": True}
+)
+
+for chunk in response:
+ if chunk.choices and chunk.choices[0].delta.content:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+## Usage with LiteLLM Proxy
+
+1. **Config**:
+```yaml
+model_list:
+ - model_name: baseten-model
+ litellm_params:
+ model: baseten/openai/gpt-oss-120b
+ api_key: your-baseten-api-key
+```
+
+2. **Request**:
+```python
+import openai
+client = openai.OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="baseten-model",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md
index 8217f429ff3..28cae80cc42 100644
--- a/docs/my-website/docs/providers/bedrock.md
+++ b/docs/my-website/docs/providers/bedrock.md
@@ -25,11 +25,32 @@ For **Amazon Nova Models**: Bump to v1.53.5+
:::
+## Authentication
+
:::info
LiteLLM uses boto3 to handle authentication. All these options are supported - https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#credentials.
:::
+
+LiteLLM supports API key authentication in addition to traditional boto3 authentication methods. For additional API key details, refer to [docs](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html).
+
+Option 1: use the AWS_BEARER_TOKEN_BEDROCK environment variable
+
+```bash
+export AWS_BEARER_TOKEN_BEDROCK="your-api-key"
+```
+
+Option 2: use the api_key parameter to pass in API key for completion, embedding, image_generation API calls.
+
+```python
+response = completion(
+ model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+ messages=[{ "content": "Hello, how are you?","role": "user"}],
+ api_key="your-api-key"
+)
+```
+
## Usage
@@ -80,6 +101,7 @@ aws_profile_name: Optional[str],
aws_role_name: Optional[str],
aws_web_identity_token: Optional[str],
aws_bedrock_runtime_endpoint: Optional[str],
+api_key: Optional[str],
```
### 2. Start the proxy
@@ -287,6 +309,65 @@ print(response)
+## Usage - Request Metadata
+
+Attach metadata to Bedrock requests for logging and cost attribution.
+
+
+
+
+```python
+import os
+from litellm import completion
+
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = ""
+
+response = completion(
+ model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ requestMetadata={
+ "cost_center": "engineering",
+ "user_id": "user123"
+ }
+)
+```
+
+
+
+**Set on yaml**
+
+```yaml
+model_list:
+ - model_name: bedrock-claude-v1
+ litellm_params:
+ model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
+ requestMetadata:
+ cost_center: "engineering"
+```
+
+**Set on request**
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="bedrock-claude-v1",
+ messages=[{"role": "user", "content": "Hello"}],
+ extra_body={
+ "requestMetadata": {"cost_center": "engineering"}
+ }
+)
+```
+
+
+
+
## Usage - Function Calling / Tool calling
LiteLLM supports tool calling via Bedrock's Converse and Invoke API's.
@@ -446,7 +527,7 @@ print(f"\nResponse: {resp}")
## Usage - 'thinking' / 'reasoning content'
-This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1.
+This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1 + GPT-OSS models.
Works on v1.61.20+.
@@ -563,6 +644,150 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
Same as [Anthropic API response](../providers/anthropic#usage---thinking--reasoning_content).
+## Usage - Anthropic Beta Features
+
+LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
+
+- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
+- **Computer Use Tools** - AI that can interact with computer interfaces
+- **Token-Efficient Tools** - More efficient tool usage patterns
+- **Extended Output** - Up to 128K output tokens
+- **Enhanced Thinking** - Advanced reasoning capabilities
+
+### Supported Beta Features
+
+| Beta Feature | Header Value | Compatible Models | Description |
+|--------------|-------------|------------------|-------------|
+| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
+| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
+| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
+| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |
+| Interleaved Thinking | `interleaved-thinking-2025-05-14` | Claude 4 models | Enhanced thinking capabilities |
+| Extended Output | `output-128k-2025-02-19` | Claude 3.7 Sonnet | Up to 128K output tokens |
+| Developer Thinking | `dev-full-thinking-2025-05-14` | Claude 4 models | Raw thinking mode for developers |
+
+
+
+
+**Single Beta Feature**
+
+```python
+from litellm import completion
+import os
+
+# set env
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = ""
+
+# Use 1M context window with Claude Sonnet 4
+response = completion(
+ model="bedrock/anthropic.claude-sonnet-4-20250115-v1:0",
+ messages=[{"role": "user", "content": "Hello! Testing 1M context window."}],
+ max_tokens=100,
+ extra_headers={
+ "anthropic-beta": "context-1m-2025-08-07" # 👈 Enable 1M context
+ }
+)
+```
+
+**Multiple Beta Features**
+
+```python
+from litellm import completion
+
+# Combine multiple beta features (comma-separated)
+response = completion(
+ model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
+ messages=[{"role": "user", "content": "Testing multiple beta features"}],
+ max_tokens=100,
+ extra_headers={
+ "anthropic-beta": "computer-use-2024-10-22,context-1m-2025-08-07"
+ }
+)
+```
+
+**Computer Use Tools with Beta Features**
+
+```python
+from litellm import completion
+
+# Computer use tools automatically add computer-use-2024-10-22
+# You can add additional beta features
+response = completion(
+ model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
+ messages=[{"role": "user", "content": "Take a screenshot"}],
+ tools=[{
+ "type": "computer_20241022",
+ "name": "computer",
+ "display_width_px": 1920,
+ "display_height_px": 1080
+ }],
+ extra_headers={
+ "anthropic-beta": "context-1m-2025-08-07" # Additional beta feature
+ }
+)
+```
+
+
+
+
+**Set on YAML Config**
+
+```yaml
+model_list:
+ - model_name: claude-sonnet-4-1m
+ litellm_params:
+ model: bedrock/anthropic.claude-sonnet-4-20250115-v1:0
+ extra_headers:
+ anthropic-beta: "context-1m-2025-08-07" # 👈 Enable 1M context
+
+ - model_name: claude-computer-use
+ litellm_params:
+ model: bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0
+ extra_headers:
+ anthropic-beta: "computer-use-2024-10-22,context-1m-2025-08-07"
+
+general_settings:
+ forward_client_headers_to_llm_api: true # 👈 Required for client-side header forwarding
+```
+
+**Set on Request**
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-sonnet-4-1m",
+ messages=[{
+ "role": "user",
+ "content": "Testing 1M context window"
+ }],
+ extra_headers={
+ "anthropic-beta": "context-1m-2025-08-07"
+ }
+)
+```
+
+:::info
+**For client-side header forwarding**: When using the proxy and sending `anthropic-beta` headers from the client (like the OpenAI SDK), you need to enable `forward_client_headers_to_llm_api: true` in your proxy's `general_settings`. This tells the proxy to extract headers from HTTP requests and forward them to the underlying LLM provider.
+:::
+
+
+
+
+:::info
+
+Beta features may require special access or permissions in your AWS account. Some features are only available in specific AWS regions. Check the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) for availability and access requirements.
+
+:::
+
+
## Usage - Structured Output / JSON mode
@@ -724,6 +949,19 @@ curl http://0.0.0.0:4000/v1/chat/completions \
Example of using [Bedrock Guardrails with LiteLLM](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-use-converse-api.html)
+### Selective Content Moderation with `guarded_text`
+
+LiteLLM supports selective content moderation using the `guarded_text` content type. This allows you to wrap only specific content that should be moderated by Bedrock Guardrails, rather than evaluating the entire conversation.
+
+**How it works:**
+- Content with `type: "guarded_text"` gets automatically wrapped in `guardrailConverseContent` blocks
+- Only the wrapped content is evaluated by Bedrock Guardrails
+- Regular content with `type: "text"` bypasses guardrail evaluation
+
+:::note
+If `guarded_text` is not used, the entire conversation history will be sent to the guardrail for evaluation, which can increase latency and costs.
+:::
+
@@ -750,6 +988,24 @@ response = completion(
"trace": "disabled", # The trace behavior for the guardrail. Can either be "disabled" or "enabled"
},
)
+
+# Selective guardrail usage with guarded_text - only specific content is evaluated
+response_guard = completion(
+ model="anthropic.claude-v2",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What is the main topic of this legal document?"},
+ {"type": "guarded_text", "text": "This document contains sensitive legal information that should be moderated by guardrails."}
+ ]
+ }
+ ],
+ guardrailConfig={
+ "guardrailIdentifier": "gr-abc123",
+ "guardrailVersion": "DRAFT"
+ }
+)
```
@@ -828,7 +1084,20 @@ response = client.chat.completions.create(model="bedrock-claude-v1", messages =
temperature=0.7
)
-print(response)
+# For adding selective guardrail usage with guarded_text
+response_guard = client.chat.completions.create(model="bedrock-claude-v1", messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What is the main topic of this legal document?"},
+ {"type": "guarded_text", "text": "This document contains sensitive legal information that should be moderated by guardrails."}
+ ]
+ }
+],
+temperature=0.7
+)
+
+print(response_guard)
```
@@ -1467,6 +1736,91 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
+### OpenAI GPT OSS
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/converse/openai.gpt-oss-20b-1:0`, `bedrock/converse/openai.gpt-oss-120b-1:0` |
+| Provider Documentation | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
+
+
+
+
+```python title="GPT OSS SDK Usage" showLineNumbers
+from litellm import completion
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+# GPT OSS 20B model
+response = completion(
+ model="bedrock/converse/openai.gpt-oss-20b-1:0",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+)
+print(response.choices[0].message.content)
+
+# GPT OSS 120B model
+response = completion(
+ model="bedrock/converse/openai.gpt-oss-120b-1:0",
+ messages=[{"role": "user", "content": "Explain machine learning in simple terms"}],
+)
+print(response.choices[0].message.content)
+```
+
+
+
+
+
+**1. Add to config**
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+ - model_name: gpt-oss-20b
+ litellm_params:
+ model: bedrock/converse/openai.gpt-oss-20b-1:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: os.environ/AWS_REGION_NAME
+
+ - model_name: gpt-oss-120b
+ litellm_params:
+ model: bedrock/converse/openai.gpt-oss-120b-1:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: os.environ/AWS_REGION_NAME
+```
+
+**2. Start proxy**
+
+```bash title="Start LiteLLM Proxy" showLineNumbers
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash title="Test GPT OSS via Proxy" showLineNumbers
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-oss-20b",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What are the key benefits of open source AI?"
+ }
+ ]
+ }'
+```
+
+
+
+
## Provisioned throughput models
To use provisioned throughput Bedrock models pass
- `model=bedrock/`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@@ -1501,7 +1855,10 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
+| GPT-OSS 20B | `completion(model='bedrock/converse/openai.gpt-oss-20b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
+| Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
@@ -1525,6 +1882,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mistral 7B Instruct | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+
## Bedrock Embedding
### API keys
@@ -1546,11 +1904,29 @@ response = embedding(
print(response)
```
+#### Titan V2 - encoding_format support
+```python
+from litellm import embedding
+# Float format (default)
+response = embedding(
+ model="bedrock/amazon.titan-embed-text-v2:0",
+ input=["good morning from litellm"],
+ encoding_format="float" # Returns float array
+)
+
+# Binary format
+response = embedding(
+ model="bedrock/amazon.titan-embed-text-v2:0",
+ input=["good morning from litellm"],
+ encoding_format="base64" # Returns base64 encoded binary
+)
+```
+
## Supported AWS Bedrock Embedding Models
| Model Name | Usage | Supported Additional OpenAI params |
|----------------------|---------------------------------------------|-----|
-| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
+| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | `dimensions`, `encoding_format` |
| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
@@ -1639,6 +2015,39 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
+### Using Inference Profiles with Image Generation
+
+For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
+
+
+
+
+```python
+from litellm import image_generation
+
+response = image_generation(
+ model="bedrock/amazon.nova-canvas-v1:0",
+ model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
+ prompt="A cute baby sea otter"
+)
+print(f"response: {response}")
+```
+
+
+
+
+```yaml
+model_list:
+ - model_name: nova-canvas-inference-profile
+ litellm_params:
+ model: bedrock/amazon.nova-canvas-v1:0
+ model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
+ aws_region_name: "eu-west-1"
+```
+
+
+
+
## Supported AWS Bedrock Image Generation Models
| Model Name | Function Call |
@@ -1933,6 +2342,39 @@ response = completion(
Make the bedrock completion call
+---
+
+### Required AWS IAM Policy for AssumeRole
+
+To use `aws_role_name` (STS AssumeRole) with LiteLLM, your IAM user or role **must** have permission to call `sts:AssumeRole` on the target role. If you see an error like:
+
+```
+An error occurred (AccessDenied) when calling the AssumeRole operation: User: arn:aws:sts::...:assumed-role/litellm-ecs-task-role/... is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::...:role/Enterprise/BedrockCrossAccountConsumer
+```
+
+This means the IAM identity running LiteLLM does **not** have permission to assume the target role. You must update your IAM policy to allow this action.
+
+#### Example IAM Policy
+
+Replace `` with the ARN of the role you want to assume (e.g., `arn:aws:iam::123456789012:role/Enterprise/BedrockCrossAccountConsumer`).
+
+```json
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "sts:AssumeRole",
+ "Resource": ""
+ }
+ ]
+}
+```
+
+**Note:** The target role itself must also trust the calling IAM identity (via its trust policy) for AssumeRole to succeed. See [AWS AssumeRole docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-api.html) for more details.
+
+---
+
diff --git a/docs/my-website/docs/providers/bedrock_agents.md b/docs/my-website/docs/providers/bedrock_agents.md
index e6368705feb..4d027cbb3d8 100644
--- a/docs/my-website/docs/providers/bedrock_agents.md
+++ b/docs/my-website/docs/providers/bedrock_agents.md
@@ -196,7 +196,51 @@ for chunk in stream:
+## Provider-specific Parameters
+
+Any non-openai parameters will be passed to the agent as custom parameters.
+
+
+
+
+```python showLineNumbers title="Using custom parameters"
+from litellm import completion
+
+response = litellm.completion(
+ model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",
+ messages=[
+ {
+ "role": "user",
+ "content": "Hi who is ishaan cto of litellm, tell me 10 things about him",
+ }
+ ],
+ invocationId="my-test-invocation-id", # PROVIDER-SPECIFIC VALUE
+)
+```
+
+
+
+
+```yaml showLineNumbers title="LiteLLM Proxy Configuration"
+model_list:
+ - model_name: bedrock-agent-1
+ litellm_params:
+ model: bedrock/agent/L1RT58GYRW/MFPSBCXYTW
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-west-2
+ invocationId: my-test-invocation-id
+```
+
+
+
+
+
+
+
+
## Further Reading
- [AWS Bedrock Agents Documentation](https://aws.amazon.com/bedrock/agents/)
- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)
+
diff --git a/docs/my-website/docs/providers/bedrock_batches.md b/docs/my-website/docs/providers/bedrock_batches.md
new file mode 100644
index 00000000000..57487f7d2c9
--- /dev/null
+++ b/docs/my-website/docs/providers/bedrock_batches.md
@@ -0,0 +1,180 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bedrock Batches
+
+Use Amazon Bedrock Batch Inference API through LiteLLM.
+
+| Property | Details |
+|----------|---------|
+| Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously |
+| Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) |
+
+## Overview
+
+Use this to:
+
+- Run batch inference on large datasets with Bedrock models
+- Control batch model access by key/user/team (same as chat completion models)
+- Manage S3 storage for batch input/output files
+
+## (Proxy Admin) Usage
+
+Here's how to give developers access to your Bedrock Batch models.
+
+### 1. Setup config.yaml
+
+- Specify `mode: batch` for each model: Allows developers to know this is a batch model
+- Configure S3 bucket and AWS credentials for batch operations
+
+```yaml showLineNumbers title="litellm_config.yaml"
+model_list:
+ - model_name: "bedrock-batch-claude"
+ litellm_params:
+ model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
+ #########################################################
+ ########## batch specific params ########################
+ s3_bucket_name: litellm-proxy
+ s3_region_name: us-west-2
+ s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
+ model_info:
+ mode: batch # 👈 SPECIFY MODE AS BATCH, to tell user this is a batch model
+```
+
+**Required Parameters:**
+
+| Parameter | Description |
+|-----------|-------------|
+| `s3_bucket_name` | S3 bucket for batch input/output files |
+| `s3_region_name` | AWS region for S3 bucket |
+| `s3_access_key_id` | AWS access key for S3 bucket |
+| `s3_secret_access_key` | AWS secret key for S3 bucket |
+| `aws_batch_role_arn` | IAM role ARN for Bedrock batch operations. Bedrock Batch APIs require an IAM role ARN to be set. |
+| `mode: batch` | Indicates to LiteLLM this is a batch model |
+
+### 2. Create Virtual Key
+
+```bash showLineNumbers title="create_virtual_key.sh"
+curl -L -X POST 'https://{PROXY_BASE_URL}/key/generate' \
+-H 'Authorization: Bearer ${PROXY_API_KEY}' \
+-H 'Content-Type: application/json' \
+-d '{"models": ["bedrock-batch-claude"]}'
+```
+
+You can now use the virtual key to access the batch models (See Developer flow).
+
+## (Developer) Usage
+
+Here's how to create a LiteLLM managed file and execute Bedrock Batch CRUD operations with the file.
+
+### 1. Create request.jsonl
+
+- Check models available via `/model_group/info`
+- See all models with `mode: batch`
+- Set `model` in .jsonl to the model from `/model_group/info`
+
+```json showLineNumbers title="bedrock_batch_completions.jsonl"
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are an unhelpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+```
+
+Expectation:
+
+- LiteLLM translates this to the bedrock deployment specific value (e.g. `bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0`)
+
+### 2. Upload File
+
+Specify `target_model_names: ""` to enable LiteLLM managed files and request validation.
+
+model-name should be the same as the model-name in the request.jsonl
+
+
+
+
+```python showLineNumbers title="bedrock_batch.py"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://0.0.0.0:4000",
+ api_key="sk-1234",
+)
+
+# Upload file
+batch_input_file = client.files.create(
+ file=open("./bedrock_batch_completions.jsonl", "rb"), # {"model": "bedrock-batch-claude"} <-> {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"}
+ purpose="batch",
+ extra_body={"target_model_names": "bedrock-batch-claude"}
+)
+print(batch_input_file)
+```
+
+
+
+
+```bash showLineNumbers title="Upload File"
+curl http://localhost:4000/v1/files \
+ -H "Authorization: Bearer sk-1234" \
+ -F purpose="batch" \
+ -F file="@bedrock_batch_completions.jsonl" \
+ -F extra_body='{"target_model_names": "bedrock-batch-claude"}'
+```
+
+
+
+
+**Where is the file written?**:
+
+The file is written to S3 bucket specified in your config and prepared for Bedrock batch inference.
+
+### 3. Create the batch
+
+
+
+
+```python showLineNumbers title="bedrock_batch.py"
+...
+# Create batch
+batch = client.batches.create(
+ input_file_id=batch_input_file.id,
+ endpoint="/v1/chat/completions",
+ completion_window="24h",
+ metadata={"description": "Test batch job"},
+)
+print(batch)
+```
+
+
+
+
+```bash showLineNumbers title="Create Batch Request"
+curl http://localhost:4000/v1/batches \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "input_file_id": "file-abc123",
+ "endpoint": "/v1/chat/completions",
+ "completion_window": "24h",
+ "metadata": {"description": "Test batch job"}
+ }'
+```
+
+
+
+
+## FAQ
+
+### Where are my files written?
+
+When a `target_model_names` is specified, the file is written to the S3 bucket configured in your Bedrock batch model configuration.
+
+### What models are supported?
+
+LiteLLM only supports Bedrock Anthropic Models for Batch API. If you want other bedrock models file an issue [here](https://github.com/BerriAI/litellm/issues/new/choose).
+
+## Further Reading
+
+- [AWS Bedrock Batch Inference Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)
+- [LiteLLM Managed Batches](../proxy/managed_batches)
+- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)
diff --git a/docs/my-website/docs/providers/bedrock_embedding.md b/docs/my-website/docs/providers/bedrock_embedding.md
new file mode 100644
index 00000000000..cd492084711
--- /dev/null
+++ b/docs/my-website/docs/providers/bedrock_embedding.md
@@ -0,0 +1,272 @@
+# Bedrock Embedding
+
+## Supported Embedding Models
+
+| Provider | LiteLLM Route | AWS Documentation |
+|----------|---------------|-------------------|
+| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) |
+| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) |
+| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) |
+
+## Async Invoke Support
+
+LiteLLM supports AWS Bedrock's async-invoke feature for embedding models that require asynchronous processing, particularly useful for large media files (video, audio) or when you need to process embeddings in the background.
+
+### Supported Models
+
+| Provider | Async Invoke Route | Use Case |
+|----------|-------------------|----------|
+| TwelveLabs Marengo | `bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0` | Video, audio, image, and text embeddings |
+
+### Required Parameters
+
+When using async-invoke, you must provide:
+
+| Parameter | Description | Required |
+|-----------|-------------|----------|
+| `output_s3_uri` | S3 URI where the embedding results will be stored | ✅ Yes |
+| `input_type` | Type of input: `"text"`, `"image"`, `"video"`, or `"audio"` | ✅ Yes |
+| `aws_region_name` | AWS region for the request | ✅ Yes |
+
+### Usage
+
+#### Basic Async Invoke
+
+```python
+from litellm import embedding
+
+# Text embedding with async-invoke
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["Hello world from LiteLLM async invoke!"],
+ aws_region_name="us-east-1",
+ input_type="text",
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+print(f"Job submitted! Invocation ARN: {response._hidden_params._invocation_arn}")
+```
+
+#### Video/Audio Embedding
+
+```python
+# Video embedding (requires async-invoke)
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["s3://your-bucket/video.mp4"], # S3 URL for video
+ aws_region_name="us-east-1",
+ input_type="video",
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+print(f"Video embedding job submitted! ARN: {response._hidden_params._invocation_arn}")
+```
+
+#### Image Embedding with Base64
+
+```python
+import base64
+
+# Load and encode image
+with open("image.jpg", "rb") as img_file:
+ img_data = base64.b64encode(img_file.read()).decode('utf-8')
+ img_base64 = f"data:image/jpeg;base64,{img_data}"
+
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=[img_base64],
+ aws_region_name="us-east-1",
+ input_type="image",
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+```
+
+### Retrieving Job Information
+
+#### Getting Job ID and Invocation ARN
+
+The async-invoke response includes the invocation ARN in the hidden parameters:
+
+```python
+response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["Hello world"],
+ aws_region_name="us-east-1",
+ input_type="text",
+ output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+# Access invocation ARN
+invocation_arn = response._hidden_params._invocation_arn
+print(f"Invocation ARN: {invocation_arn}")
+
+# Extract job ID from ARN (last part after the last slash)
+job_id = invocation_arn.split("/")[-1]
+print(f"Job ID: {job_id}")
+```
+
+#### Checking Job Status
+
+Use LiteLLM's `retrieve_batch` function to check if your job is still processing:
+
+```python
+from litellm import retrieve_batch
+
+def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
+ """Check the status of an async invoke job using LiteLLM batch API"""
+ try:
+ response = retrieve_batch(
+ batch_id=invocation_arn,
+ custom_llm_provider="bedrock",
+ aws_region_name=aws_region_name
+ )
+ return response
+ except Exception as e:
+ print(f"Error checking job status: {e}")
+ return None
+
+# Check status
+status = check_async_job_status(invocation_arn, "us-east-1")
+if status:
+ print(f"Job Status: {status.status}")
+ print(f"Output Location: {status.output_file_id}")
+```
+
+**Note:** The actual embedding results are stored in S3. The `output_file_id` from the batch status can be used to locate the results file in your S3 bucket.
+
+### Error Handling
+
+#### Common Errors
+
+| Error | Cause | Solution |
+|-------|-------|----------|
+| `ValueError: output_s3_uri cannot be empty` | Missing S3 output URI | Provide a valid S3 URI |
+| `ValueError: Input type 'video' requires async_invoke route` | Using video/audio without async-invoke | Use `bedrock/async_invoke/` model prefix |
+| `ValueError: input_type is required` | Missing input type parameter | Specify `input_type` parameter |
+
+#### Example Error Handling
+
+```python
+try:
+ response = embedding(
+ model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+ input=["Hello world"],
+ aws_region_name="us-east-1",
+ input_type="text",
+ output_s3_uri="s3://your-bucket/output/" # Required for async-invoke
+ )
+ print("Job submitted successfully!")
+
+except ValueError as e:
+ if "output_s3_uri cannot be empty" in str(e):
+ print("Error: Please provide a valid S3 output URI")
+ elif "requires async_invoke route" in str(e):
+ print("Error: Use async_invoke model for video/audio inputs")
+ else:
+ print(f"Error: {e}")
+except Exception as e:
+ print(f"Unexpected error: {e}")
+```
+
+### Best Practices
+
+1. **Use async-invoke for large files**: Video and audio files are better processed asynchronously
+2. **Use LiteLLM batch API**: Use `retrieve_batch()` instead of direct Bedrock API calls for status checking
+3. **Monitor job status**: Check job status periodically using the batch API to know when results are ready
+4. **Handle errors gracefully**: Implement proper error handling for network issues and job failures
+5. **Set appropriate timeouts**: Consider the processing time for large files
+6. **Use S3 for large inputs**: For video/audio, use S3 URLs instead of base64 encoding
+
+### Limitations
+
+- Async-invoke is currently only supported for TwelveLabs Marengo models
+- Results are stored in S3 and must be retrieved separately using the output file ID
+- Job status checking requires using LiteLLM's `retrieve_batch()` function
+- No built-in polling mechanism in LiteLLM (must implement your own status checking loop)
+
+### API keys
+This can be set as env variables or passed as **params to litellm.embedding()**
+```python
+import os
+os.environ["AWS_ACCESS_KEY_ID"] = "" # Access key
+os.environ["AWS_SECRET_ACCESS_KEY"] = "" # Secret access key
+os.environ["AWS_REGION_NAME"] = "" # us-east-1, us-east-2, us-west-1, us-west-2
+```
+
+## Usage
+### LiteLLM Python SDK
+```python
+from litellm import embedding
+response = embedding(
+ model="bedrock/amazon.titan-embed-text-v1",
+ input=["good morning from litellm"],
+)
+print(response)
+```
+
+### LiteLLM Proxy Server
+
+#### 1. Setup config.yaml
+```yaml
+model_list:
+ - model_name: titan-embed-v1
+ litellm_params:
+ model: bedrock/amazon.titan-embed-text-v1
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-east-1
+ - model_name: titan-embed-v2
+ litellm_params:
+ model: bedrock/amazon.titan-embed-text-v2:0
+ aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+ aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+ aws_region_name: us-east-1
+```
+
+#### 2. Start Proxy
+```bash
+litellm --config /path/to/config.yaml
+```
+
+#### 3. Use with OpenAI Python SDK
+```python
+import openai
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.embeddings.create(
+ input=["good morning from litellm"],
+ model="titan-embed-v1"
+)
+print(response)
+```
+
+#### 4. Use with LiteLLM Python SDK
+```python
+import litellm
+response = litellm.embedding(
+ model="titan-embed-v1", # model alias from config.yaml
+ input=["good morning from litellm"],
+ api_base="http://0.0.0.0:4000",
+ api_key="anything"
+)
+print(response)
+```
+
+## Supported AWS Bedrock Embedding Models
+
+| Model Name | Usage | Supported Additional OpenAI params |
+|----------------------|---------------------------------------------|-----|
+| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
+| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
+| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
+| TwelveLabs Marengo Embed 2.7 | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input)` | Supports multimodal input (text, video, audio, image) |
+| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
+| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
+| Cohere Embed v4 | `embedding(model="bedrock/cohere.embed-v4:0", input=input)` | Supports text and image input, configurable dimensions (256, 512, 1024, 1536), 128k context length |
+
+### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)
+
+### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/bytez.md b/docs/my-website/docs/providers/bytez.md
new file mode 100644
index 00000000000..fc7a684ee8d
--- /dev/null
+++ b/docs/my-website/docs/providers/bytez.md
@@ -0,0 +1,186 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bytez
+
+LiteLLM supports all chat models on [Bytez](https://www.bytez.com)!
+
+That also means multi-modal models are supported 🔥
+
+Tasks supported: `chat`, `image-text-to-text`, `audio-text-to-text`, `video-text-to-text`
+
+## Usage
+
+
+
+
+### API KEYS
+
+```py
+import os
+os.environ["BYTEZ_API_KEY"] = "YOUR_BYTEZ_KEY_GOES_HERE"
+```
+
+### Example Call
+
+```py
+from litellm import completion
+import os
+## set ENV variables
+os.environ["BYTEZ_API_KEY"] = "YOUR_BYTEZ_KEY_GOES_HERE"
+
+response = completion(
+ model="bytez/google/gemma-3-4b-it",
+ messages = [{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+
+
+
+1. Add models to your config.yaml
+
+```yaml
+model_list:
+ - model_name: gemma-3
+ litellm_params:
+ model: bytez/google/gemma-3-4b-it
+ api_key: os.environ/BYTEZ_API_KEY
+```
+
+2. Start the proxy
+
+```bash
+$ BYTEZ_API_KEY=YOUR_BYTEZ_API_KEY_HERE litellm --config /path/to/config.yaml --debug
+```
+
+3. Send Request to LiteLLM Proxy Server
+
+
+
+
+
+```py
+import openai
+client = openai.OpenAI(
+ api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+ base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+)
+
+response = client.chat.completions.create(
+ model="gemma-3",
+ messages = [
+ {
+ "role": "system",
+ "content": "Be a good human!"
+ },
+ {
+ "role": "user",
+ "content": "What do you know about earth?"
+ }
+ ]
+)
+
+print(response)
+```
+
+
+
+
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gemma-3",
+ "messages": [
+ {
+ "role": "system",
+ "content": "Be a good human!"
+ },
+ {
+ "role": "user",
+ "content": "What do you know about earth?"
+ }
+ ],
+}'
+```
+
+
+
+
+
+
+
+
+
+## Automatic Prompt Template Handling
+
+All prompt formatting is handled automatically by our API when you send a messages list to it!
+
+If you wish to use custom formatting, please let us know via either [help@bytez.com](mailto:help@bytez.com) or on our [Discord](https://discord.com/invite/Z723PfCFWf) and we will work to provide it!
+
+## Passing additional params - max_tokens, temperature
+
+See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input)
+
+```py
+# !pip install litellm
+from litellm import completion
+import os
+## set ENV variables
+os.environ["BYTEZ_API_KEY"] = "YOUR_BYTEZ_KEY_HERE"
+
+# bytez gemma-3 call
+response = completion(
+ model="bytez/google/gemma-3-4b-it",
+ messages = [{ "content": "Hello, how are you?","role": "user"}],
+ max_tokens=20,
+ temperature=0.5
+)
+```
+
+**proxy**
+
+```yaml
+model_list:
+ - model_name: gemma-3
+ litellm_params:
+ model: bytez/google/gemma-3-4b-it
+ api_key: os.environ/BYTEZ_API_KEY
+ max_tokens: 20
+ temperature: 0.5
+```
+
+## Passing Bytez-specific params
+
+Any kwarg supported by huggingface we also support! (Provided the model supports it.)
+
+Example `repetition_penalty`
+
+```py
+# !pip install litellm
+from litellm import completion
+import os
+## set ENV variables
+os.environ["BYTEZ_API_KEY"] = "YOUR_BYTEZ_KEY_HERE"
+
+# bytez llama3 call with additional params
+response = completion(
+ model="bytez/google/gemma-3-4b-it",
+ messages = [{ "content": "Hello, how are you?","role": "user"}],
+ repetition_penalty=1.2,
+)
+```
+
+**proxy**
+
+```yaml
+model_list:
+ - model_name: gemma-3
+ litellm_params:
+ model: bytez/google/gemma-3-4b-it
+ api_key: os.environ/BYTEZ_API_KEY
+ repetition_penalty: 1.2
+```
diff --git a/docs/my-website/docs/providers/cometapi.md b/docs/my-website/docs/providers/cometapi.md
new file mode 100644
index 00000000000..1245bacfad4
--- /dev/null
+++ b/docs/my-website/docs/providers/cometapi.md
@@ -0,0 +1,144 @@
+# CometAPI
+LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
+
+## Authentication
+
+To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.
+
+## Usage
+
+Set your CometAPI key as an environment variable and use the completion function:
+
+```python
+import os
+from litellm import completion
+
+# Set API key
+os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+# Define messages
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Method 1: Using environment variable (recommended)
+response = completion(
+ model="cometapi/gpt-5",
+ messages=messages
+)
+
+print(response.choices[0].message.content)
+```
+
+### Alternative Usage - Explicit API Key
+
+You can also pass the API key explicitly:
+
+```python
+import os
+from litellm import completion
+
+# Define messages
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Method 2: Explicitly passing API key
+response = completion(
+ model="cometapi/gpt-4o",
+ messages=messages,
+ api_key="your_comet_api_key_here"
+)
+
+print(response.choices[0].message.content)
+```
+
+## Usage - Streaming
+
+Just set `stream=True` when calling completion:
+
+```python
+import os
+from litellm import completion
+
+os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+response = completion(
+ model="cometapi/gpt-5",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk.choices[0].delta.content or "", end="")
+```
+
+## Usage - Async Streaming
+
+For async streaming, use `acompletion`:
+
+```python
+from litellm import acompletion
+import asyncio, os, traceback
+
+async def completion_call():
+ try:
+ os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+ print("test acompletion + streaming")
+ response = await acompletion(
+ model="cometapi/chatgpt-4o-latest",
+ messages=[{"content": "Hello, how are you?", "role": "user"}],
+ stream=True
+ )
+ print(f"response: {response}")
+ async for chunk in response:
+ print(chunk)
+ except:
+ print(f"error occurred: {traceback.format_exc()}")
+ pass
+
+# Run the async function
+await completion_call()
+```
+
+## CometAPI Models
+
+CometAPI offers access to 500+ AI models through a unified API. Some popular models include:
+
+| Model Name | Function Call |
+|------------|---------------|
+| cometapi/gpt-5 | `completion('cometapi/gpt-5', messages)` |
+| cometapi/gpt-5-mini | `completion('cometapi/gpt-5-mini', messages)` |
+| cometapi/gpt-5-nano | `completion('cometapi/gpt-5-nano', messages)` |
+| cometapi/gpt-oss-20b | `completion('cometapi/gpt-oss-20b', messages)` |
+| cometapi/gpt-oss-120b | `completion('cometapi/gpt-oss-120b', messages)` |
+| cometapi/chatgpt-4o-latest | `completion('cometapi/chatgpt-4o-latest', messages)` |
+
+For a complete list of available models, visit the [CometAPI Models page](https://www.cometapi.com/model/).
+
+## Environment Variables
+
+| Variable | Description | Required |
+|----------|-------------|----------|
+| `COMETAPI_KEY` | Your CometAPI API key | Yes |
+
+## Error Handling
+
+```python
+import os
+from litellm import completion
+
+try:
+ os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
+
+ messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+ response = completion(
+ model="cometapi/gpt-5",
+ messages=messages
+ )
+
+ print(response.choices[0].message.content)
+
+except Exception as e:
+ print(f"Error: {e}")
+```
diff --git a/docs/my-website/docs/providers/compactifai.md b/docs/my-website/docs/providers/compactifai.md
new file mode 100644
index 00000000000..1aa81463071
--- /dev/null
+++ b/docs/my-website/docs/providers/compactifai.md
@@ -0,0 +1,223 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CompactifAI
+https://docs.compactif.ai/
+
+CompactifAI offers highly compressed versions of leading language models, delivering up to **70% lower inference costs**, **4x throughput gains**, and **low-latency inference** with minimal quality loss (under 5%). CompactifAI's OpenAI-compatible API makes integration straightforward, enabling developers to build ultra-efficient, scalable AI applications with superior concurrency and resource efficiency.
+
+| Property | Details |
+|-------|-------|
+| Description | CompactifAI offers compressed versions of leading language models with up to 70% cost reduction and 4x throughput gains |
+| Provider Route on LiteLLM | `compactifai/` (add this prefix to the model name - e.g. `compactifai/cai-llama-3-1-8b-slim`) |
+| Provider Doc | [CompactifAI ↗](https://docs.compactif.ai/) |
+| API Endpoint for Provider | https://api.compactif.ai/v1 |
+| Supported Endpoints | `/chat/completions`, `/completions` |
+
+## Supported OpenAI Parameters
+
+CompactifAI is fully OpenAI-compatible and supports the following parameters:
+
+```
+"stream",
+"stop",
+"temperature",
+"top_p",
+"max_tokens",
+"presence_penalty",
+"frequency_penalty",
+"logit_bias",
+"user",
+"response_format",
+"seed",
+"tools",
+"tool_choice",
+"parallel_tool_calls",
+"extra_headers"
+```
+
+## API Key Setup
+
+CompactifAI API keys are available through AWS Marketplace subscription:
+
+1. Subscribe via [AWS Marketplace](https://aws.amazon.com/marketplace)
+2. Complete subscription verification (24-hour review process)
+3. Access MultiverseIAM dashboard with provided credentials
+4. Retrieve your API key from the dashboard
+
+```python
+import os
+
+os.environ["COMPACTIFAI_API_KEY"] = "your-api-key"
+```
+
+## Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
+
+response = completion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[
+ {"role": "user", "content": "Hello from LiteLLM!"}
+ ],
+)
+print(response)
+```
+
+
+
+
+```yaml
+model_list:
+ - model_name: llama-2-compressed
+ litellm_params:
+ model: compactifai/cai-llama-3-1-8b-slim
+ api_key: os.environ/COMPACTIFAI_API_KEY
+```
+
+
+
+
+## Streaming
+
+```python
+from litellm import completion
+import os
+
+os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
+
+response = completion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[
+ {"role": "user", "content": "Write a short story"}
+ ],
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+## Advanced Usage
+
+### Custom Parameters
+
+```python
+from litellm import completion
+
+response = completion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[{"role": "user", "content": "Explain quantum computing"}],
+ temperature=0.7,
+ max_tokens=500,
+ top_p=0.9,
+ stop=["Human:", "AI:"]
+)
+```
+
+### Function Calling
+
+CompactifAI supports OpenAI-compatible function calling:
+
+```python
+from litellm import completion
+
+functions = [
+ {
+ "name": "get_weather",
+ "description": "Get current weather information",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state"
+ }
+ },
+ "required": ["location"]
+ }
+ }
+]
+
+response = completion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[{"role": "user", "content": "What's the weather in San Francisco?"}],
+ tools=[{"type": "function", "function": f} for f in functions],
+ tool_choice="auto"
+)
+```
+
+### Async Usage
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def async_call():
+ response = await acompletion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[{"role": "user", "content": "Hello async world!"}]
+ )
+ return response
+
+# Run async function
+response = asyncio.run(async_call())
+print(response)
+```
+
+## Available Models
+
+CompactifAI offers compressed versions of popular models. Use the `/models` endpoint to get the latest list:
+
+```python
+import httpx
+
+headers = {"Authorization": f"Bearer {your_api_key}"}
+response = httpx.get("https://api.compactif.ai/v1/models", headers=headers)
+models = response.json()
+```
+
+Common model formats:
+- `compactifai/cai-llama-3-1-8b-slim`
+- `compactifai/mistral-7b-compressed`
+- `compactifai/codellama-7b-compressed`
+
+## Benefits
+
+- **Cost Efficient**: Up to 70% lower inference costs compared to standard models
+- **High Performance**: 4x throughput gains with minimal quality loss (under 5%)
+- **Low Latency**: Optimized for fast response times
+- **Drop-in Replacement**: Full OpenAI API compatibility
+- **Scalable**: Superior concurrency and resource efficiency
+
+## Error Handling
+
+CompactifAI returns standard OpenAI-compatible error responses:
+
+```python
+from litellm import completion
+from litellm.exceptions import AuthenticationError, RateLimitError
+
+try:
+ response = completion(
+ model="compactifai/cai-llama-3-1-8b-slim",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+except AuthenticationError:
+ print("Invalid API key")
+except RateLimitError:
+ print("Rate limit exceeded")
+```
+
+## Support
+
+- Documentation: https://docs.compactif.ai/
+- LinkedIn: [MultiverseComputing](https://www.linkedin.com/company/multiversecomputing)
+- Analysis: [Artificial Analysis Provider Comparison](https://artificialanalysis.ai/providers/compactifai)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/custom_llm_server.md b/docs/my-website/docs/providers/custom_llm_server.md
index 2adb6a67cf8..61099d1a358 100644
--- a/docs/my-website/docs/providers/custom_llm_server.md
+++ b/docs/my-website/docs/providers/custom_llm_server.md
@@ -1,3 +1,7 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
# Custom API Server (Custom Format)
Call your custom torch-serve / internal LLM APIs via LiteLLM
@@ -8,9 +12,17 @@ Call your custom torch-serve / internal LLM APIs via LiteLLM
- For modifying incoming/outgoing calls on proxy, [go here](../proxy/call_hooks.md)
:::
+Supported Routes:
+- `/v1/chat/completions` -> `litellm.acompletion`
+- `/v1/completions` -> `litellm.atext_completion`
+- `/v1/embeddings` -> `litellm.aembedding`
+- `/v1/images/generations` -> `litellm.aimage_generation`
+
+- `/v1/messages` -> `litellm.acompletion`
+
## Quick Start
-```python
+```python showLineNumbers
import litellm
from litellm import CustomLLM, completion, get_llm_provider
@@ -251,6 +263,102 @@ Expected Response
}
```
+## Anthropic `/v1/messages`
+
+- Write the integration for .acompletion
+- litellm will transform it to /v1/messages
+
+1. Setup your `custom_handler.py` file
+
+```python
+import litellm
+from litellm import CustomLLM, completion, get_llm_provider
+
+
+class MyCustomLLM(CustomLLM):
+ async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
+ return litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello world"}],
+ mock_response="Hi!",
+ ) # type: ignore
+
+
+my_custom_llm = MyCustomLLM()
+```
+
+2. Add to `config.yaml`
+
+In the config below, we pass
+
+python_filename: `custom_handler.py`
+custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1
+
+custom_handler: `custom_handler.my_custom_llm`
+
+```yaml
+model_list:
+ - model_name: "test-model"
+ litellm_params:
+ model: "openai/text-embedding-ada-002"
+ - model_name: "my-custom-model"
+ litellm_params:
+ model: "my-custom-llm/my-model"
+
+litellm_settings:
+ custom_provider_map:
+ - {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
+-H 'anthropic-version: 2023-06-01' \
+-H 'content-type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model": "my-custom-model",
+ "max_tokens": 1024,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What are the key findings in this document 12?"
+ }]
+ }]
+}'
+```
+
+Expected Response
+
+```json
+{
+ "id": "chatcmpl-Bm4qEp4h4vCe7Zi4Gud1MAxTWgibO",
+ "type": "message",
+ "role": "assistant",
+ "model": "gpt-3.5-turbo-0125",
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 18,
+ "output_tokens": 44
+ },
+ "content": [
+ {
+ "type": "text",
+ "text": "Without the specific document being provided, it is not possible to determine the key findings within it. If you can provide the content or a summary of document 12, I would be happy to help identify the key findings."
+ }
+ ],
+ "stop_reason": "end_turn"
+}
+```
+
+
## Additional Parameters
Additional parameters are passed inside `optional_params` key in the `completion` or `image_generation` function.
diff --git a/docs/my-website/docs/providers/dashscope.md b/docs/my-website/docs/providers/dashscope.md
new file mode 100644
index 00000000000..565776d6c4c
--- /dev/null
+++ b/docs/my-website/docs/providers/dashscope.md
@@ -0,0 +1,67 @@
+# Dashscope (Qwen API)
+https://dashscope.console.aliyun.com/
+
+**We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**
+
+## API Key
+```python
+# env variable
+os.environ['DASHSCOPE_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['DASHSCOPE_API_KEY'] = ""
+response = completion(
+ model="dashscope/qwen-turbo",
+ messages=[
+ {"role": "user", "content": "hello from litellm"}
+ ],
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['DASHSCOPE_API_KEY'] = ""
+response = completion(
+ model="dashscope/qwen-turbo",
+ messages=[
+ {"role": "user", "content": "hello from litellm"}
+ ],
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+
+## Supported Models - ALL Qwen Models Supported!
+We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests
+
+
+[DashScope Model List](https://help.aliyun.com/zh/model-studio/compatibility-of-openai-with-dashscope?spm=a2c4g.11186623.help-menu-2400256.d_2_8_0.1efd516e2tTXBn&scm=20140722.H_2833609._.OR_help-T_cn~zh-V_1#7f9c78ae99pwz)
+
+| Model Name | Function Call |
+|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| qwen-turbo | `completion(model="dashscope/qwen-turbo", messages)` |
+| qwen-plus | `completion(model="dashscope/qwen-plus", messages)` |
+| qwen-max | `completion(model="dashscope/qwen-max", messages)` |
+| qwen-turbo-latest | `completion(model="dashscope/qwen-turbo-latest", messages)` |
+| qwen-plus-latest | `completion(model="dashscope/qwen-plus-latest", messages)` |
+| qwen-max-latest | `completion(model="dashscope/qwen-max-latest", messages)` |
+| qwen-vl-plus | `completion(model="dashscope/qwen-vl-plus", messages)` |
+| qwen-vl-max | `completion(model="dashscope/qwen-vl-max", messages)` |
+| qwq-32b | `completion(model="dashscope/qwq-32b", messages)` |
+| qwq-32b-preview | `completion(model="dashscope/qwq-32b-preview", messages)` |
+| qwen3-235b-a22b | `completion(model="dashscope/qwen3-235b-a22b", messages)` |
+| qwen3-32b | `completion(model="dashscope/qwen3-32b", messages)` |
+| qwen3-30b-a3b | `completion(model="dashscope/qwen3-30b-a3b", messages)` |
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/databricks.md b/docs/my-website/docs/providers/databricks.md
index 8631cbfdad9..921b06a17b7 100644
--- a/docs/my-website/docs/providers/databricks.md
+++ b/docs/my-website/docs/providers/databricks.md
@@ -282,6 +282,11 @@ ModelResponse(
)
```
+### Citations
+
+Anthropic models served through Databricks can return citation metadata. LiteLLM
+exposes these via `response.choices[0].message.provider_specific_fields["citations"]`.
+
### Pass `thinking` to Anthropic models
You can also pass the `thinking` parameter to Anthropic models.
diff --git a/docs/my-website/docs/providers/datarobot.md b/docs/my-website/docs/providers/datarobot.md
new file mode 100644
index 00000000000..3f4a0f71ac4
--- /dev/null
+++ b/docs/my-website/docs/providers/datarobot.md
@@ -0,0 +1,43 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# DataRobot
+LiteLLM supports all models from [DataRobot](https://datarobot.com). Select `datarobot` as the provider to route your request through the `datarobot` OpenAI-compatible endpoint using the upstream [official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md).
+
+## Usage
+
+### Environment variables
+```python
+import os
+from litellm import completion
+os.environ["DATAROBOT_API_KEY"] = ""
+os.environ["DATAROBOT_API_BASE"] = "" # [OPTIONAL] defaults to https://app.datarobot.com
+
+response = completion(
+ model="datarobot/openai/gpt-4o-mini",
+ messages=messages,
+ )
+
+
+### Completion
+```python
+import litellm
+import os
+
+response = litellm.completion(
+ model="datarobot/openai/gpt-4o-mini", # add `datarobot/` prefix to model so litellm knows to route through DataRobot
+ messages=[
+ {
+ "role": "user",
+ "content": "Hey, how's it going?",
+ }
+ ],
+)
+print(response)
+```
+
+## DataRobot completion models
+
+🚨 LiteLLM supports _all_ DataRobot LLM gateway models. To get a list for your installation and user account, send the following CURL command:
+`curl -X GET -H "Authorization: Bearer $DATAROBOT_API_TOKEN" "$DATAROBOT_ENDPOINT/genai/llmgw/catalog/" | jq | grep 'model":'DATAROBOT_ENDPOINT/genai/llmgw/catalog/`
+
diff --git a/docs/my-website/docs/providers/deepinfra.md b/docs/my-website/docs/providers/deepinfra.md
index 1360117445f..ddf6122cac8 100644
--- a/docs/my-website/docs/providers/deepinfra.md
+++ b/docs/my-website/docs/providers/deepinfra.md
@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
# DeepInfra
https://deepinfra.com/
@@ -7,6 +10,11 @@ https://deepinfra.com/
:::
+## Table of Contents
+
+- [API Key](#api-key)
+- [Chat Models](#chat-models)
+- [Rerank Endpoint](#rerank-endpoint)
## API Key
```python
@@ -53,3 +61,135 @@ for chunk in response:
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |
+
+## Rerank Endpoint
+
+LiteLLM provides a Cohere API compatible `/rerank` endpoint for DeepInfra rerank models.
+
+### Supported Rerank Models
+
+| Model Name | Description |
+|------------|-------------|
+| `deepinfra/Qwen/Qwen3-Reranker-0.6B` | Lightweight rerank model (0.6B parameters) |
+| `deepinfra/Qwen/Qwen3-Reranker-4B` | Medium rerank model (4B parameters) |
+| `deepinfra/Qwen/Qwen3-Reranker-8B` | Large rerank model (8B parameters) |
+
+### Usage - LiteLLM Python SDK
+
+
+
+
+```python
+from litellm import rerank
+import os
+
+os.environ["DEEPINFRA_API_KEY"] = "your-api-key"
+
+response = rerank(
+ model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
+ query="What is the capital of France?",
+ documents=[
+ "Paris is the capital of France.",
+ "London is the capital of the United Kingdom.",
+ "Berlin is the capital of Germany.",
+ "Madrid is the capital of Spain.",
+ "Rome is the capital of Italy."
+ ]
+)
+print(response)
+```
+
+
+
+
+1. Add to config.yaml
+```yaml
+model_list:
+ - model_name: Qwen/Qwen3-Reranker-0.6B
+ litellm_params:
+ model: deepinfra/Qwen/Qwen3-Reranker-0.6B
+ api_key: os.environ/DEEPINFRA_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000/
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/rerank' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "model": "Qwen/Qwen3-Reranker-0.6B",
+ "query": "What is the capital of France?",
+ "documents": [
+ "Paris is the capital of France.",
+ "London is the capital of the United Kingdom.",
+ "Berlin is the capital of Germany.",
+ "Madrid is the capital of Spain.",
+ "Rome is the capital of Italy."
+ ]
+}'
+```
+
+
+
+
+### Supported Cohere Rerank API Params
+
+| Param | Type | Description |
+| ------------------ | ----------- | ----------------------------------------------- |
+| `query` | `str` | The query to rerank the documents against |
+| `documents` | `list[str]` | The documents to rerank |
+
+
+### Provider-specific parameters
+Pass any deepinfra specific parameters as a keyword argument to the rerank function, e.g.
+
+```
+response = rerank(
+ model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
+ query="What is the capital of France?",
+ documents=[
+ "Paris is the capital of France.",
+ "London is the capital of the United Kingdom.",
+ "Berlin is the capital of Germany.",
+ "Madrid is the capital of Spain.",
+ "Rome is the capital of Italy."
+ ],
+ my_custom_param="my_custom_value", # any other deepinfra specific parameters
+)
+```
+
+### Response Format
+
+```json
+{
+ "id": "request-id",
+ "results": [
+ {
+ "index": 0,
+ "relevance_score": 0.9975274205207825
+ },
+ {
+ "index": 1,
+ "relevance_score": 0.011687257327139378
+ }
+ ],
+ "meta": {
+ "billed_units": {
+ "total_tokens": 427
+ },
+ "tokens": {
+ "input_tokens": 427,
+ "output_tokens": 0
+ }
+ }
+}
+```
diff --git a/docs/my-website/docs/providers/elevenlabs.md b/docs/my-website/docs/providers/elevenlabs.md
new file mode 100644
index 00000000000..e80ea534f55
--- /dev/null
+++ b/docs/my-website/docs/providers/elevenlabs.md
@@ -0,0 +1,231 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# ElevenLabs
+
+ElevenLabs provides high-quality AI voice technology, including speech-to-text capabilities through their transcription API.
+
+| Property | Details |
+|----------|---------|
+| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription capabilities that support multiple languages and speaker diarization. |
+| Provider Route on LiteLLM | `elevenlabs/` |
+| Provider Doc | [ElevenLabs API ↗](https://elevenlabs.io/docs/api-reference) |
+| Supported Endpoints | `/audio/transcriptions` |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+
+
+
+```python showLineNumbers title="Basic audio transcription with ElevenLabs"
+import litellm
+
+# Transcribe audio file
+with open("audio.mp3", "rb") as audio_file:
+ response = litellm.transcription(
+ model="elevenlabs/scribe_v1",
+ file=audio_file,
+ api_key="your-elevenlabs-api-key" # or set ELEVENLABS_API_KEY env var
+ )
+
+print(response.text)
+```
+
+
+
+
+
+```python showLineNumbers title="Audio transcription with advanced features"
+import litellm
+
+# Transcribe with speaker diarization and language specification
+with open("audio.wav", "rb") as audio_file:
+ response = litellm.transcription(
+ model="elevenlabs/scribe_v1",
+ file=audio_file,
+ language="en", # Language hint (maps to language_code)
+ temperature=0.3, # Control randomness in transcription
+ diarize=True, # Enable speaker diarization
+ api_key="your-elevenlabs-api-key"
+ )
+
+print(f"Transcription: {response.text}")
+print(f"Language: {response.language}")
+
+# Access word-level timestamps if available
+if hasattr(response, 'words') and response.words:
+ for word_info in response.words:
+ print(f"Word: {word_info['word']}, Start: {word_info['start']}, End: {word_info['end']}")
+```
+
+
+
+
+
+```python showLineNumbers title="Async audio transcription"
+import litellm
+import asyncio
+
+async def transcribe_audio():
+ with open("audio.mp3", "rb") as audio_file:
+ response = await litellm.atranscription(
+ model="elevenlabs/scribe_v1",
+ file=audio_file,
+ api_key="your-elevenlabs-api-key"
+ )
+
+ return response.text
+
+# Run async transcription
+result = asyncio.run(transcribe_audio())
+print(result)
+```
+
+
+
+
+### LiteLLM Proxy
+
+#### 1. Configure your proxy
+
+
+
+
+```yaml showLineNumbers title="ElevenLabs configuration in config.yaml"
+model_list:
+ - model_name: elevenlabs-transcription
+ litellm_params:
+ model: elevenlabs/scribe_v1
+ api_key: os.environ/ELEVENLABS_API_KEY
+
+general_settings:
+ master_key: your-master-key
+```
+
+
+
+
+
+```bash showLineNumbers title="Required environment variables"
+export ELEVENLABS_API_KEY="your-elevenlabs-api-key"
+export LITELLM_MASTER_KEY="your-master-key"
+```
+
+
+
+
+#### 2. Start the proxy
+
+```bash showLineNumbers title="Start LiteLLM proxy server"
+litellm --config config.yaml
+
+# Proxy will be available at http://localhost:4000
+```
+
+#### 3. Make transcription requests
+
+
+
+
+```bash showLineNumbers title="Audio transcription with curl"
+curl http://localhost:4000/v1/audio/transcriptions \
+ -H "Authorization: Bearer $LITELLM_API_KEY" \
+ -H "Content-Type: multipart/form-data" \
+ -F file="@audio.mp3" \
+ -F model="elevenlabs-transcription" \
+ -F language="en" \
+ -F temperature="0.3"
+```
+
+
+
+
+
+```python showLineNumbers title="Using OpenAI SDK with LiteLLM proxy"
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000",
+ api_key="your-litellm-api-key"
+)
+
+# Transcribe audio file
+with open("audio.mp3", "rb") as audio_file:
+ response = client.audio.transcriptions.create(
+ model="elevenlabs-transcription",
+ file=audio_file,
+ language="en",
+ temperature=0.3,
+ # ElevenLabs-specific parameters
+ diarize=True,
+ speaker_boost=True,
+ custom_vocabulary="technical,AI,machine learning"
+ )
+
+print(response.text)
+```
+
+
+
+
+
+```javascript showLineNumbers title="Audio transcription with JavaScript"
+import OpenAI from 'openai';
+import fs from 'fs';
+
+const openai = new OpenAI({
+ baseURL: 'http://localhost:4000',
+ apiKey: 'your-litellm-api-key'
+});
+
+async function transcribeAudio() {
+ const response = await openai.audio.transcriptions.create({
+ file: fs.createReadStream('audio.mp3'),
+ model: 'elevenlabs-transcription',
+ language: 'en',
+ temperature: 0.3,
+ diarize: true,
+ speaker_boost: true
+ });
+
+ console.log(response.text);
+}
+
+transcribeAudio();
+```
+
+
+
+
+## Response Format
+
+ElevenLabs returns transcription responses in OpenAI-compatible format:
+
+```json showLineNumbers title="Example transcription response"
+{
+ "text": "Hello, this is a sample transcription with multiple speakers.",
+ "task": "transcribe",
+ "language": "en",
+ "words": [
+ {
+ "word": "Hello",
+ "start": 0.0,
+ "end": 0.5
+ },
+ {
+ "word": "this",
+ "start": 0.5,
+ "end": 0.8
+ }
+ ]
+}
+```
+
+### Common Issues
+
+1. **Invalid API Key**: Ensure `ELEVENLABS_API_KEY` is set correctly
+
+
diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md
index 80f68679105..40d64656528 100644
--- a/docs/my-website/docs/providers/gemini.md
+++ b/docs/my-website/docs/providers/gemini.md
@@ -51,6 +51,7 @@ response = completion(
- frequency_penalty
- modalities
- reasoning_content
+- audio (for TTS models only)
**Anthropic Params**
- thinking (used to set max budget tokens across anthropic/gemini models)
@@ -63,10 +64,13 @@ response = completion(
LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
+Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests.
+
**Mapping**
| reasoning_effort | thinking |
| ---------------- | -------- |
+| "disable" | "budget_tokens": 0 |
| "low" | "budget_tokens": 1024 |
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
@@ -198,6 +202,119 @@ curl http://0.0.0.0:4000/v1/chat/completions \
+## Text-to-Speech (TTS) Audio Output
+
+:::info
+
+LiteLLM supports Gemini TTS models that can generate audio responses using the OpenAI-compatible `audio` parameter format.
+
+:::
+
+### Supported Models
+
+LiteLLM supports Gemini TTS models with audio capabilities (e.g. `gemini-2.5-flash-preview-tts` and `gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
+
+### Limitations
+
+:::warning
+
+**Important Limitations**:
+- Gemini TTS models only support the `pcm16` audio format
+- **Streaming support has not been added** to TTS models yet
+- The `modalities` parameter must be set to `['audio']` for TTS requests
+
+:::
+
+### Quick Start
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ['GEMINI_API_KEY'] = "your-api-key"
+
+response = completion(
+ model="gemini/gemini-2.5-flash-preview-tts",
+ messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+ modalities=["audio"], # Required for TTS models
+ audio={
+ "voice": "Kore",
+ "format": "pcm16" # Required: must be "pcm16"
+ }
+)
+
+print(response)
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: gemini-tts-flash
+ litellm_params:
+ model: gemini/gemini-2.5-flash-preview-tts
+ api_key: os.environ/GEMINI_API_KEY
+ - model_name: gemini-tts-pro
+ litellm_params:
+ model: gemini/gemini-2.5-pro-preview-tts
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Make TTS request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "gemini-tts-flash",
+ "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+ "modalities": ["audio"],
+ "audio": {
+ "voice": "Kore",
+ "format": "pcm16"
+ }
+ }'
+```
+
+
+
+
+### Advanced Usage
+
+You can combine TTS with other Gemini features:
+
+```python
+response = completion(
+ model="gemini/gemini-2.5-pro-preview-tts",
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+ {"role": "user", "content": "Explain quantum computing in simple terms"}
+ ],
+ modalities=["audio"],
+ audio={
+ "voice": "Charon",
+ "format": "pcm16"
+ },
+ temperature=0.7,
+ max_tokens=150
+)
+```
+
+For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
+
## Passing Gemini Specific Params
### Response schema
LiteLLM supports sending `response_schema` as a param for Gemini-1.5-Pro on Google AI Studio.
@@ -643,6 +760,66 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+### URL Context
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = ".."
+
+# 👇 ADD URL CONTEXT
+tools = [{"urlContext": {}}]
+
+response = completion(
+ model="gemini/gemini-2.0-flash",
+ messages=[{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+ tools=tools,
+)
+
+print(response)
+
+# Access URL context metadata
+url_context_metadata = response.model_extra['vertex_ai_url_context_metadata']
+urlMetadata = url_context_metadata[0]['urlMetadata'][0]
+print(f"Retrieved URL: {urlMetadata['retrievedUrl']}")
+print(f"Retrieval Status: {urlMetadata['urlRetrievalStatus']}")
+```
+
+
+
+
+1. Setup config.yaml
+```yaml
+model_list:
+ - model_name: gemini-2.0-flash
+ litellm_params:
+ model: gemini/gemini-2.0-flash
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start Proxy
+```bash
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request!
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "gemini-2.0-flash",
+ "messages": [{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+ "tools": [{"urlContext": {}}]
+ }'
+```
+
+
+
### Google Search Retrieval
@@ -1022,6 +1199,10 @@ response = litellm.completion(
| gemini-2.0-flash | `completion(model='gemini/gemini-2.0-flash', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-2.0-flash-exp | `completion(model='gemini/gemini-2.0-flash-exp', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-2.0-flash-lite-preview-02-05 | `completion(model='gemini/gemini-2.0-flash-lite-preview-02-05', messages)` | `os.environ['GEMINI_API_KEY']` |
+| gemini-2.5-flash-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` |
+| gemini-2.5-flash-lite-preview-09-2025 | `completion(model='gemini/gemini-2.5-flash-lite-preview-09-2025', messages)` | `os.environ['GEMINI_API_KEY']` |
+| gemini-flash-latest | `completion(model='gemini/gemini-flash-latest', messages)` | `os.environ['GEMINI_API_KEY']` |
+| gemini-flash-lite-latest | `completion(model='gemini/gemini-flash-lite-latest', messages)` | `os.environ['GEMINI_API_KEY']` |
@@ -1042,12 +1223,38 @@ Use Google AI Studio context caching is supported by
in your message content block.
+### Custom TTL Support
+
+You can now specify a custom Time-To-Live (TTL) for your cached content using the `ttl` parameter:
+
+```bash
+{
+ {
+ "role": "system",
+ "content": ...,
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "3600s" # 👈 Cache for 1 hour
+ }
+ },
+ ...
+}
+```
+
+**TTL Format Requirements:**
+- Must be a string ending with 's' for seconds
+- Must contain a positive number (can be decimal)
+- Examples: `"3600s"` (1 hour), `"7200s"` (2 hours), `"1800s"` (30 minutes), `"1.5s"` (1.5 seconds)
+
+**TTL Behavior:**
+- If multiple cached messages have different TTLs, the first valid TTL encountered will be used
+- Invalid TTL formats are ignored and the cache will use Google's default expiration time
+- If no TTL is specified, Google's default cache expiration (approximately 1 hour) applies
+
### Architecture Diagram
-
-
**Notes:**
- [Relevant code](https://github.com/BerriAI/litellm/blob/main/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py#L255)
@@ -1056,7 +1263,6 @@ in your message content block.
- If multiple non-continuous blocks contain `cache_control` - the first continuous block will be used. (sent to `/cachedContent` in the [Gemini format](https://ai.google.dev/api/caching#cache_create-SHELL))
-
- The raw request to Gemini's `/generateContent` endpoint looks like this:
```bash
@@ -1076,7 +1282,6 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
```
-
### Example Usage
@@ -1116,6 +1321,48 @@ for _ in range(2):
print(resp.usage) # 👈 2nd usage block will be less, since cached tokens used
```
+
+
+
+```python
+from litellm import completion
+
+# Cache for 2 hours (7200 seconds)
+resp = completion(
+ model="gemini/gemini-1.5-pro",
+ messages=[
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Here is the full text of a complex legal agreement" * 4000,
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "7200s" # 👈 Cache for 2 hours
+ },
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What are the key terms and conditions in this agreement?",
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "3600s" # 👈 This TTL will be ignored (first one is used)
+ },
+ }
+ ],
+ }
+ ]
+)
+
+print(resp.usage)
+```
+
@@ -1173,6 +1420,44 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gemini-1.5-pro",
+ "messages": [
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Here is the full text of a complex legal agreement" * 4000,
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "7200s"
+ }
+ }
+ ]
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What are the key terms and conditions in this agreement?",
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "3600s"
+ }
+ }
+ ]
+ }
+ ]
+}'
+```
+
```python
@@ -1205,6 +1490,40 @@ response = await client.chat.completions.create(
```
+
+
+
+```python
+import openai
+client = openai.AsyncOpenAI(
+ api_key="anything", # litellm proxy api key
+ base_url="http://0.0.0.0:4000" # litellm proxy base url
+)
+
+response = await client.chat.completions.create(
+ model="gemini-1.5-pro",
+ messages=[
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Here is the full text of a complex legal agreement" * 4000,
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "7200s" # Cache for 2 hours
+ }
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": "what are the key terms and conditions in this agreement?",
+ },
+ ]
+)
+```
+
diff --git a/docs/my-website/docs/providers/github.md b/docs/my-website/docs/providers/github.md
index 7594b6af4c0..b9e525ef5c1 100644
--- a/docs/my-website/docs/providers/github.md
+++ b/docs/my-website/docs/providers/github.md
@@ -151,13 +151,13 @@ We support ALL Github models, just set `github/` as a prefix when sending comple
| Model Name | Usage |
|--------------------|---------------------------------------------------------|
-| llama-3.1-8b-instant | `completion(model="github/llama-3.1-8b-instant", messages)` |
-| llama-3.1-70b-versatile | `completion(model="github/llama-3.1-70b-versatile", messages)` |
+| llama-3.1-8b-Instant | `completion(model="github/Llama-3.1-8b-Instant", messages)` |
+| Llama-3.1-70b-Versatile | `completion(model="github/Llama-3.1-70b-Versatile", messages)` |
| Llama-3.2-11B-Vision-Instruct | `completion(model="github/Llama-3.2-11B-Vision-Instruct", messages)` |
-| llama3-70b-8192 | `completion(model="github/llama3-70b-8192", messages)` |
-| llama2-70b-4096 | `completion(model="github/llama2-70b-4096", messages)` |
-| mixtral-8x7b-32768 | `completion(model="github/mixtral-8x7b-32768", messages)` |
-| gemma-7b-it | `completion(model="github/gemma-7b-it", messages)` |
+| Llama3-70b-8192 | `completion(model="github/Llama3-70b-8192", messages)` |
+| Llama2-70b-4096 | `completion(model="github/Llama2-70b-4096", messages)` |
+| Mixtral-8x7b-32768 | `completion(model="github/Mixtral-8x7b-32768", messages)` |
+| Phi-4 | `completion(model="github/Phi-4", messages)` |
## Github - Tool / Function Calling Example
diff --git a/docs/my-website/docs/providers/github_copilot.md b/docs/my-website/docs/providers/github_copilot.md
new file mode 100644
index 00000000000..2ebe6eacb1c
--- /dev/null
+++ b/docs/my-website/docs/providers/github_copilot.md
@@ -0,0 +1,186 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# GitHub Copilot
+
+https://docs.github.com/en/copilot
+
+:::tip
+
+**We support GitHub Copilot Chat API with automatic authentication handling**
+
+:::
+
+| Property | Details |
+|-------|-------|
+| Description | GitHub Copilot Chat API provides access to GitHub's AI-powered coding assistant. |
+| Provider Route on LiteLLM | `github_copilot/` |
+| Supported Endpoints | `/chat/completions` |
+| API Reference | [GitHub Copilot docs](https://docs.github.com/en/copilot) |
+
+## Authentication
+
+GitHub Copilot uses OAuth device flow for authentication. On first use, you'll be prompted to authenticate via GitHub:
+
+1. LiteLLM will display a device code and verification URL
+2. Visit the URL and enter the code to authenticate
+3. Your credentials will be stored locally for future use
+
+## Usage - LiteLLM Python SDK
+
+### Chat Completion
+
+```python showLineNumbers title="GitHub Copilot Chat Completion"
+from litellm import completion
+
+response = completion(
+ model="github_copilot/gpt-4",
+ messages=[{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}],
+ extra_headers={
+ "editor-version": "vscode/1.85.1",
+ "Copilot-Integration-Id": "vscode-chat"
+ }
+)
+print(response)
+```
+
+```python showLineNumbers title="GitHub Copilot Chat Completion - Streaming"
+from litellm import completion
+
+stream = completion(
+ model="github_copilot/gpt-4",
+ messages=[{"role": "user", "content": "Explain async/await in Python"}],
+ stream=True,
+ extra_headers={
+ "editor-version": "vscode/1.85.1",
+ "Copilot-Integration-Id": "vscode-chat"
+ }
+)
+
+for chunk in stream:
+ if chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: github_copilot/gpt-4
+ litellm_params:
+ model: github_copilot/gpt-4
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+
+
+
+```python showLineNumbers title="GitHub Copilot via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+ model="github_copilot/gpt-4",
+ messages=[{"role": "user", "content": "How do I optimize this SQL query?"}],
+ extra_headers={
+ "editor-version": "vscode/1.85.1",
+ "Copilot-Integration-Id": "vscode-chat"
+ }
+)
+
+print(response.choices[0].message.content)
+```
+
+
+
+
+
+```python showLineNumbers title="GitHub Copilot via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+ model="litellm_proxy/github_copilot/gpt-4",
+ messages=[{"role": "user", "content": "Review this code for bugs"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key",
+ extra_headers={
+ "editor-version": "vscode/1.85.1",
+ "Copilot-Integration-Id": "vscode-chat"
+ }
+)
+
+print(response.choices[0].message.content)
+```
+
+
+
+
+
+```bash showLineNumbers title="GitHub Copilot via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -H "editor-version: vscode/1.85.1" \
+ -H "Copilot-Integration-Id: vscode-chat" \
+ -d '{
+ "model": "github_copilot/gpt-4",
+ "messages": [{"role": "user", "content": "Explain this error message"}]
+ }'
+```
+
+
+
+
+## Getting Started
+
+1. Ensure you have GitHub Copilot access (paid GitHub subscription required)
+2. Run your first LiteLLM request - you'll be prompted to authenticate
+3. Follow the device flow authentication process
+4. Start making requests to GitHub Copilot through LiteLLM
+
+## Configuration
+
+### Environment Variables
+
+You can customize token storage locations:
+
+```bash showLineNumbers title="Environment Variables"
+# Optional: Custom token directory
+export GITHUB_COPILOT_TOKEN_DIR="~/.config/litellm/github_copilot"
+
+# Optional: Custom access token file name
+export GITHUB_COPILOT_ACCESS_TOKEN_FILE="access-token"
+
+# Optional: Custom API key file name
+export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
+```
+
+### Headers
+
+GitHub Copilot supports various editor-specific headers:
+
+```python showLineNumbers title="Common Headers"
+extra_headers = {
+ "editor-version": "vscode/1.85.1", # Editor version
+ "editor-plugin-version": "copilot/1.155.0", # Plugin version
+ "Copilot-Integration-Id": "vscode-chat", # Integration ID
+ "user-agent": "GithubCopilot/1.155.0" # User agent
+}
+```
+
diff --git a/docs/my-website/docs/providers/google_ai_studio/image_gen.md b/docs/my-website/docs/providers/google_ai_studio/image_gen.md
new file mode 100644
index 00000000000..31b1766e450
--- /dev/null
+++ b/docs/my-website/docs/providers/google_ai_studio/image_gen.md
@@ -0,0 +1,214 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Google AI Studio Image Generation
+
+Google AI Studio provides powerful image generation capabilities using Google's Imagen models to create high-quality images from text descriptions.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Google AI Studio Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. |
+| Provider Route on LiteLLM | `gemini/` |
+| Provider Doc | [Google AI Studio Image Generation ↗](https://ai.google.dev/gemini-api/docs/imagen) |
+| Supported Operations | [`/images/generations`](#image-generation) |
+
+## Setup
+
+### API Key
+
+```python showLineNumbers
+# Set your Google AI Studio API key
+import os
+os.environ["GEMINI_API_KEY"] = "your-api-key-here"
+```
+
+Get your API key from [Google AI Studio](https://aistudio.google.com/app/apikey).
+
+## Image Generation
+
+### Usage - LiteLLM Python SDK
+
+
+
+
+```python showLineNumbers title="Basic Image Generation"
+import litellm
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key-here"
+
+# Generate a single image
+response = litellm.image_generation(
+ model="gemini/imagen-4.0-generate-001",
+ prompt="A cute baby sea otter swimming in crystal clear water"
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```python showLineNumbers title="Async Image Generation"
+import litellm
+import asyncio
+import os
+
+async def generate_image():
+ # Set your API key
+ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
+
+ # Generate image asynchronously
+ response = await litellm.aimage_generation(
+ model="gemini/imagen-4.0-generate-001",
+ prompt="A beautiful sunset over mountains with vibrant colors",
+ n=1,
+ )
+
+ print(response.data[0].url)
+ return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+
+
+
+
+```python showLineNumbers title="Advanced Image Generation with Parameters"
+import litellm
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key-here"
+
+# Generate image with additional parameters
+response = litellm.image_generation(
+ model="gemini/imagen-4.0-generate-001",
+ prompt="A futuristic cityscape at night with neon lights",
+ n=1,
+ size="1024x1024",
+ quality="standard",
+ response_format="url"
+)
+
+for image in response.data:
+ print(f"Generated image URL: {image.url}")
+```
+
+
+
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Google AI Studio Image Generation Configuration"
+model_list:
+ - model_name: google-imagen
+ litellm_params:
+ model: gemini/imagen-4.0-generate-001
+ api_key: os.environ/GEMINI_API_KEY
+ model_info:
+ mode: image_generation
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make requests with OpenAI Python SDK
+
+
+
+
+```python showLineNumbers title="Google AI Studio Image Generation via Proxy - OpenAI SDK"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="sk-1234" # Your proxy API key
+)
+
+# Generate image
+response = client.images.generate(
+ model="google-imagen",
+ prompt="A majestic eagle soaring over snow-capped mountains",
+ n=1,
+ size="1024x1024"
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```python showLineNumbers title="Google AI Studio Image Generation via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.image_generation(
+ model="litellm_proxy/google-imagen",
+ prompt="A serene Japanese garden with cherry blossoms",
+ api_base="http://localhost:4000",
+ api_key="sk-1234"
+)
+
+print(response.data[0].url)
+```
+
+
+
+
+
+```bash showLineNumbers title="Google AI Studio Image Generation via Proxy - cURL"
+curl --location 'http://localhost:4000/v1/images/generations' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer sk-1234' \
+--data '{
+ "model": "google-imagen",
+ "prompt": "A cozy coffee shop interior with warm lighting",
+ "n": 1,
+ "size": "1024x1024"
+}'
+```
+
+
+
+
+## Supported Parameters
+
+Google AI Studio Image Generation supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description | Default | Example |
+|-----------|------|-------------|---------|---------|
+| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
+| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` |
+| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
+| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
+
+1. Create an account at [Google AI Studio](https://aistudio.google.com/)
+2. Generate an API key from [API Keys section](https://aistudio.google.com/app/apikey)
+3. Set your `GEMINI_API_KEY` environment variable
+4. Start generating images using LiteLLM
+
+## Additional Resources
+
+- [Google AI Studio Documentation](https://ai.google.dev/gemini-api/docs)
+- [Imagen Model Overview](https://ai.google.dev/gemini-api/docs/imagen)
+- [LiteLLM Image Generation Guide](../../completion/image_generation)
diff --git a/docs/my-website/docs/providers/gradient_ai.md b/docs/my-website/docs/providers/gradient_ai.md
new file mode 100644
index 00000000000..7b5eef04dcd
--- /dev/null
+++ b/docs/my-website/docs/providers/gradient_ai.md
@@ -0,0 +1,79 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# GradientAI
+https://digitalocean.com/products/gradientai
+
+
+LiteLLM provides native support for GradientAI models.
+To use a GradientAI model, specify it as `gradient_ai/` in your LiteLLM requests.
+
+
+## API Key & Endpoint
+
+Set your credentials and endpoint as environment variables:
+
+```python
+import os
+os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
+os.environ['GRADIENT_AI_AGENT_ENDPOINT'] = "https://api.gradient_ai.com/api/v1/chat" # default endpoint
+```
+
+## Sample Usage
+
+```python
+from litellm import completion
+import os
+
+os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
+response = completion(
+ model="gradient_ai/model-name",
+ messages=[
+ {"role": "user", "content": "Hello, how are you?"}
+ ],
+)
+print(response.choices[0].message.content)
+```
+
+## Streaming Example
+
+```python
+from litellm import completion
+import os
+
+os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
+response = completion(
+ model="gradient_ai/model-name",
+ messages=[
+ {"role": "user", "content": "Write a story about a robot learning to love"}
+ ],
+ stream=True,
+)
+
+for chunk in response:
+ print(chunk.choices[0].delta.content or "", end="")
+```
+
+## Supported Parameters
+
+| Parameter | Type | Description |
+|-----------------------------------|--------------|--------------------------------------------------------------------|
+| `temperature` | float | Controls randomness (0.0-2.0) |
+| `top_p` | float | Nucleus sampling parameter (0.0-1.0) |
+| `max_tokens` | int | Maximum tokens to generate |
+| `max_completion_tokens` | int | Alternative to max_tokens |
+| `stream` | bool | Whether to stream the response |
+| `k` | int | Top results to return from knowledge bases |
+| `retrieval_method` | string | Retrieval strategy (rewrite/step_back/sub_queries/none) |
+| `frequency_penalty` | float | Penalizes repeated tokens (-2.0 to 2.0) |
+| `presence_penalty` | float | Penalizes tokens based on presence (-2.0 to 2.0) |
+| `stop` | string/list | Sequences to stop generation |
+| `kb_filters` | List[Dict] | Filters for knowledge base retrieval |
+| `instruction_override` | string | Override agent's default instruction |
+| `include_retrieval_info` | bool | Include document retrieval metadata |
+| `include_guardrails_info` | bool | Include guardrail trigger metadata |
+| `provide_citations` | bool | Include citations in response |
+
+---
+
+For more details, see [DigitalOcean GradientAI documentation](https://digitalocean.com/products/gradientai).
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md
index 23393bcc825..59668b5eb5f 100644
--- a/docs/my-website/docs/providers/groq.md
+++ b/docs/my-website/docs/providers/groq.md
@@ -156,7 +156,9 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
-| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
+| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
+| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` |
+| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
## Groq - Tool / Function Calling Example
diff --git a/docs/my-website/docs/providers/heroku.md b/docs/my-website/docs/providers/heroku.md
new file mode 100644
index 00000000000..bf37ed64b19
--- /dev/null
+++ b/docs/my-website/docs/providers/heroku.md
@@ -0,0 +1,76 @@
+# Heroku
+
+## Provision a Model
+
+To use Heroku with LiteLLM, [configure a Heroku app and attach a supported model](https://devcenter.heroku.com/articles/heroku-inference#provision-access-to-an-ai-model-resource).
+
+
+## Supported Models
+
+Heroku for LiteLLM supports various [chat](https://devcenter.heroku.com/articles/heroku-inference-api-v1-chat-completions) models:
+
+| Model | Region |
+|-----------------------------------|---------|
+| [`heroku/claude-sonnet-4`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-4-sonnet) | US, EU |
+| [`heroku/claude-3-7-sonnet`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-7-sonnet) | US, EU |
+| [`heroku/claude-3-5-sonnet-latest`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-sonnet-latest) | US |
+| [`heroku/claude-3-5-haiku`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-5-haiku) | US |
+| [`heroku/claude-3`](https://devcenter.heroku.com/articles/heroku-inference-api-model-claude-3-haiku) | EU |
+
+## Environment Variables
+
+When you attach a model to a Heroku app, three config variables are set:
+
+- `INFERENCE_KEY`: The API key used for authenticating requests to the model.
+- `INFERENCE_MODEL_ID`: The name of the model, for example`claude-3-5-haiku`.
+- `INFERENCE_URL`: The base URL for calling the model.
+
+Both `INFERENCE_KEY` and `INFERENCE_URL` are required to make calls to your model.
+
+For more information on these variables, see the [Heroku documentation](https://devcenter.heroku.com/articles/heroku-inference#model-resource-config-vars).
+
+## Usage Examples
+### Using Config Variables
+
+Heroku uses the following LiteLLM API config variables:
+
+- `HEROKU_API_KEY`: This value corresponds to [LiteLLM's `api_key` param](https://docs.litellm.ai/docs/set_keys#litellmapi_key). Set this variable to the value of Heroku's `INFERENCE_KEY` config variable.
+- `HEROKU_API_BASE`: This value corresponds to [LiteLLM's `api_base` param](https://docs.litellm.ai/docs/set_keys#litellmapi_base). Set this variable to the value of Heroku's `INFERENCE_URL` config variable.
+
+In this example, we don't explicitly pass the `api_key` and `api_base` variables. Instead, we set the config variables which Heroku will use:
+
+```python
+import os
+from litellm import completion
+
+os.environ["HEROKU_API_BASE"] = "https://us.inference.heroku.com"
+os.environ["HEROKU_API_KEY"] = "fake-heroku-key"
+
+response = completion(
+ model="heroku/claude-3-5-haiku",
+ messages=[
+ {"role": "user", "content": "write code for saying hey from LiteLLM"}
+ ]
+)
+
+print(response)
+```
+
+> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use.
+
+### Explicitly Setting `api_key` and `api_base`
+
+```python
+from litellm import completion
+
+response = completion(
+ model="heroku/claude-sonnet-4",
+ api_key="fake-heroku-key",
+ api_base="https://us.inference.heroku.com",
+ messages=[
+ {"role": "user", "content": "write code for saying hey from LiteLLM"}
+ ],
+)
+```
+
+> Include the `heroku/` prefix in the model name so LiteLLM knows the model provider to use.
diff --git a/docs/my-website/docs/providers/huggingface_rerank.md b/docs/my-website/docs/providers/huggingface_rerank.md
new file mode 100644
index 00000000000..c28908b74ed
--- /dev/null
+++ b/docs/my-website/docs/providers/huggingface_rerank.md
@@ -0,0 +1,263 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# HuggingFace Rerank
+
+HuggingFace Rerank allows you to use reranking models hosted on Hugging Face infrastructure or your custom endpoints to reorder documents based on their relevance to a query.
+
+| Property | Details |
+|----------|---------|
+| Description | HuggingFace Rerank enables semantic reranking of documents using models hosted on Hugging Face infrastructure or custom endpoints. |
+| Provider Route on LiteLLM | `huggingface/` in model name |
+| Provider Doc | [Hugging Face Hub ↗](https://huggingface.co/models?pipeline_tag=sentence-similarity) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set your HuggingFace token
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+# Basic rerank usage
+response = litellm.rerank(
+ model="huggingface/BAAI/bge-reranker-base",
+ query="What is the capital of the United States?",
+ documents=[
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country.",
+ ],
+ top_n=3,
+)
+
+print(response)
+```
+
+### Custom Endpoint Usage
+
+```python showLineNumbers title="Using custom HuggingFace endpoint"
+import litellm
+
+response = litellm.rerank(
+ model="huggingface/BAAI/bge-reranker-base",
+ query="hello",
+ documents=["hello", "world"],
+ top_n=2,
+ api_base="https://my-custom-hf-endpoint.com",
+ api_key="test_api_key",
+)
+
+print(response)
+```
+
+### Async Usage
+
+```python showLineNumbers title="Async rerank example"
+import litellm
+import asyncio
+import os
+
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+async def async_rerank_example():
+ response = await litellm.arerank(
+ model="huggingface/BAAI/bge-reranker-base",
+ query="What is the capital of the United States?",
+ documents=[
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country.",
+ ],
+ top_n=3,
+ )
+ print(response)
+
+asyncio.run(async_rerank_example())
+```
+
+## LiteLLM Proxy
+
+### 1. Configure your model in config.yaml
+
+
+
+
+```yaml
+model_list:
+ - model_name: bge-reranker-base
+ litellm_params:
+ model: huggingface/BAAI/bge-reranker-base
+ api_key: os.environ/HF_TOKEN
+ - model_name: bge-reranker-large
+ litellm_params:
+ model: huggingface/BAAI/bge-reranker-large
+ api_key: os.environ/HF_TOKEN
+ - model_name: custom-reranker
+ litellm_params:
+ model: huggingface/BAAI/bge-reranker-base
+ api_base: https://my-custom-hf-endpoint.com
+ api_key: your-custom-api-key
+```
+
+
+
+
+### 2. Start the proxy
+
+```bash
+export HF_TOKEN="hf_xxxxxx"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+### 3. Make rerank requests
+
+
+
+
+```bash
+curl http://localhost:4000/rerank \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_API_KEY" \
+ -d '{
+ "model": "bge-reranker-base",
+ "query": "What is the capital of the United States?",
+ "documents": [
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country."
+ ],
+ "top_n": 3
+ }'
+```
+
+
+
+
+
+```python
+import litellm
+
+# Initialize with your LiteLLM proxy URL
+response = litellm.rerank(
+ model="bge-reranker-base",
+ query="What is the capital of the United States?",
+ documents=[
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country.",
+ ],
+ top_n=3,
+ api_base="http://localhost:4000",
+ api_key="your-litellm-api-key"
+)
+
+print(response)
+```
+
+
+
+
+
+```python
+import requests
+
+url = "http://localhost:4000/rerank"
+headers = {
+ "Authorization": "Bearer your-litellm-api-key",
+ "Content-Type": "application/json"
+}
+
+data = {
+ "model": "bge-reranker-base",
+ "query": "What is the capital of the United States?",
+ "documents": [
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country."
+ ],
+ "top_n": 3
+}
+
+response = requests.post(url, headers=headers, json=data)
+print(response.json())
+```
+
+
+
+
+
+
+## Configuration Options
+
+### Authentication
+
+#### Using HuggingFace Token (Serverless)
+```python
+import os
+os.environ["HF_TOKEN"] = "hf_xxxxxx"
+
+# Or pass directly
+litellm.rerank(
+ model="huggingface/BAAI/bge-reranker-base",
+ api_key="hf_xxxxxx",
+ # ... other params
+)
+```
+
+#### Using Custom Endpoint
+```python
+litellm.rerank(
+ model="huggingface/BAAI/bge-reranker-base",
+ api_base="https://your-custom-endpoint.com",
+ api_key="your-custom-key",
+ # ... other params
+)
+```
+
+
+
+## Response Format
+
+The response follows the standard rerank API format:
+
+```json
+{
+ "results": [
+ {
+ "index": 3,
+ "relevance_score": 0.999071
+ },
+ {
+ "index": 4,
+ "relevance_score": 0.7867867
+ },
+ {
+ "index": 0,
+ "relevance_score": 0.32713068
+ }
+ ],
+ "id": "07734bd2-2473-4f07-94e1-0d9f0e6843cf",
+ "meta": {
+ "api_version": {
+ "version": "2",
+ "is_experimental": false
+ },
+ "billed_units": {
+ "search_units": 1
+ }
+ }
+}
+```
+
diff --git a/docs/my-website/docs/providers/hyperbolic.md b/docs/my-website/docs/providers/hyperbolic.md
new file mode 100644
index 00000000000..7bad527fcfe
--- /dev/null
+++ b/docs/my-website/docs/providers/hyperbolic.md
@@ -0,0 +1,331 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Hyperbolic
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Hyperbolic provides access to the latest models at a fraction of legacy cloud costs, with OpenAI-compatible APIs for LLMs, image generation, and more. |
+| Provider Route on LiteLLM | `hyperbolic/` |
+| Link to Provider Doc | [Hyperbolic Documentation ↗](https://docs.hyperbolic.xyz) |
+| Base URL | `https://api.hyperbolic.xyz/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+
+
+
+https://docs.hyperbolic.xyz
+
+**We support ALL Hyperbolic models, just set `hyperbolic/` as a prefix when sending completion requests**
+
+## Available Models
+
+### Language Models
+
+| Model | Description | Context Window | Pricing per 1M tokens |
+|-------|-------------|----------------|----------------------|
+| `hyperbolic/deepseek-ai/DeepSeek-V3` | DeepSeek V3 - Fast and efficient | 131,072 tokens | $0.25 |
+| `hyperbolic/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 March 2024 version | 131,072 tokens | $0.25 |
+| `hyperbolic/deepseek-ai/DeepSeek-R1` | DeepSeek R1 - Reasoning model | 131,072 tokens | $2.00 |
+| `hyperbolic/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 May 2028 version | 131,072 tokens | $0.25 |
+| `hyperbolic/Qwen/Qwen2.5-72B-Instruct` | Qwen 2.5 72B Instruct | 131,072 tokens | $0.40 |
+| `hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct` | Qwen 2.5 Coder 32B for code generation | 131,072 tokens | $0.20 |
+| `hyperbolic/Qwen/Qwen3-235B-A22B` | Qwen 3 235B A22B variant | 131,072 tokens | $2.00 |
+| `hyperbolic/Qwen/QwQ-32B` | Qwen QwQ 32B | 131,072 tokens | $0.20 |
+| `hyperbolic/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B Instruct | 131,072 tokens | $0.80 |
+| `hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct` | Llama 3.1 405B Instruct | 131,072 tokens | $5.00 |
+| `hyperbolic/moonshotai/Kimi-K2-Instruct` | Kimi K2 Instruct | 131,072 tokens | $2.00 |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
+```
+
+Get your API key from [Hyperbolic dashboard](https://app.hyperbolic.ai).
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Hyperbolic Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
+
+messages = [{"content": "What is the capital of France?", "role": "user"}]
+
+# Hyperbolic call
+response = completion(
+ model="hyperbolic/Qwen/Qwen2.5-72B-Instruct",
+ messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Hyperbolic Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
+
+messages = [{"content": "Write a short poem about AI", "role": "user"}]
+
+# Hyperbolic call with streaming
+response = completion(
+ model="hyperbolic/deepseek-ai/DeepSeek-V3",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+### Function Calling
+
+```python showLineNumbers title="Hyperbolic Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
+
+tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather in a location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA"
+ },
+ "unit": {
+ "type": "string",
+ "enum": ["celsius", "fahrenheit"]
+ }
+ },
+ "required": ["location"]
+ }
+ }
+ }
+]
+
+response = completion(
+ model="hyperbolic/deepseek-ai/DeepSeek-V3",
+ messages=[{"role": "user", "content": "What's the weather like in New York?"}],
+ tools=tools,
+ tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: deepseek-fast
+ litellm_params:
+ model: hyperbolic/deepseek-ai/DeepSeek-V3
+ api_key: os.environ/HYPERBOLIC_API_KEY
+
+ - model_name: qwen-coder
+ litellm_params:
+ model: hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct
+ api_key: os.environ/HYPERBOLIC_API_KEY
+
+ - model_name: deepseek-reasoning
+ litellm_params:
+ model: hyperbolic/deepseek-ai/DeepSeek-R1
+ api_key: os.environ/HYPERBOLIC_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+
+
+
+```python showLineNumbers title="Hyperbolic via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+ model="deepseek-fast",
+ messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Hyperbolic via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+ model="qwen-coder",
+ messages=[{"role": "user", "content": "Write a Python function to sort a list"}],
+ stream=True
+)
+
+for chunk in response:
+ if chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+ model="litellm_proxy/deepseek-fast",
+ messages=[{"role": "user", "content": "What are the benefits of renewable energy?"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+ model="litellm_proxy/qwen-coder",
+ messages=[{"role": "user", "content": "Implement a binary search algorithm"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key",
+ stream=True
+)
+
+for chunk in response:
+ if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```bash showLineNumbers title="Hyperbolic via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "deepseek-fast",
+ "messages": [{"role": "user", "content": "What is machine learning?"}]
+ }'
+```
+
+```bash showLineNumbers title="Hyperbolic via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "qwen-coder",
+ "messages": [{"role": "user", "content": "Write a REST API in Python"}],
+ "stream": true
+ }'
+```
+
+
+
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
+
+## Supported OpenAI Parameters
+
+Hyperbolic supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
+| `model` | string | **Required**. Model ID (e.g., deepseek-ai/DeepSeek-V3, Qwen/Qwen2.5-72B-Instruct) |
+| `stream` | boolean | Optional. Enable streaming responses |
+| `temperature` | float | Optional. Sampling temperature (0.0 to 2.0) |
+| `top_p` | float | Optional. Nucleus sampling parameter |
+| `max_tokens` | integer | Optional. Maximum tokens to generate |
+| `frequency_penalty` | float | Optional. Penalize frequent tokens |
+| `presence_penalty` | float | Optional. Penalize tokens based on presence |
+| `stop` | string/array | Optional. Stop sequences |
+| `n` | integer | Optional. Number of completions to generate |
+| `tools` | array | Optional. List of available tools/functions |
+| `tool_choice` | string/object | Optional. Control tool/function calling |
+| `response_format` | object | Optional. Response format specification |
+| `seed` | integer | Optional. Random seed for reproducibility |
+| `user` | string | Optional. User identifier |
+
+## Advanced Usage
+
+### Custom API Base
+
+If you're using a custom Hyperbolic deployment:
+
+```python showLineNumbers title="Custom API Base"
+import litellm
+
+response = litellm.completion(
+ model="hyperbolic/deepseek-ai/DeepSeek-V3",
+ messages=[{"role": "user", "content": "Hello"}],
+ api_base="https://your-custom-hyperbolic-endpoint.com/v1",
+ api_key="your-api-key"
+)
+```
+
+### Rate Limits
+
+Hyperbolic offers different tiers:
+- **Basic**: 60 requests per minute (RPM)
+- **Pro**: 600 RPM
+- **Enterprise**: Custom limits
+
+## Pricing
+
+Hyperbolic offers competitive pay-as-you-go pricing with no hidden fees or long-term commitments. See the model table above for specific pricing per million tokens.
+
+### Precision Options
+- **BF16**: Best precision and performance, suitable for tasks where accuracy is critical
+- **FP8**: Optimized for efficiency and speed, ideal for high-throughput applications at lower cost
+
+## Additional Resources
+
+- [Hyperbolic Official Documentation](https://docs.hyperbolic.xyz)
+- [Hyperbolic Dashboard](https://app.hyperbolic.ai)
+- [API Reference](https://docs.hyperbolic.xyz/docs/rest-api)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/lambda_ai.md b/docs/my-website/docs/providers/lambda_ai.md
new file mode 100644
index 00000000000..91800faab70
--- /dev/null
+++ b/docs/my-website/docs/providers/lambda_ai.md
@@ -0,0 +1,280 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Lambda AI
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Lambda AI provides access to a wide range of open-source language models through their cloud GPU infrastructure, optimized for inference at scale. |
+| Provider Route on LiteLLM | `lambda_ai/` |
+| Link to Provider Doc | [Lambda AI API Documentation ↗](https://docs.lambda.ai/api) |
+| Base URL | `https://api.lambda.ai/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+
+
+
+https://docs.lambda.ai/api
+
+**We support ALL Lambda AI models, just set `lambda_ai/` as a prefix when sending completion requests**
+
+## Available Models
+
+Lambda AI offers a diverse selection of state-of-the-art open-source models:
+
+### Large Language Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/llama3.3-70b-instruct-fp8` | Llama 3.3 70B with FP8 quantization | 8,192 tokens |
+| `lambda_ai/llama3.1-405b-instruct-fp8` | Llama 3.1 405B with FP8 quantization | 8,192 tokens |
+| `lambda_ai/llama3.1-70b-instruct-fp8` | Llama 3.1 70B with FP8 quantization | 8,192 tokens |
+| `lambda_ai/llama3.1-8b-instruct` | Llama 3.1 8B instruction-tuned | 8,192 tokens |
+| `lambda_ai/llama3.1-nemotron-70b-instruct-fp8` | Llama 3.1 Nemotron 70B | 8,192 tokens |
+
+### DeepSeek Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/deepseek-llama3.3-70b` | DeepSeek Llama 3.3 70B | 8,192 tokens |
+| `lambda_ai/deepseek-r1-0528` | DeepSeek R1 0528 | 8,192 tokens |
+| `lambda_ai/deepseek-r1-671b` | DeepSeek R1 671B | 8,192 tokens |
+| `lambda_ai/deepseek-v3-0324` | DeepSeek V3 0324 | 8,192 tokens |
+
+### Hermes Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/hermes3-405b` | Hermes 3 405B | 8,192 tokens |
+| `lambda_ai/hermes3-70b` | Hermes 3 70B | 8,192 tokens |
+| `lambda_ai/hermes3-8b` | Hermes 3 8B | 8,192 tokens |
+
+### Coding Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/qwen25-coder-32b-instruct` | Qwen 2.5 Coder 32B | 8,192 tokens |
+| `lambda_ai/qwen3-32b-fp8` | Qwen 3 32B with FP8 | 8,192 tokens |
+
+### Vision Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/llama3.2-11b-vision-instruct` | Llama 3.2 11B with vision capabilities | 8,192 tokens |
+
+### Specialized Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `lambda_ai/llama-4-maverick-17b-128e-instruct-fp8` | Llama 4 Maverick with 128k context | 131,072 tokens |
+| `lambda_ai/llama-4-scout-17b-16e-instruct` | Llama 4 Scout with 16k context | 16,384 tokens |
+| `lambda_ai/lfm-40b` | LFM 40B model | 8,192 tokens |
+| `lambda_ai/lfm-7b` | LFM 7B model | 8,192 tokens |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Lambda AI Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Lambda AI call
+response = completion(
+ model="lambda_ai/llama3.1-8b-instruct",
+ messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Lambda AI Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
+
+messages = [{"content": "Write a short story about AI", "role": "user"}]
+
+# Lambda AI call with streaming
+response = completion(
+ model="lambda_ai/llama3.1-70b-instruct-fp8",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+### Vision/Multimodal Support
+
+The Llama 3.2 Vision model supports image inputs:
+
+```python showLineNumbers title="Lambda AI Vision/Multimodal"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
+
+messages = [{
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What's in this image?"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.jpg"
+ }
+ }
+ ]
+}]
+
+# Lambda AI vision model call
+response = completion(
+ model="lambda_ai/llama3.2-11b-vision-instruct",
+ messages=messages
+)
+
+print(response)
+```
+
+### Function Calling
+
+Lambda AI models support function calling:
+
+```python showLineNumbers title="Lambda AI Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
+
+# Define tools
+tools = [{
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather in a location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA"
+ }
+ },
+ "required": ["location"]
+ }
+ }
+}]
+
+messages = [{"role": "user", "content": "What's the weather in Boston?"}]
+
+# Lambda AI call with function calling
+response = completion(
+ model="lambda_ai/hermes3-70b",
+ messages=messages,
+ tools=tools,
+ tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy Server
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: llama-8b
+ litellm_params:
+ model: lambda_ai/llama3.1-8b-instruct
+ api_key: os.environ/LAMBDA_API_KEY
+ - model_name: deepseek-70b
+ litellm_params:
+ model: lambda_ai/deepseek-llama3.3-70b
+ api_key: os.environ/LAMBDA_API_KEY
+ - model_name: hermes-405b
+ litellm_params:
+ model: lambda_ai/hermes3-405b
+ api_key: os.environ/LAMBDA_API_KEY
+ - model_name: qwen-coder
+ litellm_params:
+ model: lambda_ai/qwen25-coder-32b-instruct
+ api_key: os.environ/LAMBDA_API_KEY
+```
+
+## Custom API Base
+
+If you need to use a custom API base URL:
+
+```python showLineNumbers title="Custom API Base"
+import os
+import litellm
+from litellm import completion
+
+# Using environment variable
+os.environ["LAMBDA_API_BASE"] = "https://custom.lambda-api.com/v1"
+os.environ["LAMBDA_API_KEY"] = "" # your API key
+
+# Or pass directly
+response = completion(
+ model="lambda_ai/llama3.1-8b-instruct",
+ messages=[{"content": "Hello!", "role": "user"}],
+ api_base="https://custom.lambda-api.com/v1",
+ api_key="your-api-key"
+)
+```
+
+## Supported OpenAI Parameters
+
+Lambda AI supports all standard OpenAI parameters since it's fully OpenAI-compatible:
+
+- `temperature`
+- `max_tokens`
+- `top_p`
+- `frequency_penalty`
+- `presence_penalty`
+- `stop`
+- `n`
+- `stream`
+- `tools`
+- `tool_choice`
+- `response_format`
+- `seed`
+- `user`
+- `logit_bias`
+
+Example with parameters:
+
+```python showLineNumbers title="Lambda AI with Parameters"
+response = completion(
+ model="lambda_ai/hermes3-405b",
+ messages=[{"content": "Explain quantum computing", "role": "user"}],
+ temperature=0.7,
+ max_tokens=500,
+ top_p=0.9,
+ frequency_penalty=0.2,
+ presence_penalty=0.1
+)
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/lemonade.md b/docs/my-website/docs/providers/lemonade.md
new file mode 100644
index 00000000000..fc77b78a76c
--- /dev/null
+++ b/docs/my-website/docs/providers/lemonade.md
@@ -0,0 +1,191 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Lemonade
+
+[Lemonade Server](https://lemonade-server.ai/) is an OpenAI-compatible local language model inference provider optimized for AMD GPUs and NPUs. The `lemonade` litellm provider supports standard chat completions with full OpenAI API compatibility.
+
+| Property | Details |
+|-------|-------|
+| Description | OpenAI-compatible AI provider for local and cloud-based language model inference |
+| Provider Route on LiteLLM | `lemonade/` (add this prefix to the model name - e.g. `lemonade/your-model-name`) |
+| API Endpoint for Provider | http://localhost:8000/api/v1 (default) |
+| Supported Endpoints | `/chat/completions` |
+
+## Supported OpenAI Parameters
+
+Lemonade is fully OpenAI-compatible and supports the following parameters:
+
+```
+"repeat_penalty"
+"functions"
+"logit_bias"
+"max_tokens"
+"max_completion_tokens"
+"presence_penalty"
+"stop"
+"temperature"
+"top_p"
+"top_k"
+"response_format"
+"tools"
+```
+
+
+## API Key Setup
+
+Lemonade can be configured with custom API URLs and doesn't require strict API key validation. Set the `LEMONADE_API_BASE` environment variable to modify the base URL.
+
+## Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+# Optional: Set custom API base. Useful if your lemonade server is on
+# a different port
+os.environ['LEMONADE_API_BASE'] = "http://localhost:8000/api/v1"
+
+response = completion(
+ model="lemonade/your-model-name",
+ messages=[
+ {"role": "user", "content": "Hello from LiteLLM!"}
+ ],
+)
+print(response)
+```
+
+## Streaming
+
+```python
+from litellm import completion
+import os
+
+# Optional: Set custom API base. Useful if your lemonade server is on
+# a different port
+os.environ['LEMONADE_API_BASE'] = "http://localhost:8000/api/v1"
+
+response = completion(
+ model="lemonade/your-model-name",
+ messages=[
+ {"role": "user", "content": "Write a short story"}
+ ],
+ stream=True
+)
+
+for chunk in response:
+ print(chunk.choices[0].delta.content, end='', flush=True)
+```
+
+## Advanced Usage
+
+### Custom Parameters
+
+Lemonade supports additional parameters beyond the standard OpenAI set:
+
+```python
+from litellm import completion
+
+response = completion(
+ model="lemonade/your-model-name",
+ messages=[{"role": "user", "content": "Explain quantum computing"}],
+ temperature=0.7,
+ max_tokens=500,
+ top_p=0.9,
+ top_k=50,
+ repeat_penalty=1.1,
+ stop=["Human:", "AI:"]
+)
+print(response)
+```
+
+### Function Calling
+
+Lemonade supports OpenAI-compatible function calling:
+
+```python
+from litellm import completion
+
+functions = [
+ {
+ "name": "get_weather",
+ "description": "Get current weather information",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state"
+ }
+ },
+ "required": ["location"]
+ }
+ }
+]
+
+response = completion(
+ model="lemonade/your-model-name",
+ messages=[{"role": "user", "content": "What's the weather in San Francisco?"}],
+ tools=[{"type": "function", "function": f} for f in functions],
+ tool_choice="auto"
+)
+print(response)
+```
+
+### Response Format
+
+Lemonade supports structured output with response format:
+
+```python
+from litellm import completion
+import json
+
+# Define schema in response_format
+response = completion(
+ model="lemonade/Qwen3-Coder-30B-A3B-Instruct-GGUF",
+ messages=[{"role": "user", "content": "Generate JSON data for a person with their name, age, and city."}],
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "name": "person",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "age": {"type": "integer"},
+ "city": {"type": "string"}
+ },
+ "required": ["name", "age"]
+ }
+ }
+ }
+)
+
+print(f"Model: {response.model}")
+print(f"JSON Output:")
+json_data = json.loads(response.choices[0].message.content)
+print(json.dumps(json_data, indent=2))
+```
+
+## Available Models
+
+Lemonade automatically validates available models by querying the `/models` endpoint. You can check available models programmatically:
+
+```python
+import httpx
+
+api_base = "http://localhost:8000" # or your custom base
+response = httpx.get(f"{api_base}/api/v1/models")
+models = response.json()
+print("Available models:", [model['id'] for model in models.get('data', [])])
+```
+
+## Support
+
+For more information regarding Lemonade please go to to the [Lemonade website](https://lemonade-server.ai/) or [Lemonade repository](https://github.com/lemonade-sdk/lemonade).
+
+
+
diff --git a/docs/my-website/docs/providers/litellm_proxy.md b/docs/my-website/docs/providers/litellm_proxy.md
index a9de5d5913d..bfefc8a787c 100644
--- a/docs/my-website/docs/providers/litellm_proxy.md
+++ b/docs/my-website/docs/providers/litellm_proxy.md
@@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
| Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. |
| Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) |
| Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) |
-| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` |
+| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` |
@@ -111,6 +111,21 @@ response = litellm.image_generation(
)
```
+## Image Edit
+
+```python
+import litellm
+
+with open("your-image.png", "rb") as f:
+ response = litellm.image_edit(
+ model="litellm_proxy/gpt-image-1",
+ prompt="Make this image a watercolor painting",
+ image=[f],
+ api_base="your-litellm-proxy-url",
+ api_key="your-litellm-proxy-api-key",
+ )
+```
+
## Audio Transcription
```python
@@ -165,6 +180,12 @@ LiteLLM Proxy works seamlessly with Langchain, LlamaIndex, OpenAI JS, Anthropic
## Send all SDK requests to LiteLLM Proxy
+:::info
+
+Requires v1.72.1 or higher.
+
+:::
+
Use this when calling LiteLLM Proxy from any library / codebase already using the LiteLLM SDK.
These flags will route all requests through your LiteLLM proxy, regardless of the model specified.
@@ -205,3 +226,38 @@ response = litellm.completion(
use_litellm_proxy=True
)
```
+
+## Sending `tags` to LiteLLM Proxy
+
+Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter.
+
+### Usage
+
+Send tags by including them in the `extra_body` parameter of your completion request:
+
+```python showLineNumbers title="Usage"
+import litellm
+
+response = litellm.completion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ api_base="http://localhost:4000",
+ api_key="sk-1234",
+ extra_body={"tags": ["user:ishaan", "department:engineering", "priority:high"]}
+)
+```
+
+### Async Usage
+
+```python showLineNumbers title="Async Usage"
+import litellm
+
+response = await litellm.acompletion(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ api_base="http://localhost:4000",
+ api_key="sk-1234",
+ extra_body={"tags": ["user:ishaan", "department:engineering"]}
+)
+```
+
diff --git a/docs/my-website/docs/providers/meta_llama.md b/docs/my-website/docs/providers/meta_llama.md
index 8219bef12b2..f4bcbf7692d 100644
--- a/docs/my-website/docs/providers/meta_llama.md
+++ b/docs/my-website/docs/providers/meta_llama.md
@@ -45,7 +45,7 @@ os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Meta Llama call
-response = completion(model="meta_llama/Llama-3.3-70B-Instruct", messages=messages)
+response = completion(model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8", messages=messages)
```
### Streaming
@@ -61,7 +61,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
# Meta Llama call with streaming
response = completion(
- model="meta_llama/Llama-3.3-70B-Instruct",
+ model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
messages=messages,
stream=True
)
@@ -70,6 +70,104 @@ for chunk in response:
print(chunk)
```
+### Function Calling
+
+```python showLineNumbers title="Meta Llama Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
+
+messages = [{"content": "What's the weather like in San Francisco?", "role": "user"}]
+
+# Define the function
+tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather in a given location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and state, e.g. San Francisco, CA"
+ },
+ "unit": {
+ "type": "string",
+ "enum": ["celsius", "fahrenheit"]
+ }
+ },
+ "required": ["location"]
+ }
+ }
+ }
+]
+
+# Meta Llama call with function calling
+response = completion(
+ model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+ messages=messages,
+ tools=tools,
+ tool_choice="auto"
+)
+
+print(response.choices[0].message.tool_calls)
+```
+
+### Tool Use
+
+```python showLineNumbers title="Meta Llama Tool Use"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = "" # your Meta Llama API key
+
+messages = [{"content": "Create a chart showing the population growth of New York City from 2010 to 2020", "role": "user"}]
+
+# Define the tools
+tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "create_chart",
+ "description": "Create a chart with the provided data",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "chart_type": {
+ "type": "string",
+ "enum": ["bar", "line", "pie", "scatter"],
+ "description": "The type of chart to create"
+ },
+ "title": {
+ "type": "string",
+ "description": "The title of the chart"
+ },
+ "data": {
+ "type": "object",
+ "description": "The data to plot in the chart"
+ }
+ },
+ "required": ["chart_type", "title", "data"]
+ }
+ }
+ }
+]
+
+# Meta Llama call with tool use
+response = completion(
+ model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+ messages=messages,
+ tools=tools,
+ tool_choice="auto"
+)
+
+print(response.choices[0].message.content)
+```
## Usage - LiteLLM Proxy
@@ -111,7 +209,7 @@ client = OpenAI(
# Non-streaming response
response = client.chat.completions.create(
- model="meta_llama/Llama-3.3-70B-Instruct",
+ model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
messages=[{"role": "user", "content": "Write a short poem about AI."}]
)
@@ -129,7 +227,7 @@ client = OpenAI(
# Streaming response
response = client.chat.completions.create(
- model="meta_llama/Llama-3.3-70B-Instruct",
+ model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
messages=[{"role": "user", "content": "Write a short poem about AI."}],
stream=True
)
diff --git a/docs/my-website/docs/providers/mistral.md b/docs/my-website/docs/providers/mistral.md
index 62a91c687ae..e0fccba7866 100644
--- a/docs/my-website/docs/providers/mistral.md
+++ b/docs/my-website/docs/providers/mistral.md
@@ -144,20 +144,22 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported.
:::
-| Model Name | Function Call |
-|----------------|--------------------------------------------------------------|
-| Mistral Small | `completion(model="mistral/mistral-small-latest", messages)` |
-| Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`|
-| Mistral Large 2 | `completion(model="mistral/mistral-large-2407", messages)` |
-| Mistral Large Latest | `completion(model="mistral/mistral-large-latest", messages)` |
-| Mistral 7B | `completion(model="mistral/open-mistral-7b", messages)` |
-| Mixtral 8x7B | `completion(model="mistral/open-mixtral-8x7b", messages)` |
-| Mixtral 8x22B | `completion(model="mistral/open-mixtral-8x22b", messages)` |
-| Codestral | `completion(model="mistral/codestral-latest", messages)` |
-| Mistral NeMo | `completion(model="mistral/open-mistral-nemo", messages)` |
-| Mistral NeMo 2407 | `completion(model="mistral/open-mistral-nemo-2407", messages)` |
-| Codestral Mamba | `completion(model="mistral/open-codestral-mamba", messages)` |
-| Codestral Mamba | `completion(model="mistral/codestral-mamba-latest"", messages)` |
+| Model Name | Function Call | Reasoning Support |
+|----------------|--------------------------------------------------------------|-------------------|
+| Mistral Small | `completion(model="mistral/mistral-small-latest", messages)` | No |
+| Mistral Medium | `completion(model="mistral/mistral-medium-latest", messages)`| No |
+| Mistral Large 2 | `completion(model="mistral/mistral-large-2407", messages)` | No |
+| Mistral Large Latest | `completion(model="mistral/mistral-large-latest", messages)` | No |
+| **Magistral Small** | `completion(model="mistral/magistral-small-2506", messages)` | Yes |
+| **Magistral Medium** | `completion(model="mistral/magistral-medium-2506", messages)`| Yes |
+| Mistral 7B | `completion(model="mistral/open-mistral-7b", messages)` | No |
+| Mixtral 8x7B | `completion(model="mistral/open-mixtral-8x7b", messages)` | No |
+| Mixtral 8x22B | `completion(model="mistral/open-mixtral-8x22b", messages)` | No |
+| Codestral | `completion(model="mistral/codestral-latest", messages)` | No |
+| Mistral NeMo | `completion(model="mistral/open-mistral-nemo", messages)` | No |
+| Mistral NeMo 2407 | `completion(model="mistral/open-mistral-nemo-2407", messages)` | No |
+| Codestral Mamba | `completion(model="mistral/open-codestral-mamba", messages)` | No |
+| Codestral Mamba | `completion(model="mistral/codestral-mamba-latest"", messages)` | No |
## Function Calling
@@ -203,6 +205,112 @@ assert isinstance(
)
```
+## Reasoning
+
+Mistral does not directly support reasoning, instead it recommends a specific [system prompt](https://docs.mistral.ai/capabilities/reasoning/) to use with their magistral models. By setting the `reasoning_effort` parameter, LiteLLM will prepend the system prompt to the request.
+
+If an existing system message is provided, LiteLLM will send both as a list of system messages (you can verify this by enabling `litellm._turn_on_debug()`).
+
+### Supported Models
+
+| Model Name | Function Call |
+|----------------|--------------------------------------------------------------|
+| Magistral Small | `completion(model="mistral/magistral-small-2506", messages)` |
+| Magistral Medium | `completion(model="mistral/magistral-medium-2506", messages)`|
+
+### Using Reasoning Effort
+
+The `reasoning_effort` parameter controls how much effort the model puts into reasoning. When used with magistral models.
+
+```python
+from litellm import completion
+import os
+
+os.environ['MISTRAL_API_KEY'] = "your-api-key"
+
+response = completion(
+ model="mistral/magistral-medium-2506",
+ messages=[
+ {"role": "user", "content": "What is 15 multiplied by 7?"}
+ ],
+ reasoning_effort="medium" # Options: "low", "medium", "high"
+)
+
+print(response)
+```
+
+### Example with System Message
+
+If you already have a system message, LiteLLM will prepend the reasoning instructions:
+
+```python
+response = completion(
+ model="mistral/magistral-medium-2506",
+ messages=[
+ {"role": "system", "content": "You are a helpful math tutor."},
+ {"role": "user", "content": "Explain how to solve quadratic equations."}
+ ],
+ reasoning_effort="high"
+)
+
+# The system message becomes:
+# "When solving problems, think step-by-step in tags before providing your final answer...
+#
+# You are a helpful math tutor."
+```
+
+### Usage with LiteLLM Proxy
+
+You can also use reasoning capabilities through the LiteLLM proxy:
+
+
+
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data '{
+ "model": "magistral-medium-2506",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the square root of 144? Show your reasoning."
+ }
+ ],
+ "reasoning_effort": "medium"
+ }'
+```
+
+
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="magistral-medium-2506",
+ messages=[
+ {
+ "role": "user",
+ "content": "Calculate the area of a circle with radius 5. Show your work."
+ }
+ ],
+ reasoning_effort="high"
+)
+
+print(response)
+```
+
+
+
+### Important Notes
+
+- **Model Compatibility**: Reasoning parameters only work with magistral models
+- **Backward Compatibility**: Non-magistral models will ignore reasoning parameters and work normally
+
## Sample Usage - Embedding
```python
from litellm import embedding
diff --git a/docs/my-website/docs/providers/moonshot.md b/docs/my-website/docs/providers/moonshot.md
new file mode 100644
index 00000000000..2e00bae3551
--- /dev/null
+++ b/docs/my-website/docs/providers/moonshot.md
@@ -0,0 +1,238 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Moonshot AI
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Moonshot AI provides large language models including the moonshot-v1 series and kimi models. |
+| Provider Route on LiteLLM | `moonshot/` |
+| Link to Provider Doc | [Moonshot AI ↗](https://platform.moonshot.ai/) |
+| Base URL | `https://api.moonshot.ai/` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+
+
+
+https://platform.moonshot.ai/
+
+**We support ALL Moonshot AI models, just set `moonshot/` as a prefix when sending completion requests**
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key
+```
+
+**ATTENTION:**
+
+Moonshot AI offers two distinct API endpoints: a global one and a China-specific one.
+- Global API Base URL: `https://api.moonshot.ai/v1` (This is the one currently implemented)
+- China API Base URL: `https://api.moonshot.cn/v1`
+
+You can overwrite the base url with:
+
+```
+os.environ["MOONSHOT_API_BASE"] = "https://api.moonshot.cn/v1"
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Moonshot Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Moonshot call
+response = completion(
+ model="moonshot/moonshot-v1-8k",
+ messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Moonshot Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Moonshot call with streaming
+response = completion(
+ model="moonshot/moonshot-v1-8k",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: moonshot-v1-8k
+ litellm_params:
+ model: moonshot/moonshot-v1-8k
+ api_key: os.environ/MOONSHOT_API_KEY
+
+ - model_name: moonshot-v1-32k
+ litellm_params:
+ model: moonshot/moonshot-v1-32k
+ api_key: os.environ/MOONSHOT_API_KEY
+
+ - model_name: moonshot-v1-128k
+ litellm_params:
+ model: moonshot/moonshot-v1-128k
+ api_key: os.environ/MOONSHOT_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+
+
+
+```python showLineNumbers title="Moonshot via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+ model="moonshot-v1-8k",
+ messages=[{"role": "user", "content": "hello from litellm"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Moonshot via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+ model="moonshot-v1-8k",
+ messages=[{"role": "user", "content": "hello from litellm"}],
+ stream=True
+)
+
+for chunk in response:
+ if chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```python showLineNumbers title="Moonshot via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+ model="litellm_proxy/moonshot-v1-8k",
+ messages=[{"role": "user", "content": "hello from litellm"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Moonshot via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+ model="litellm_proxy/moonshot-v1-8k",
+ messages=[{"role": "user", "content": "hello from litellm"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key",
+ stream=True
+)
+
+for chunk in response:
+ if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```bash showLineNumbers title="Moonshot via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "moonshot-v1-8k",
+ "messages": [{"role": "user", "content": "hello from litellm"}]
+ }'
+```
+
+```bash showLineNumbers title="Moonshot via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "moonshot-v1-8k",
+ "messages": [{"role": "user", "content": "hello from litellm"}],
+ "stream": true
+ }'
+```
+
+
+
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
+
+## Moonshot AI Limitations & LiteLLM Handling
+
+LiteLLM automatically handles the following [Moonshot AI limitations](https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-api-compatibility) to provide seamless OpenAI compatibility:
+
+### Temperature Range Limitation
+**Limitation**: Moonshot AI only supports temperature range [0, 1] (vs OpenAI's [0, 2])
+**LiteLLM Handling**: Automatically clamps any temperature > 1 to 1
+
+### Temperature + Multiple Outputs Limitation
+**Limitation**: If temperature < 0.3 and n > 1, Moonshot AI raises an exception
+**LiteLLM Handling**: Automatically sets temperature to 0.3 when this condition is detected
+
+### Tool Choice "Required" Not Supported
+**Limitation**: Moonshot AI doesn't support `tool_choice="required"`
+**LiteLLM Handling**: Converts this by:
+- Adding message: "Please select a tool to handle the current issue."
+- Removing the `tool_choice` parameter from the request
diff --git a/docs/my-website/docs/providers/morph.md b/docs/my-website/docs/providers/morph.md
new file mode 100644
index 00000000000..e49c60b5665
--- /dev/null
+++ b/docs/my-website/docs/providers/morph.md
@@ -0,0 +1,123 @@
+# Morph
+
+LiteLLM supports all models on [Morph](https://morphllm.com)
+
+## Overview
+
+Morph provides specialized AI models designed for agentic workflows, particularly excelling at precise code editing and manipulation. Their "Apply" models enable targeted code changes without full file rewrites, making them ideal for AI agents that need to make intelligent, context-aware code modifications.
+
+## API Key
+```python
+import os
+os.environ["MORPH_API_KEY"] = "your-api-key"
+```
+
+## Sample Usage
+
+```python
+from litellm import completion
+
+# set env variable
+os.environ["MORPH_API_KEY"] = "your-api-key"
+
+messages = [
+ {"role": "user", "content": "Write a Python function to calculate factorial"}
+]
+
+## Morph v3 Fast - Optimized for speed
+response = completion(
+ model="morph/morph-v3-fast",
+ messages=messages,
+)
+print(response)
+
+## Morph v3 Large - Most capable model
+response = completion(
+ model="morph/morph-v3-large",
+ messages=messages,
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+
+# set env variable
+os.environ["MORPH_API_KEY"] = "your-api-key"
+
+messages = [
+ {"role": "user", "content": "Write a Python function to calculate factorial"}
+]
+
+## Morph v3 Fast with streaming
+response = completion(
+ model="morph/morph-v3-fast",
+ messages=messages,
+ stream=True,
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+## Supported Models
+
+| Model Name | Function Call | Description | Context Window |
+|--------------------------|--------------------------------------------|-----------------------|----------------|
+| morph-v3-fast | `completion('morph/morph-v3-fast', messages)` | Fastest model, optimized for quick responses | 16k tokens |
+| morph-v3-large | `completion('morph/morph-v3-large', messages)` | Most capable model for complex tasks | 16k tokens |
+
+## Usage - LiteLLM Proxy Server
+
+Here's how to use Morph with the LiteLLM Proxy Server:
+
+1. Save API key in your environment
+```bash
+export MORPH_API_KEY="your-api-key"
+```
+
+2. Add model to config.yaml
+```yaml
+model_list:
+ - model_name: morph-v3-fast
+ litellm_params:
+ model: morph/morph-v3-fast
+
+ - model_name: morph-v3-large
+ litellm_params:
+ model: morph/morph-v3-large
+```
+
+3. Start the proxy server
+```bash
+litellm --config config.yaml
+```
+
+## Advanced Usage
+
+### Setting API Base
+```python
+import litellm
+
+# set custom api base
+response = completion(
+ model="morph/morph-v3-large",
+ messages=[{"role": "user", "content": "Hello, world!"}],
+ api_base="https://api.morphllm.com/v1"
+)
+print(response)
+```
+
+### Setting API Key
+```python
+import litellm
+
+# set api key via completion
+response = completion(
+ model="morph/morph-v3-large",
+ messages=[{"role": "user", "content": "Hello, world!"}],
+ api_key="your-api-key"
+)
+print(response)
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/nebius.md b/docs/my-website/docs/providers/nebius.md
index 26b5098c9f2..a5d0661fef0 100644
--- a/docs/my-website/docs/providers/nebius.md
+++ b/docs/my-website/docs/providers/nebius.md
@@ -168,7 +168,7 @@ The Nebius provider supports the following parameters:
| max_tokens | integer | Maximum number of tokens to generate |
| n | integer | Number of completions to generate |
| presence_penalty | number | Penalizes tokens based on if they appear in the text so far |
-| response_format | object | Format of the response, e.g., {"type": "json"} |
+| response_format | object | Format of the response, e.g., `{"type": "json"}` |
| seed | integer | Sampling seed for deterministic results |
| stop | string/array | Sequences where the API will stop generating tokens |
| stream | boolean | Whether to stream the response |
diff --git a/docs/my-website/docs/providers/nvidia_nim.md b/docs/my-website/docs/providers/nvidia_nim.md
index 270b356c917..9dbfc80f4e4 100644
--- a/docs/my-website/docs/providers/nvidia_nim.md
+++ b/docs/my-website/docs/providers/nvidia_nim.md
@@ -15,8 +15,8 @@ https://docs.api.nvidia.com/nim/reference/
| Description | Nvidia NIM is a platform that provides a simple API for deploying and using AI models. LiteLLM supports all models from [Nvidia NIM](https://developer.nvidia.com/nim/) |
| Provider Route on LiteLLM | `nvidia_nim/` |
| Provider Doc | [Nvidia NIM Docs ↗](https://developer.nvidia.com/nim/) |
-| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ |
-| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings` |
+| API Endpoint for Provider | https://integrate.api.nvidia.com/v1/ (chat/embeddings), https://ai.api.nvidia.com/v1/ (rerank) |
+| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/responses`, `/embeddings`, `/rerank` |
## API Key
```python
diff --git a/docs/my-website/docs/providers/nvidia_nim_rerank.md b/docs/my-website/docs/providers/nvidia_nim_rerank.md
new file mode 100644
index 00000000000..7373014a960
--- /dev/null
+++ b/docs/my-website/docs/providers/nvidia_nim_rerank.md
@@ -0,0 +1,261 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Nvidia NIM - Rerank
+
+Use Nvidia NIM Rerank models through LiteLLM.
+
+| Property | Details |
+|----------|---------|
+| Description | Nvidia NIM provides high-performance reranking models for semantic search and retrieval-augmented generation (RAG) |
+| Provider Doc | [Nvidia NIM Rerank API ↗](https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer) |
+| Supported Endpoint | `/rerank` |
+
+## Overview
+
+Nvidia NIM rerank models help you:
+- Reorder search results by relevance to a query
+- Improve RAG (Retrieval-Augmented Generation) accuracy
+- Filter and rank large document sets efficiently
+
+**Supported Models:**
+- All Nvidia NIM rerank models on their platform
+
+:::tip
+
+See the full list of LiteLLM supported Nvidia NIM rerank models on [Nvidia NIM](https://models.litellm.ai)
+
+:::
+
+## Usage
+
+### LiteLLM Python SDK
+
+
+
+
+```python
+import litellm
+import os
+
+os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
+
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
+ query="What is the GPU memory bandwidth of H100 SXM?",
+ documents=[
+ "The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth.",
+ "A100 provides up to 20X higher performance over the prior generation.",
+ "Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
+ ],
+ top_n=3,
+)
+
+print(response)
+```
+
+
+
+
+```python
+import litellm
+import os
+
+os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
+
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3",
+ query="What is the GPU memory bandwidth of H100 SXM?",
+ documents=[
+ "The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth.",
+ "A100 provides up to 20X higher performance over the prior generation.",
+ "Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
+ ],
+ top_n=3,
+)
+
+print(response)
+```
+
+
+
+
+**Response:**
+```json
+{
+ "results": [
+ {
+ "index": 2,
+ "relevance_score": 6.828125,
+ "document": {
+ "text": "Accelerated servers with H100 deliver 3 terabytes per second (TB/s) of memory bandwidth per GPU."
+ }
+ },
+ {
+ "index": 0,
+ "relevance_score": -1.564453125,
+ "document": {
+ "text": "The Hopper GPU is paired with the Grace CPU using NVIDIA's ultra-fast chip-to-chip interconnect, delivering 900GB/s of bandwidth."
+ }
+ }
+ ]
+}
+```
+
+
+## Usage with LiteLLM Proxy
+
+### 1. Setup Config
+
+Add Nvidia NIM rerank models to your proxy configuration:
+
+```yaml
+model_list:
+ - model_name: nvidia-rerank
+ litellm_params:
+ model: nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2
+ api_key: os.environ/NVIDIA_NIM_API_KEY
+```
+
+### 2. Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+### 3. Make Rerank Requests
+
+```bash
+curl -X POST http://0.0.0.0:4000/rerank \
+ -H "Authorization: Bearer sk-1234" \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "nvidia-rerank",
+ "query": "What is the GPU memory bandwidth of H100?",
+ "documents": [
+ "H100 delivers 3TB/s memory bandwidth",
+ "A100 has 2TB/s memory bandwidth",
+ "V100 offers 900GB/s memory bandwidth"
+ ],
+ "top_n": 2
+ }'
+```
+
+## API Parameters
+
+### Required Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `model` | string | The Nvidia NIM rerank model name with `nvidia_nim/` prefix |
+| `query` | string | The search query to rank documents against |
+| `documents` | array | List of documents to rank (1-1000 documents) |
+
+### Optional Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `top_n` | integer | All documents | Number of top-ranked documents to return |
+
+### Nvidia-Specific Parameters
+
+**`truncate`**: Controls how text is truncated if it exceeds the model's context window
+- `"NONE"`: No truncation (request may fail if too long)
+- `"END"`: Truncate from the end of the text
+
+```python
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
+ query="GPU performance",
+ documents=["High performance computing", "Fast GPU processing"],
+ top_n=2,
+ truncate="END", # Nvidia-specific parameter
+)
+```
+
+## Authentication
+
+Set your Nvidia NIM API key:
+
+
+
+
+```bash
+export NVIDIA_NIM_API_KEY="nvapi-..."
+```
+
+
+
+
+```python
+import os
+os.environ['NVIDIA_NIM_API_KEY'] = "nvapi-..."
+
+# Or pass directly
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
+ query="test",
+ documents=["doc1"],
+ api_key="nvapi-...",
+)
+```
+
+
+
+
+## API Endpoint
+
+The rerank endpoint uses a different base URL than chat/embeddings:
+
+- **Chat/Embeddings:** `https://integrate.api.nvidia.com/v1/`
+- **Rerank:** `https://ai.api.nvidia.com/v1/`
+
+LiteLLM automatically uses the correct endpoint for rerank requests.
+
+### Custom API Base URL
+
+You can override the default base URL in several ways:
+
+**Option 1: Environment Variable**
+
+```bash
+export NVIDIA_NIM_API_BASE="https://your-custom-endpoint.com"
+```
+
+**Option 2: Pass as parameter**
+
+```python
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
+ query="test",
+ documents=["doc1"],
+ api_base="https://your-custom-endpoint.com",
+)
+```
+
+**Option 3: Full URL (including model path)**
+
+If you have the complete endpoint URL, you can pass it directly:
+
+```python
+response = litellm.rerank(
+ model="nvidia_nim/nvidia/llama-3_2-nv-rerankqa-1b-v2",
+ query="test",
+ documents=["doc1"],
+ api_base="https://your-custom-endpoint.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking",
+)
+```
+
+LiteLLM will detect the full URL (by checking for `/retrieval/` in the path) and use it as-is.
+
+### How do I get an API key?
+
+Get your Nvidia NIM API key from [Nvidia's website](https://developer.nvidia.com/nim/).
+
+## Related Documentation
+
+- [Nvidia NIM - Main Documentation](./nvidia_nim)
+- [Nvidia NIM Chat Completions](./nvidia_nim#sample-usage)
+- [LiteLLM Rerank Endpoint](../rerank)
+- [Nvidia NIM Official Docs ↗](https://docs.api.nvidia.com/nim/reference/)
+
diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md
new file mode 100644
index 00000000000..c11d64f4553
--- /dev/null
+++ b/docs/my-website/docs/providers/oci.md
@@ -0,0 +1,85 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Oracle Cloud Infrastructure (OCI)
+LiteLLM supports the following models for OCI on-demand GenAI API.
+
+Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region.
+
+- `meta.llama-4-maverick-17b-128e-instruct-fp8`
+- `meta.llama-4-scout-17b-16e-instruct`
+- `meta.llama-3.3-70b-instruct`
+- `meta.llama-3.2-90b-vision-instruct`
+- `meta.llama-3.1-405b-instruct`
+
+- `xai.grok-4`
+- `xai.grok-3`
+- `xai.grok-3-fast`
+- `xai.grok-3-mini`
+- `xai.grok-3-mini-fast`
+
+## Authentication
+
+LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
+
+- `user`
+- `fingerprint`
+- `tenancy`
+- `region`
+- `key_file`
+
+## Usage
+
+Input the parameters obtained from the OCI signing key creation process into the `completion` function.
+
+```python
+import os
+from litellm import completion
+
+messages = [{"role": "user", "content": "Hey! how's it going?"}]
+response = completion(
+ model="oci/xai.grok-4",
+ messages=messages,
+ oci_region=,
+ oci_user=,
+ oci_fingerprint=,
+ oci_tenancy=,
+ oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED"
+ # Provide either the private key string OR the path to the key file:
+ # Option 1: pass the private key as a string
+ oci_key=,
+ # Option 2: pass the private key file path
+ # oci_key_file="",
+ oci_compartment_id=,
+)
+print(response)
+```
+
+
+## Usage - Streaming
+Just set `stream=True` when calling completion.
+
+```python
+import os
+from litellm import completion
+
+messages = [{"role": "user", "content": "Hey! how's it going?"}]
+response = completion(
+ model="oci/xai.grok-4",
+ messages=messages,
+ stream=True,
+ oci_region=,
+ oci_user=,
+ oci_fingerprint=,
+ oci_tenancy=,
+ oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED"
+ # Provide either the private key string OR the path to the key file:
+ # Option 1: pass the private key as a string
+ oci_key=,
+ # Option 2: pass the private key file path
+ # oci_key_file="",
+ oci_compartment_id=,
+)
+for chunk in response:
+ print(chunk["choices"][0]["delta"]["content"]) # same as openai format
+```
diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md
index 4fd75035fb0..dbd0b9927a9 100644
--- a/docs/my-website/docs/providers/openai.md
+++ b/docs/my-website/docs/providers/openai.md
@@ -163,6 +163,15 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
+| gpt-5 | `response = completion(model="gpt-5", messages=messages)` |
+| gpt-5-mini | `response = completion(model="gpt-5-mini", messages=messages)` |
+| gpt-5-nano | `response = completion(model="gpt-5-nano", messages=messages)` |
+| gpt-5-chat | `response = completion(model="gpt-5-chat", messages=messages)` |
+| gpt-5-chat-latest | `response = completion(model="gpt-5-chat-latest", messages=messages)` |
+| gpt-5-2025-08-07 | `response = completion(model="gpt-5-2025-08-07", messages=messages)` |
+| gpt-5-mini-2025-08-07 | `response = completion(model="gpt-5-mini-2025-08-07", messages=messages)` |
+| gpt-5-nano-2025-08-07 | `response = completion(model="gpt-5-nano-2025-08-07", messages=messages)` |
+| gpt-5-pro | `response = completion(model="gpt-5-pro", messages=messages)` |
| gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` |
| gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` |
| gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` |
@@ -331,6 +340,70 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
| fine tuned `gpt-3.5-turbo-0613` | `response = completion(model="ft:gpt-3.5-turbo-0613", messages=messages)` |
+## OpenAI Chat Completion to Responses API Bridge
+
+Call any Responses API model from OpenAI's `/chat/completions` endpoint.
+
+
+
+
+```python
+import litellm
+import os
+
+os.environ["OPENAI_API_KEY"] = "sk-1234"
+
+response = litellm.completion(
+ model="o3-deep-research-2025-06-26",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ tools=[
+ {"type": "web_search_preview"},
+ {"type": "code_interpreter", "container": {"type": "auto"}},
+ ],
+)
+print(response)
+```
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: openai-model
+ litellm_params:
+ model: o3-deep-research-2025-06-26
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start the proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model": "openai-model",
+ "messages": [
+ {"role": "user", "content": "What is the capital of France?"}
+ ],
+ "tools": [
+ {"type": "web_search_preview"},
+ {"type": "code_interpreter", "container": {"type": "auto"}},
+ ],
+}'
+```
+
+
+
+
+
## OpenAI Audio Transcription
LiteLLM supports OpenAI Audio Transcription endpoint.
@@ -677,4 +750,24 @@ In your logs you should see the forwarded org id
```bash
LiteLLM:DEBUG: utils.py:255 - Request to litellm:
LiteLLM:DEBUG: utils.py:255 - litellm.acompletion(... organization='my-special-org',)
+```
+
+## GPT-5 Pro Special Notes
+
+GPT-5 Pro is OpenAI's most advanced reasoning model with unique characteristics:
+
+- **Responses API Only**: GPT-5 Pro is only available through the `/v1/responses` endpoint
+- **No Streaming**: Does not support streaming responses
+- **High Reasoning**: Designed for complex reasoning tasks with highest effort reasoning
+- **Context Window**: 400,000 tokens input, 272,000 tokens output
+- **Pricing**: $15.00 input / $120.00 output per 1M tokens (Standard), $7.50 input / $60.00 output (Batch)
+- **Tools**: Supports Web Search, File Search, Image Generation, MCP (but not Code Interpreter or Computer Use)
+- **Modalities**: Text and Image input, Text output only
+
+```python
+# GPT-5 Pro usage example
+response = completion(
+ model="gpt-5-pro",
+ messages=[{"role": "user", "content": "Solve this complex reasoning problem..."}]
+)
```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md
index e88512ecfd4..8d91ca674b7 100644
--- a/docs/my-website/docs/providers/openai/responses_api.md
+++ b/docs/my-website/docs/providers/openai/responses_api.md
@@ -37,6 +37,29 @@ for event in response:
print(event)
```
+#### Image Generation with Streaming
+```python showLineNumbers title="OpenAI Streaming Image Generation"
+import litellm
+import base64
+
+# Streaming image generation with partial images
+stream = litellm.responses(
+ model="gpt-4.1", # Use an actual image generation model
+ input="Generate a gorgeous image of a river made of white owl feathers",
+ stream=True,
+ tools=[{"type": "image_generation", "partial_images": 2}],
+
+)
+
+for event in stream:
+ if event.type == "response.image_generation_call.partial_image":
+ idx = event.partial_image_index
+ image_base64 = event.partial_image_b64
+ image_bytes = base64.b64decode(image_base64)
+ with open(f"river{idx}.png", "wb") as f:
+ f.write(image_bytes)
+```
+
#### GET a Response
```python showLineNumbers title="Get Response by ID"
import litellm
@@ -150,6 +173,33 @@ for event in response:
print(event)
```
+#### Image Generation with Streaming
+```python showLineNumbers title="OpenAI Proxy Streaming Image Generation"
+from openai import OpenAI
+import base64
+
+# Initialize client with your proxy URL
+client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
+
+stream = client.responses.create(
+ model="gpt-4.1",
+ input="Draw a gorgeous image of a river made of white owl feathers, snaking its way through a serene winter landscape",
+ stream=True,
+ tools=[{"type": "image_generation", "partial_images": 2}],
+)
+
+
+for event in stream:
+ print(f"event: {event}")
+ if event.type == "response.image_generation_call.partial_image":
+ idx = event.partial_image_index
+ image_base64 = event.partial_image_b64
+ image_bytes = base64.b64decode(image_base64)
+ with open(f"river{idx}.png", "wb") as f:
+ f.write(image_bytes)
+
+```
+
#### GET a Response
```python showLineNumbers title="Get Response by ID with OpenAI SDK"
from openai import OpenAI
@@ -207,6 +257,50 @@ print(delete_response)
|----------|---------------------|
| `openai` | [All Responses API parameters are supported](https://github.com/BerriAI/litellm/blob/7c3df984da8e4dff9201e4c5353fdc7a2b441831/litellm/llms/openai/responses/transformation.py#L23) |
+## Reusable Prompts
+
+Use the `prompt` parameter to reference a stored prompt template and optionally supply variables.
+
+```python showLineNumbers title="Stored Prompt"
+import litellm
+
+response = litellm.responses(
+ model="openai/o1-pro",
+ prompt={
+ "id": "pmpt_abc123",
+ "version": "2",
+ "variables": {
+ "customer_name": "Jane Doe",
+ "product": "40oz juice box",
+ },
+ },
+)
+
+print(response)
+```
+
+The same parameter is supported when calling the LiteLLM proxy with the OpenAI SDK:
+
+```python showLineNumbers title="Stored Prompt via Proxy"
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:4000", api_key="your-api-key")
+
+response = client.responses.create(
+ model="openai/o1-pro",
+ prompt={
+ "id": "pmpt_abc123",
+ "version": "2",
+ "variables": {
+ "customer_name": "Jane Doe",
+ "product": "40oz juice box",
+ },
+ },
+)
+
+print(response)
+```
+
## Computer Use
@@ -448,3 +542,355 @@ print(response_with_mcp_call)
+## Verbosity Parameter
+
+The `verbosity` parameter is supported for the `responses` API.
+
+
+
+
+```python showLineNumbers title="Verbosity Parameter"
+from litellm import responses
+
+question = "Write a poem about a boy and his first pet dog."
+
+for verbosity in ["low", "medium", "high"]:
+ response = responses(
+ model="gpt-5-mini",
+ input=question,
+ text={"verbosity": verbosity}
+ )
+
+ print(response)
+```
+
+
+
+
+```python
+from openai import OpenAI
+import pandas as pd
+from IPython.display import display
+
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+question = "Write a poem about a boy and his first pet dog."
+
+data = []
+
+for verbosity in ["low", "medium", "high"]:
+ response = client.responses.create(
+ model="gpt-5-mini",
+ input=question,
+ text={"verbosity": verbosity}
+ )
+
+ # Extract text
+ output_text = ""
+ for item in response.output:
+ if hasattr(item, "content"):
+ for content in item.content:
+ if hasattr(content, "text"):
+ output_text += content.text
+
+ usage = response.usage
+ data.append({
+ "Verbosity": verbosity,
+ "Sample Output": output_text,
+ "Output Tokens": usage.output_tokens
+ })
+
+# Create DataFrame
+df = pd.DataFrame(data)
+
+# Display nicely with centered headers
+pd.set_option('display.max_colwidth', None)
+styled_df = df.style.set_table_styles(
+ [
+ {'selector': 'th', 'props': [('text-align', 'center')]}, # Center column headers
+ {'selector': 'td', 'props': [('text-align', 'left')]} # Left-align table cells
+ ]
+)
+
+display(styled_df)
+
+```
+
+
+
+
+
+## Free-form Function Calling
+
+
+
+
+
+```python showLineNumbers title="Free-form Function Calling"
+import litellm
+
+response = litellm.responses(
+ response = client.responses.create(
+ model="gpt-5-mini",
+ input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
+ text={"format": {"type": "text"}},
+ tools=[
+ {
+ "type": "custom",
+ "name": "code_exec",
+ "description": "Executes arbitrary python code",
+ }
+ ]
+)
+print(response.output)
+```
+
+
+
+
+```python showLineNumbers title="Free-form Function Calling"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+response = client.responses.create(
+ model="gpt-5-mini",
+ input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
+ text={"format": {"type": "text"}},
+ tools=[
+ {
+ "type": "custom",
+ "name": "code_exec",
+ "description": "Executes arbitrary python code",
+ }
+ ]
+)
+print(response.output)
+```
+
+
+
+
+
+## Context-Free Grammar
+
+
+
+
+```python showLineNumbers title="Context-Free Grammar"
+import litellm
+
+import textwrap
+
+# ----------------- grammars for MS SQL dialect -----------------
+mssql_grammar = textwrap.dedent(r"""
+ // ---------- Punctuation & operators ----------
+ SP: " "
+ COMMA: ","
+ GT: ">"
+ EQ: "="
+ SEMI: ";"
+
+ // ---------- Start ----------
+ start: "SELECT" SP "TOP" SP NUMBER SP select_list SP "FROM" SP table SP "WHERE" SP amount_filter SP "AND" SP date_filter SP "ORDER" SP "BY" SP sort_cols SEMI
+
+ // ---------- Projections ----------
+ select_list: column (COMMA SP column)*
+ column: IDENTIFIER
+
+ // ---------- Tables ----------
+ table: IDENTIFIER
+
+ // ---------- Filters ----------
+ amount_filter: "total_amount" SP GT SP NUMBER
+ date_filter: "order_date" SP GT SP DATE
+
+ // ---------- Sorting ----------
+ sort_cols: "order_date" SP "DESC"
+
+ // ---------- Terminals ----------
+ IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
+ NUMBER: /[0-9]+/
+ DATE: /'[0-9]{4}-[0-9]{2}-[0-9]{2}'/
+ """)
+
+sql_prompt_mssql = (
+ "Call the mssql_grammar to generate a query for Microsoft SQL Server that retrieve the "
+ "five most recent orders per customer, showing customer_id, order_id, order_date, and total_amount, "
+ "where total_amount > 500 and order_date is after '2025-01-01'. "
+)
+
+
+response = litellm.responses(
+ model="gpt-5",
+ input=sql_prompt_mssql,
+ text={"format": {"type": "text"}},
+ tools=[
+ {
+ "type": "custom",
+ "name": "mssql_grammar",
+ "description": "Executes read-only Microsoft SQL Server queries limited to SELECT statements with TOP and basic WHERE/ORDER BY. YOU MUST REASON HEAVILY ABOUT THE QUERY AND MAKE SURE IT OBEYS THE GRAMMAR.",
+ "format": {
+ "type": "grammar",
+ "syntax": "lark",
+ "definition": mssql_grammar
+ }
+ },
+ ],
+ parallel_tool_calls=False
+)
+
+print("--- MS SQL Query ---")
+print(response_mssql.output[1].input)
+```
+
+
+
+
+```python showLineNumbers title="Context-Free Grammar"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+import textwrap
+
+# ----------------- grammars for MS SQL dialect -----------------
+mssql_grammar = textwrap.dedent(r"""
+ // ---------- Punctuation & operators ----------
+ SP: " "
+ COMMA: ","
+ GT: ">"
+ EQ: "="
+ SEMI: ";"
+
+ // ---------- Start ----------
+ start: "SELECT" SP "TOP" SP NUMBER SP select_list SP "FROM" SP table SP "WHERE" SP amount_filter SP "AND" SP date_filter SP "ORDER" SP "BY" SP sort_cols SEMI
+
+ // ---------- Projections ----------
+ select_list: column (COMMA SP column)*
+ column: IDENTIFIER
+
+ // ---------- Tables ----------
+ table: IDENTIFIER
+
+ // ---------- Filters ----------
+ amount_filter: "total_amount" SP GT SP NUMBER
+ date_filter: "order_date" SP GT SP DATE
+
+ // ---------- Sorting ----------
+ sort_cols: "order_date" SP "DESC"
+
+ // ---------- Terminals ----------
+ IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
+ NUMBER: /[0-9]+/
+ DATE: /'[0-9]{4}-[0-9]{2}-[0-9]{2}'/
+ """)
+
+sql_prompt_mssql = (
+ "Call the mssql_grammar to generate a query for Microsoft SQL Server that retrieve the "
+ "five most recent orders per customer, showing customer_id, order_id, order_date, and total_amount, "
+ "where total_amount > 500 and order_date is after '2025-01-01'. "
+)
+
+
+response = client.responses.create(
+ model="gpt-5",
+ input=sql_prompt_mssql,
+ text={"format": {"type": "text"}},
+ tools=[
+ {
+ "type": "custom",
+ "name": "mssql_grammar",
+ "description": "Executes read-only Microsoft SQL Server queries limited to SELECT statements with TOP and basic WHERE/ORDER BY. YOU MUST REASON HEAVILY ABOUT THE QUERY AND MAKE SURE IT OBEYS THE GRAMMAR.",
+ "format": {
+ "type": "grammar",
+ "syntax": "lark",
+ "definition": mssql_grammar
+ }
+ },
+ ],
+ parallel_tool_calls=False
+)
+
+print("--- MS SQL Query ---")
+print(response_mssql.output[1].input)
+```
+
+
+
+
+## Minimal Reasoning
+
+
+
+
+
+```python showLineNumbers title="Minimal Reasoning"
+import litellm
+
+response = litellm.responses(
+ model="gpt-5",
+ input= [{ 'role': 'developer', 'content': prompt },
+ { 'role': 'user', 'content': 'The food that the restaurant was great! I recommend it to everyone.' }],
+ reasoning = {
+ "effort": "minimal"
+ },
+)
+
+print(response)
+```
+
+
+
+```python showLineNumbers title="Minimal Reasoning"
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-api-key" # Your proxy API key
+)
+
+
+prompt = "Classify sentiment of the review as positive|neutral|negative. Return one word only."
+
+
+response = client.responses.create(
+ model="gpt-5",
+ input= [{ 'role': 'developer', 'content': prompt },
+ { 'role': 'user', 'content': 'The food that the restaurant was great! I recommend it to everyone.' }],
+ reasoning = {
+ "effort": "minimal"
+ },
+)
+
+# Extract model's text output
+output_text = ""
+for item in response.output:
+ if hasattr(item, "content"):
+ for content in item.content:
+ if hasattr(content, "text"):
+ output_text += content.text
+
+# Token usage details
+usage = response.usage
+
+print("--------------------------------")
+print("Output:")
+print(output_text)
+
+
+
+```
+
+
+
+
diff --git a/docs/my-website/docs/providers/ovhcloud.md b/docs/my-website/docs/providers/ovhcloud.md
new file mode 100644
index 00000000000..6c42208f2cc
--- /dev/null
+++ b/docs/my-website/docs/providers/ovhcloud.md
@@ -0,0 +1,380 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# 🆕 OVHCloud AI Endpoints
+Leading French Cloud provider in Europe with data sovereignty and privacy.
+
+You can explore the last models we made available in our [catalog](https://endpoints.ai.cloud.ovh.net/catalog).
+
+:::tip
+
+We support ALL OVHCloud AI Endpoints models, just set `model=ovhcloud/` as a prefix when sending litellm requests.
+For the complete models catalog, visit https://endpoints.ai.cloud.ovh.net/catalog. **
+
+:::
+
+## Sample usage
+### Chat completion
+You can define your API key by setting the `OVHCLOUD_API_KEY` environment variable or by overriding the `api_key` parameter. You can generate a key on the [OVHCloud Manager](https://www.ovh.com/manager).
+
+```python
+from litellm import completion
+import os
+
+# Our API is free but ratelimited for calls without an API key.
+os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
+
+response = completion(
+ model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
+ messages = [
+ {
+ "role": "user",
+ "content": "Hello, how are you?",
+ }
+ ],
+ max_tokens = 10,
+ stop = [],
+ temperature = 0.2,
+ top_p = 0.9,
+ user = "user",
+ api_key = "your-api-key" # Optional if set through the enviromnent variable.
+)
+
+print(response)
+```
+
+### Streaming
+Set the parameter `stream` to `True` to stream a response.
+```python
+from litellm import completion
+import os
+
+os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
+
+response = completion(
+ model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
+ messages = [
+ {
+ "role": "user",
+ "content": "Hello, how are you?",
+ }
+ ],
+ max_tokens = 10,
+ stop = [],
+ temperature = 0.2,
+ top_p = 0.9,
+ user = "user",
+ api_key = "your-api-key" # Optional if set through the enviromnent variable,
+ stream = True
+)
+
+for part in response:
+ print(response)
+```
+
+### Tool Calling
+
+```python
+from litellm import completion
+import json
+
+def get_current_weather(location, unit="celsius"):
+ if unit == "celsius":
+ return {"location": location, "temperature": "22", "unit": "celsius"}
+ else:
+ return {"location": location, "temperature": "72", "unit": "fahrenheit"}
+
+def print_message(role, content, is_tool_call=False, function_name=None):
+ if role == "user":
+ print(f"🧑 User: {content}")
+ elif role == "assistant":
+ if is_tool_call:
+ print(f"🤖 Assistant: I will call the function '{function_name}' to get some informations.")
+ else:
+ print(f"🤖 Assistant: {content}")
+ elif role == "tool":
+ print(f"🔧 Tool ({function_name}): {content}")
+ print()
+
+messages = [{"role": "user", "content": "What's the weather like in Paris?"}]
+model = "ovhcloud/Meta-Llama-3_3-70B-Instruct"
+
+tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "description": "Get the current weather in a given location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "location": {
+ "type": "string",
+ "description": "The city and country, e.g. Montréal, Canada",
+ },
+ "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+ },
+ "required": ["location"],
+ },
+ },
+ }
+]
+
+print("🌟 Beginning of the conversation")
+
+# Initial user message
+print_message("user", messages[0]["content"])
+
+# First request to the model
+print("📡 Sending first request to the model...")
+response = completion(
+ model=model,
+ messages=messages,
+ tools=tools,
+ tool_choice="auto",
+)
+
+response_message = response.choices[0].message
+tool_calls = response_message.tool_calls
+
+if tool_calls:
+ available_functions = {
+ "get_current_weather": get_current_weather,
+ }
+
+ # Display the tool calls suggested by the model
+ for tool_call in tool_calls:
+ print_message("assistant", "", is_tool_call=True, function_name=tool_call.function.name)
+ print(f" 📋 Arguments: {tool_call.function.arguments}")
+ print()
+
+ # Add assistant message with tool calls to the conversation history
+ assistant_message = {
+ "role": "assistant",
+ "content": response_message.content,
+ "tool_calls": [
+ {
+ "id": tool_call.id,
+ "type": "function",
+ "function": {
+ "name": tool_call.function.name,
+ "arguments": tool_call.function.arguments
+ }
+ } for tool_call in tool_calls
+ ]
+ }
+
+ messages.append(assistant_message)
+
+ # Execute each tool call and add the results to the conversation history
+ for tool_call in tool_calls:
+ function_name = tool_call.function.name
+ function_to_call = available_functions[function_name]
+ function_args = json.loads(tool_call.function.arguments)
+
+ print(f"🔧 Executing function '{function_name}'...")
+ function_response = function_to_call(
+ location=function_args.get("location"),
+ unit=function_args.get("unit"),
+ )
+
+ # Display tool response
+ print_message("tool", json.dumps(function_response, indent=2), function_name=function_name)
+
+ messages.append({
+ "tool_call_id": tool_call.id,
+ "role": "tool",
+ "name": function_name,
+ "content": json.dumps(function_response),
+ })
+
+ print("📡 Sending second request to the model with results...")
+
+ # Second request with function results
+ second_response = completion(
+ model=model,
+ messages=messages
+ )
+
+ # Display final response
+ final_content = second_response.choices[0].message.content
+ print_message("assistant", final_content)
+
+else:
+ print("❌ No function call detected")
+ print_message("assistant", response_message.content)
+```
+
+### Vision Example
+
+```python
+from base64 import b64encode
+from mimetypes import guess_type
+import litellm
+
+# Auxiliary function to get b64 images
+def data_url_from_image(file_path):
+ mime_type, _ = guess_type(file_path)
+ if mime_type is None:
+ raise ValueError("Could not determine MIME type of the file")
+
+ with open(file_path, "rb") as image_file:
+ encoded_string = b64encode(image_file.read()).decode("utf-8")
+
+ data_url = f"data:{mime_type};base64,{encoded_string}"
+ return data_url
+
+response = litellm.completion(
+ model = "ovhcloud/Mistral-Small-3.2-24B-Instruct-2506",
+ messages=[
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What's in this image?"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": data_url_from_image("your_image.jpg"),
+ "format": "image/jpeg"
+ }
+ }
+ ]
+ }
+ ],
+ stream=False
+)
+
+print(response.choices[0].message.content)
+```
+
+
+### Structured Output
+
+```python
+from litellm import completion
+
+response = completion(
+ model="ovhcloud/Meta-Llama-3_3-70B-Instruct",
+ messages=[
+ {
+ "role": "system",
+ "content": (
+ "You are a specialist in extracting structured data from unstructured text. "
+ "Your task is to identify relevant entities and categories, then format them "
+ "according to the requested structure."
+ ),
+ },
+ {
+ "role": "user",
+ "content": "Room 12 contains books, a desk, and a lamp."
+ },
+ ],
+ response_format={
+ "type": "json_schema",
+ "json_schema": {
+ "title": "data",
+ "name": "data_extraction",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "section": {"type": "string"},
+ "products": {
+ "type": "array",
+ "items": {"type": "string"}
+ }
+ },
+ "required": ["section", "products"],
+ "additionalProperties": False
+ },
+ "strict": False
+ }
+ },
+ stream=False
+)
+
+print(response.choices[0].message.content)
+```
+
+### Embeddings
+
+```python
+from litellm import embedding
+
+response = embedding(
+ model="ovhcloud/BGE-M3",
+ input=["sample text to embed", "another sample text to embed"]
+)
+
+print(response.data)
+```
+
+## Usage with LiteLLM Proxy Server
+
+Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server
+
+1. Modify the config.yaml
+
+ ```yaml
+ model_list:
+ - model_name: my-model
+ litellm_params:
+ model: ovhcloud/ # add ovhcloud/ prefix to route as OVHCloud provider
+ api_key: api-key # api key to send your model
+ ```
+
+
+2. Start the proxy
+
+ ```bash
+ $ litellm --config /path/to/config.yaml
+ ```
+
+3. Send Request to LiteLLM Proxy Server
+
+
+
+
+
+ ```python
+ import openai
+ client = openai.OpenAI(
+ api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+ base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+ )
+
+ response = client.chat.completions.create(
+ model="my-model",
+ messages = [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ )
+
+ print(response)
+ ```
+
+
+
+
+ ```shell
+ curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "my-model",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+ ```
+
+
+
diff --git a/docs/my-website/docs/providers/perplexity.md b/docs/my-website/docs/providers/perplexity.md
index 5ef1f8861a6..2fcb49c60fa 100644
--- a/docs/my-website/docs/providers/perplexity.md
+++ b/docs/my-website/docs/providers/perplexity.md
@@ -39,6 +39,69 @@ for chunk in response:
print(chunk)
```
+## Reasoning Effort
+
+Requires v1.72.6+
+
+:::info
+
+See full guide on Reasoning with LiteLLM [here](../reasoning_content)
+
+:::
+
+You can set the reasoning effort by setting the `reasoning_effort` parameter.
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ['PERPLEXITYAI_API_KEY'] = ""
+response = completion(
+ model="perplexity/sonar-reasoning",
+ messages=messages,
+ reasoning_effort="high"
+)
+print(response)
+```
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: perplexity-sonar-reasoning-model
+ litellm_params:
+ model: perplexity/sonar-reasoning
+ api_key: os.environ/PERPLEXITYAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+Replace `anything` with your LiteLLM Proxy Virtual Key, if [setup](../proxy/virtual_keys).
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer anything" \
+ -d '{
+ "model": "perplexity-sonar-reasoning-model",
+ "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}],
+ "reasoning_effort": "high"
+ }'
+```
+
+
+
## Supported Models
All models listed here https://docs.perplexity.ai/docs/model-cards are supported. Just do `model=perplexity/`.
diff --git a/docs/my-website/docs/providers/recraft.md b/docs/my-website/docs/providers/recraft.md
new file mode 100644
index 00000000000..d4a29c38aa0
--- /dev/null
+++ b/docs/my-website/docs/providers/recraft.md
@@ -0,0 +1,303 @@
+# Recraft
+https://www.recraft.ai/
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Recraft is an AI-powered design tool that generates high-quality images with precise control over style and content. |
+| Provider Route on LiteLLM | `recraft/` |
+| Link to Provider Doc | [Recraft ↗](https://www.recraft.ai/docs) |
+| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-edit) |
+
+LiteLLM supports Recraft Image Generation and Image Edit calls.
+
+## API Base, Key
+```python
+# env variable
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+os.environ['RECRAFT_API_BASE'] = "https://external.api.recraft.ai" # [optional]
+```
+
+## Image Generation
+
+### Usage - LiteLLM Python SDK
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+# recraft image generation call
+response = image_generation(
+ model="recraft/recraftv3",
+ prompt="A beautiful sunset over a calm ocean",
+)
+print(response)
+```
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Setup config.yaml
+
+```yaml showLineNumbers
+model_list:
+ - model_name: recraft-v3
+ litellm_params:
+ model: recraft/recraftv3
+ api_key: os.environ/RECRAFT_API_KEY
+ model_info:
+ mode: image_generation
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start the proxy
+
+```bash showLineNumbers
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Test it
+
+```bash showLineNumbers
+curl --location 'http://0.0.0.0:4000/v1/images/generations' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer sk-1234' \
+--data '{
+ "model": "recraft-v3",
+ "prompt": "A beautiful sunset over a calm ocean",
+}'
+```
+
+### Advanced Usage - With Additional Parameters
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+response = image_generation(
+ model="recraft/recraftv3",
+ prompt="A beautiful sunset over a calm ocean",
+)
+print(response)
+```
+
+### Supported Parameters
+
+Recraft supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description | Example |
+|-----------|------|-------------|---------|
+| `n` | integer | Number of images to generate (1-4) | `1` |
+| `response_format` | string | Format of response (`url` or `b64_json`) | `"url"` |
+| `size` | string | Image dimensions | `"1024x1024"` |
+| `style` | string | Image style/artistic direction | `"realistic"` |
+
+### Using Non-OpenAI Parameters
+
+If you want to pass parameters that are not supported by OpenAI, you can pass them in your request body, LiteLLM will automatically route it to recraft.
+
+In this example we will pass `style_id` parameter to the recraft image generation call.
+
+**Usage with LiteLLM Python SDK**
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+response = image_generation(
+ model="recraft/recraftv3",
+ prompt="A beautiful sunset over a calm ocean",
+ style_id="your-style-id",
+)
+```
+
+**Usage with LiteLLM Proxy Server + OpenAI Python SDK**
+
+```python showLineNumbers
+from openai import OpenAI
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+client = OpenAI(api_key=os.environ['RECRAFT_API_KEY'])
+
+response = client.images.generate(
+ model="recraft/recraftv3",
+ prompt="A beautiful sunset over a calm ocean",
+ extra_body={
+ "style_id": "your-style-id",
+ },
+)
+print(response)
+```
+
+### Supported Image Generation Models
+
+**Note: All recraft models are supported by LiteLLM** Just pass the model name with `recraft/` and litellm will route it to recraft.
+
+| Model Name | Function Call |
+|------------|---------------|
+| recraftv3 | `image_generation(model="recraft/recraftv3", prompt="...")` |
+| recraftv2 | `image_generation(model="recraft/recraftv2", prompt="...")` |
+
+For more details on available models and features, see: https://www.recraft.ai/docs
+
+## Image Edit
+
+### Usage - LiteLLM Python SDK
+
+```python showLineNumbers
+from litellm import image_edit
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+# Open the image file
+with open("reference_image.png", "rb") as image_file:
+ # recraft image edit call
+ response = image_edit(
+ model="recraft/recraftv3",
+ prompt="Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO.",
+ image=image_file,
+ )
+print(response)
+```
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Setup config.yaml
+
+```yaml showLineNumbers
+model_list:
+ - model_name: recraft-v3
+ litellm_params:
+ model: recraft/recraftv3
+ api_key: os.environ/RECRAFT_API_KEY
+ model_info:
+ mode: image_edit
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start the proxy
+
+```bash showLineNumbers
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Test it
+
+```bash showLineNumbers
+curl --location 'http://0.0.0.0:4000/v1/images/edits' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'model="recraft-v3"' \
+--form 'prompt="Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO."' \
+--form 'image=@"reference_image.png"'
+```
+
+### Advanced Usage - With Additional Parameters
+
+```python showLineNumbers
+from litellm import image_edit
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+with open("reference_image.png", "rb") as image_file:
+ response = image_edit(
+ model="recraft/recraftv3",
+ prompt="Create a studio ghibli style image",
+ image=image_file,
+ n=2, # Generate 2 variations
+ response_format="url", # Return URLs instead of base64
+ style="realistic_image", # Set artistic style
+ strength=0.5 # Control transformation strength (0-1)
+ )
+print(response)
+```
+
+### Supported Image Edit Parameters
+
+Recraft supports the following OpenAI-compatible parameters for image editing:
+
+| Parameter | Type | Description | Default | Example |
+|-----------|------|-------------|---------|---------|
+| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
+| `response_format` | string | Format of response (`url` or `b64_json`) | `"url"` | `"b64_json"` |
+| `style` | string | Image style/artistic direction | - | `"realistic_image"` |
+| `strength` | float | Controls how much to transform the image (0.0-1.0) | `0.2` | `0.5` |
+
+### Using Non-OpenAI Parameters
+
+You can pass Recraft-specific parameters that are not part of the OpenAI API by including them in your request:
+
+**Usage with LiteLLM Python SDK**
+
+```python showLineNumbers
+from litellm import image_edit
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+with open("reference_image.png", "rb") as image_file:
+ response = image_edit(
+ model="recraft/recraftv3",
+ prompt="Create a studio ghibli style image",
+ image=image_file,
+ style_id="your-style-id", # Recraft-specific parameter
+ strength=0.7
+ )
+```
+
+**Usage with LiteLLM Proxy Server + OpenAI Python SDK**
+
+```python showLineNumbers
+from openai import OpenAI
+import os
+
+client = OpenAI(
+ api_key="sk-1234", # your LiteLLM proxy master key
+ base_url="http://0.0.0.0:4000" # your LiteLLM proxy URL
+)
+
+with open("reference_image.png", "rb") as image_file:
+ response = client.images.edit(
+ model="recraft-v3",
+ prompt="Create a studio ghibli style image",
+ image=image_file,
+ extra_body={
+ "style_id": "your-style-id",
+ "strength": 0.7
+ }
+ )
+print(response)
+```
+
+### Supported Image Edit Models
+
+**Note: All recraft models are supported by LiteLLM** Just pass the model name with `recraft/` and litellm will route it to recraft.
+
+| Model Name | Function Call |
+|------------|---------------|
+| recraftv3 | `image_edit(model="recraft/recraftv3", ...)` |
+
+## API Key Setup
+
+Get your API key from [Recraft's website](https://www.recraft.ai/) and set it as an environment variable:
+
+```bash
+export RECRAFT_API_KEY="your-api-key"
+```
diff --git a/docs/my-website/docs/providers/sambanova.md b/docs/my-website/docs/providers/sambanova.md
index 290b64a1f09..f7be5d3ce77 100644
--- a/docs/my-website/docs/providers/sambanova.md
+++ b/docs/my-website/docs/providers/sambanova.md
@@ -307,3 +307,16 @@ response = litellm.completion(
print(response.choices[0].message.content))
```
+
+## SambaNova - Embeddings
+
+```python
+import litellm
+
+response = litellm.embedding(
+ model="sambanova/E5-Mistral-7B-Instruct",
+ input=["sample text to embed", "another sample text to embed"]
+)
+
+print(response.data)
+```
diff --git a/docs/my-website/docs/providers/snowflake.md b/docs/my-website/docs/providers/snowflake.md
index c708613e2f5..40deef87805 100644
--- a/docs/my-website/docs/providers/snowflake.md
+++ b/docs/my-website/docs/providers/snowflake.md
@@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
| Description | The Snowflake Cortex LLM REST API lets you access the COMPLETE function via HTTP POST requests|
| Provider Route on LiteLLM | `snowflake/` |
| Link to Provider Doc | [Snowflake ↗](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api) |
-| Base URL | [https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete/](https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete) |
+| Base URL | `https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete` |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
diff --git a/docs/my-website/docs/providers/v0.md b/docs/my-website/docs/providers/v0.md
new file mode 100644
index 00000000000..74b6498ca88
--- /dev/null
+++ b/docs/my-website/docs/providers/v0.md
@@ -0,0 +1,340 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# v0
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | v0 provides AI models optimized for code generation, particularly for creating Next.js applications, React components, and modern web development. |
+| Provider Route on LiteLLM | `v0/` |
+| Link to Provider Doc | [v0 API Documentation ↗](https://v0.dev/docs/v0-model-api) |
+| Base URL | `https://api.v0.dev/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+
+
+
+https://v0.dev/docs/v0-model-api
+
+**We support ALL v0 models, just set `v0/` as a prefix when sending completion requests**
+
+## Available Models
+
+| Model | Description | Context Window | Max Output |
+|-------|-------------|----------------|------------|
+| `v0/v0-1.5-lg` | Large model for advanced code generation and reasoning | 512,000 tokens | 512,000 tokens |
+| `v0/v0-1.5-md` | Medium model for everyday code generation tasks | 128,000 tokens | 128,000 tokens |
+| `v0/v0-1.0-md` | Legacy medium model | 128,000 tokens | 128,000 tokens |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["V0_API_KEY"] = "" # your v0 API key from v0.dev
+```
+
+Note: v0 API access requires a Premium or Team plan. Visit [v0.dev/chat/settings/billing](https://v0.dev/chat/settings/billing) to upgrade.
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="v0 Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["V0_API_KEY"] = "" # your v0 API key
+
+messages = [{"content": "Create a React button component with hover effects", "role": "user"}]
+
+# v0 call
+response = completion(
+ model="v0/v0-1.5-md",
+ messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="v0 Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["V0_API_KEY"] = "" # your v0 API key
+
+messages = [{"content": "Create a React button component with hover effects", "role": "user"}]
+
+# v0 call with streaming
+response = completion(
+ model="v0/v0-1.5-md",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+### Vision/Multimodal Support
+
+All v0 models support vision inputs, allowing you to send images along with text:
+
+```python showLineNumbers title="v0 Vision/Multimodal"
+import os
+import litellm
+from litellm import completion
+
+os.environ["V0_API_KEY"] = "" # your v0 API key
+
+messages = [{
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Recreate this UI design in React"
+ },
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/ui-design.png"
+ }
+ }
+ ]
+}]
+
+response = completion(
+ model="v0/v0-1.5-lg",
+ messages=messages
+)
+
+print(response)
+```
+
+### Function Calling
+
+v0 supports function calling for structured outputs:
+
+```python showLineNumbers title="v0 Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["V0_API_KEY"] = "" # your v0 API key
+
+tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "create_component",
+ "description": "Create a React component",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "component_name": {
+ "type": "string",
+ "description": "The name of the component"
+ },
+ "props": {
+ "type": "array",
+ "items": {"type": "string"},
+ "description": "List of component props"
+ }
+ },
+ "required": ["component_name"]
+ }
+ }
+ }
+]
+
+response = completion(
+ model="v0/v0-1.5-md",
+ messages=[{"role": "user", "content": "Create a Button component with onClick and disabled props"}],
+ tools=tools,
+ tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: v0-large
+ litellm_params:
+ model: v0/v0-1.5-lg
+ api_key: os.environ/V0_API_KEY
+
+ - model_name: v0-medium
+ litellm_params:
+ model: v0/v0-1.5-md
+ api_key: os.environ/V0_API_KEY
+
+ - model_name: v0-legacy
+ litellm_params:
+ model: v0/v0-1.0-md
+ api_key: os.environ/V0_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+
+
+
+```python showLineNumbers title="v0 via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+ model="v0-medium",
+ messages=[{"role": "user", "content": "Create a React card component"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="v0 via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+ model="v0-medium",
+ messages=[{"role": "user", "content": "Create a React card component"}],
+ stream=True
+)
+
+for chunk in response:
+ if chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```python showLineNumbers title="v0 via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+ model="litellm_proxy/v0-medium",
+ messages=[{"role": "user", "content": "Create a React card component"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="v0 via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+ model="litellm_proxy/v0-medium",
+ messages=[{"role": "user", "content": "Create a React card component"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key",
+ stream=True
+)
+
+for chunk in response:
+ if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```bash showLineNumbers title="v0 via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "v0-medium",
+ "messages": [{"role": "user", "content": "Create a React card component"}]
+ }'
+```
+
+```bash showLineNumbers title="v0 via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "v0-medium",
+ "messages": [{"role": "user", "content": "Create a React card component"}],
+ "stream": true
+ }'
+```
+
+
+
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
+
+## Supported OpenAI Parameters
+
+v0 supports the following OpenAI-compatible parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
+| `model` | string | **Required**. Model ID (v0-1.5-lg, v0-1.5-md, v0-1.0-md) |
+| `stream` | boolean | Optional. Enable streaming responses |
+| `tools` | array | Optional. List of available tools/functions |
+| `tool_choice` | string/object | Optional. Control tool/function calling |
+
+Note: v0 has a limited set of supported parameters compared to the full OpenAI API. Parameters like `temperature`, `max_tokens`, `top_p`, etc. are not supported.
+
+## Advanced Usage
+
+### Custom API Base
+
+If you're using a custom v0 deployment:
+
+```python showLineNumbers title="Custom API Base"
+import litellm
+
+response = litellm.completion(
+ model="v0/v0-1.5-md",
+ messages=[{"role": "user", "content": "Hello"}],
+ api_base="https://your-custom-v0-endpoint.com/v1",
+ api_key="your-api-key"
+)
+```
+
+
+## Pricing
+
+v0 models require a Premium or Team subscription. Visit [v0.dev/chat/settings/billing](https://v0.dev/chat/settings/billing) for current pricing information.
+
+## Additional Resources
+
+- [v0 Official Documentation](https://v0.dev/docs)
+- [v0 Model API Reference](https://v0.dev/docs/v0-model-api)
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/vercel_ai_gateway.md b/docs/my-website/docs/providers/vercel_ai_gateway.md
new file mode 100644
index 00000000000..91f0a18ea1c
--- /dev/null
+++ b/docs/my-website/docs/providers/vercel_ai_gateway.md
@@ -0,0 +1,219 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vercel AI Gateway
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Vercel AI Gateway provides a unified interface to access multiple AI providers through a single endpoint, with built-in caching, rate limiting, and analytics. |
+| Provider Route on LiteLLM | `vercel_ai_gateway/` |
+| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
+| Base URL | `https://ai-gateway.vercel.sh/v1` |
+| Supported Operations | `/chat/completions`, `/models` |
+
+
+
+
+https://vercel.com/docs/ai-gateway
+
+**We support ALL models available through Vercel AI Gateway, just set `vercel_ai_gateway/` as a prefix when sending completion requests**
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key
+# OR
+os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication
+```
+
+## Optional Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["VERCEL_SITE_URL"] = "" # your site url
+# OR
+os.environ["VERCEL_APP_NAME"] = "" # your app name
+```
+
+Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Vercel AI Gateway Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Vercel AI Gateway call
+response = completion(
+ model="vercel_ai_gateway/openai/gpt-4o",
+ messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Vercel AI Gateway Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Vercel AI Gateway call with streaming
+response = completion(
+ model="vercel_ai_gateway/openai/gpt-4o",
+ messages=messages,
+ stream=True
+)
+
+for chunk in response:
+ print(chunk)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-4o-gateway
+ litellm_params:
+ model: vercel_ai_gateway/openai/gpt-4o
+ api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
+
+ - model_name: claude-4-sonnet-gateway
+ litellm_params:
+ model: vercel_ai_gateway/anthropic/claude-4-sonnet
+ api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+
+
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+ model="gpt-4o-gateway",
+ messages=[{"role": "user", "content": "Hello, how are you?"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+ model="gpt-4o-gateway",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ stream=True
+)
+
+for chunk in response:
+ if chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+ model="litellm_proxy/gpt-4o-gateway",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+ model="litellm_proxy/gpt-4o-gateway",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ api_base="http://localhost:4000",
+ api_key="your-proxy-api-key",
+ stream=True
+)
+
+for chunk in response:
+ if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+ print(chunk.choices[0].delta.content, end="")
+```
+
+
+
+
+
+```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "gpt-4o-gateway",
+ "messages": [{"role": "user", "content": "Hello, how are you?"}]
+ }'
+```
+
+```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer your-proxy-api-key" \
+ -d '{
+ "model": "gpt-4o-gateway",
+ "messages": [{"role": "user", "content": "Hello, how are you?"}],
+ "stream": true
+ }'
+```
+
+
+
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
+
+## Additional Resources
+
+- [Vercel AI Gateway Documentation](https://vercel.com/docs/ai-gateway)
diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md
index dae68f4fc25..3b6562b51ae 100644
--- a/docs/my-website/docs/providers/vertex.md
+++ b/docs/my-website/docs/providers/vertex.md
@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# VertexAI [Anthropic, Gemini, Model Garden]
+# VertexAI [Gemini]
## Overview
@@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem';
| Description | Vertex AI is a fully-managed AI development platform for building and using generative AI. |
| Provider Route on LiteLLM | `vertex_ai/` |
| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) |
-| Base URL | 1. Regional endpoints [https://{vertex_location}-aiplatform.googleapis.com/](https://{vertex_location}-aiplatform.googleapis.com/) 2. Global endpoints (limited availability) [https://aiplatform.googleapis.com/](https://{aiplatform.googleapis.com/)|
+| Base URL | 1. Regional endpoints `https://{vertex_location}-aiplatform.googleapis.com/` 2. Global endpoints (limited availability) `https://aiplatform.googleapis.com/`|
| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models) |
@@ -45,7 +45,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
## COMPLETION CALL
response = completion(
- model="vertex_ai/gemini-pro",
+ model="vertex_ai/gemini-2.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@@ -69,7 +69,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
- model="vertex_ai/gemini-pro",
+ model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@@ -189,13 +189,26 @@ print(json.loads(completion.choices[0].message.content))
1. Add model to config.yaml
```yaml
model_list:
- - model_name: gemini-pro
+ - model_name: gemini-2.5-pro
litellm_params:
- model: vertex_ai/gemini-1.5-pro
+ model: vertex_ai/gemini-2.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
```
+or
+```yaml
+model_list:
+ - model_name: gemini-pro
+ litellm_params:
+ model: vertex_ai/gemini-1.5-pro
+ litellm_credential_name: vertex-global
+ vertex_project: project-name-here
+ vertex_location: global
+ base_model: gemini
+ model_info:
+ provider: Vertex
+```
2. Start Proxy
@@ -210,7 +223,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
- "model": "gemini-pro",
+ "model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@@ -262,9 +275,9 @@ except JSONSchemaValidationError as e:
1. Add model to config.yaml
```yaml
model_list:
- - model_name: gemini-pro
+ - model_name: gemini-2.5-pro
litellm_params:
- model: vertex_ai/gemini-1.5-pro
+ model: vertex_ai/gemini-2.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
@@ -283,7 +296,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
- "model": "gemini-pro",
+ "model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@@ -391,7 +404,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
- model="gemini-pro",
+ model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"googleSearch": {}}],
)
@@ -406,7 +419,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
- "model": "gemini-pro",
+ "model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@@ -424,6 +437,71 @@ curl http://localhost:4000/v1/chat/completions \
+#### **Url Context**
+Using the URL context tool, you can provide Gemini with URLs as additional context for your prompt. The model can then retrieve content from the URLs and use that content to inform and shape its response.
+
+[**Relevant Docs**](https://ai.google.dev/gemini-api/docs/url-context)
+
+See the grounding metadata with `response_obj._hidden_params["vertex_ai_url_context_metadata"]`
+
+
+
+
+```python showLineNumbers
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = ".."
+
+# 👇 ADD URL CONTEXT
+tools = [{"urlContext": {}}]
+
+response = completion(
+ model="gemini/gemini-2.0-flash",
+ messages=[{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+ tools=tools,
+)
+
+print(response)
+
+# Access URL context metadata
+url_context_metadata = response.model_extra['vertex_ai_url_context_metadata']
+urlMetadata = url_context_metadata[0]['urlMetadata'][0]
+print(f"Retrieved URL: {urlMetadata['retrievedUrl']}")
+print(f"Retrieval Status: {urlMetadata['urlRetrievalStatus']}")
+```
+
+
+
+
+1. Setup config.yaml
+```yaml
+model_list:
+ - model_name: gemini-2.0-flash
+ litellm_params:
+ model: gemini/gemini-2.0-flash
+ api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start Proxy
+```bash
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request!
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "gemini-2.0-flash",
+ "messages": [{"role": "user", "content": "Summarize this document: https://ai.google.dev/gemini-api/docs/models"}],
+ "tools": [{"urlContext": {}}]
+ }'
+```
+
+
+
#### **Enterprise Web Search**
You can also use the `enterpriseWebSearch` tool for an [enterprise compliant search](https://cloud.google.com/vertex-ai/generative-ai/docs/grounding/web-grounding-enterprise).
@@ -462,7 +540,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
- model="gemini-pro",
+ model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"enterpriseWebSearch": {}}],
)
@@ -477,7 +555,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
- "model": "gemini-pro",
+ "model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@@ -543,6 +621,163 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+#### **Google Maps**
+
+Use Google Maps to provide location-based context to your Gemini models.
+
+[**Relevant Vertex AI Docs**](https://ai.google.dev/gemini-api/docs/grounding#google-maps)
+
+
+
+
+**Basic Usage - Enable Widget Only**
+
+```python showLineNumbers
+from litellm import completion
+
+## SETUP ENVIRONMENT
+# !gcloud auth application-default login - run this to add vertex credentials to your env
+
+tools = [{"googleMaps": {"enableWidget": "ENABLE_WIDGET"}}] # 👈 ADD GOOGLE MAPS
+
+resp = litellm.completion(
+ model="vertex_ai/gemini-2.0-flash",
+ messages=[{"role": "user", "content": "What restaurants are nearby?"}],
+ tools=tools,
+)
+
+print(resp)
+```
+
+**With Location Data**
+
+You can specify a location to ground the model's responses with location-specific information:
+
+```python showLineNumbers
+from litellm import completion
+
+## SETUP ENVIRONMENT
+# !gcloud auth application-default login - run this to add vertex credentials to your env
+
+tools = [{
+ "googleMaps": {
+ "enableWidget": "ENABLE_WIDGET",
+ "latitude": 37.7749, # San Francisco latitude
+ "longitude": -122.4194, # San Francisco longitude
+ "languageCode": "en_US" # Optional: language for results
+ }
+}] # 👈 ADD GOOGLE MAPS WITH LOCATION
+
+resp = litellm.completion(
+ model="vertex_ai/gemini-2.0-flash",
+ messages=[{"role": "user", "content": "What restaurants are nearby?"}],
+ tools=tools,
+)
+
+print(resp)
+```
+
+
+
+
+
+
+
+**Basic Usage - Enable Widget Only**
+
+```python showLineNumbers
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+ base_url="http://0.0.0.0:4000/v1/" # point to litellm proxy
+)
+
+response = client.chat.completions.create(
+ model="gemini-2.0-flash",
+ messages=[{"role": "user", "content": "What restaurants are nearby?"}],
+ tools=[{"googleMaps": {"enableWidget": "ENABLE_WIDGET"}}],
+)
+
+print(response)
+```
+
+**With Location Data**
+
+```python showLineNumbers
+from openai import OpenAI
+
+client = OpenAI(
+ api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
+ base_url="http://0.0.0.0:4000/v1/" # point to litellm proxy
+)
+
+response = client.chat.completions.create(
+ model="gemini-2.0-flash",
+ messages=[{"role": "user", "content": "What restaurants are nearby?"}],
+ tools=[{
+ "googleMaps": {
+ "enableWidget": "ENABLE_WIDGET",
+ "latitude": 37.7749, # San Francisco latitude
+ "longitude": -122.4194, # San Francisco longitude
+ "languageCode": "en_US" # Optional: language for results
+ }
+ }],
+)
+
+print(response)
+```
+
+
+
+**Basic Usage - Enable Widget Only**
+
+```bash showLineNumbers
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gemini-2.0-flash",
+ "messages": [
+ {"role": "user", "content": "What restaurants are nearby?"}
+ ],
+ "tools": [
+ {
+ "googleMaps": {"enableWidget": "ENABLE_WIDGET"}
+ }
+ ]
+ }'
+```
+
+**With Location Data**
+
+```bash showLineNumbers
+curl http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer sk-1234" \
+ -d '{
+ "model": "gemini-2.0-flash",
+ "messages": [
+ {"role": "user", "content": "What restaurants are nearby?"}
+ ],
+ "tools": [
+ {
+ "googleMaps": {
+ "enableWidget": "ENABLE_WIDGET",
+ "latitude": 37.7749,
+ "longitude": -122.4194,
+ "languageCode": "en_US"
+ }
+ }
+ ]
+ }'
+```
+
+
+
+
+
+
#### **Moving from Vertex AI SDK to LiteLLM (GROUNDING)**
@@ -597,10 +832,13 @@ print(resp)
LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
+Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests.
+
**Mapping**
| reasoning_effort | thinking |
| ---------------- | -------- |
+| "disable" | "budget_tokens": 0 |
| "low" | "budget_tokens": 1024 |
| "medium" | "budget_tokens": 2048 |
| "high" | "budget_tokens": 4096 |
@@ -743,10 +981,228 @@ curl http://0.0.0.0:4000/v1/chat/completions \
### **Context Caching**
-Use Vertex AI context caching is supported by calling provider api directly. (Unified Endpoint support coming soon.).
+#### Unified Endpoint
+
+Use Vertex AI context caching in the same way as [**Google AI Studio - Context Caching**](../providers/gemini.md#context-caching)
+
+
+##### Example usage
+
+
+
+
+```python
+from litellm import completion
+
+for _ in range(2):
+ resp = completion(
+ model="vertex_ai/gemini-2.5-pro",
+ messages=[
+ # System Message
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Here is the full text of a complex legal agreement" * 4000,
+ "cache_control": {"type": "ephemeral"}, # 👈 KEY CHANGE
+ }
+ ],
+ },
+ # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What are the key terms and conditions in this agreement?",
+ "cache_control": {"type": "ephemeral"},
+ }
+ ],
+ }]
+ )
+
+ print(resp.usage) # 👈 2nd usage block will be less, since cached tokens used
+```
+
+
+
+
+```python
+from litellm import completion
+
+# Cache for 2 hours (7200 seconds)
+resp = completion(
+ model="vertex_ai/gemini-2.5-pro",
+ messages=[
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Here is the full text of a complex legal agreement" * 4000,
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "7200s" # 👈 Cache for 2 hours
+ },
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What are the key terms and conditions in this agreement?",
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "3600s" # 👈 This TTL will be ignored (first one is used)
+ },
+ }
+ ],
+ }
+ ]
+)
+
+print(resp.usage)
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: gemini-2.5-pro
+ litellm_params:
+ model: vertex_ai/gemini-2.5-pro
+ vertex_project: "project-id"
+ vertex_location: "us-central1"
+ vertex_credentials: "/path/to/service_account.json"
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model": "gemini-2.5-flash",
+ "messages": [
+ {
+ "role": "system",
+ "content": [
+ {
+ "type": "text",
+ "text": "Long cache message (must be >= 1024 tokens)",
+ "cache_control": {
+ "type": "ephemeral",
+ "ttl": "7200s"
+ }
+ }
+ ]
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What is the text about?"
+ }
+ ]
+ }
+ ]
+}'
+
+```
+
+
+
+
+#### Calling provider api directly
[**Go straight to provider**](../pass_through/vertex_ai.md#context-caching)
+##### 1. Create the Cache
+
+First, create the cache by sending a `POST` request to the `cachedContents` endpoint via the LiteLLM proxy.
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/vertex_ai/v1/projects/{project_id}/locations/{location}/cachedContents \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+ "displayName": "example_cache",
+ "contents": [{
+ "role": "user",
+ "parts": [{
+ "text": ".... a long book to be cached"
+ }]
+ }]
+ }'
+```
+
+
+
+
+##### 2. Get the Cache Name from the Response
+
+Vertex AI will return a response containing the `name` of the cached content. This name is the identifier for your cached data.
+
+```json
+{
+ "name": "projects/12341234/locations/{location}/cachedContents/123123123123123",
+ "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+ "createTime": "2025-09-23T19:13:50.674976Z",
+ "updateTime": "2025-09-23T19:13:50.674976Z",
+ "expireTime": "2025-09-23T20:13:50.655988Z",
+ "displayName": "example_cache",
+ "usageMetadata": {
+ "totalTokenCount": 1246,
+ "textCount": 5132
+ }
+}
+```
+
+##### 3. Use the Cached Content
+
+Use the `name` from the response as `cachedContent` or `cached_content` in subsequent API calls to reuse the cached information. This is passed in the body of your request to `/chat/completions`.
+
+
+
+
+```bash
+
+curl http://0.0.0.0:4000/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "cachedContent": "projects/545201925769/locations/us-central1/cachedContents/4511135542628319232",
+ "model": "gemini-2.5-flash",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what is the book about?"
+ }
+ ]
+ }'
+```
+
+
+
## Pre-requisites
* `pip install google-cloud-aiplatform` (pre-installed on proxy docker image)
@@ -767,7 +1223,7 @@ import litellm
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
-response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
+response = litellm.completion(model="gemini-2.5-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
```
## Usage with LiteLLM Proxy Server
@@ -808,9 +1264,9 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
vertex_location: "us-central1" # proj location
model_list:
- -model_name: team1-gemini-pro
+ -model_name: team1-gemini-2.5-pro
litellm_params:
- model: gemini-pro
+ model: gemini-2.5-pro
```
@@ -837,7 +1293,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
)
response = client.chat.completions.create(
- model="team1-gemini-pro",
+ model="team1-gemini-2.5-pro",
messages = [
{
"role": "user",
@@ -857,7 +1313,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
- "model": "team1-gemini-pro",
+ "model": "team1-gemini-2.5-pro",
"messages": [
{
"role": "user",
@@ -907,7 +1363,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
- model="vertex_ai/gemini-pro",
+ model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json,
vertex_project="my-special-project",
@@ -971,7 +1427,7 @@ In certain use-cases you may need to make calls to the models and pass [safety s
```python
response = completion(
- model="vertex_ai/gemini-pro",
+ model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
safety_settings=[
{
@@ -1085,7 +1541,7 @@ litellm.vertex_ai_safety_settings = [
},
]
response = completion(
- model="vertex_ai/gemini-pro",
+ model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
)
```
@@ -1140,539 +1596,13 @@ os.environ["VERTEXAI_LOCATION"] = "us-central1 # Your Location
# set directly on module
litellm.vertex_location = "us-central1 # Your Location
```
-## Anthropic
-| Model Name | Function Call |
-|------------------|--------------------------------------|
-| claude-3-opus@20240229 | `completion('vertex_ai/claude-3-opus@20240229', messages)` |
-| claude-3-5-sonnet@20240620 | `completion('vertex_ai/claude-3-5-sonnet@20240620', messages)` |
-| claude-3-sonnet@20240229 | `completion('vertex_ai/claude-3-sonnet@20240229', messages)` |
-| claude-3-haiku@20240307 | `completion('vertex_ai/claude-3-haiku@20240307', messages)` |
-| claude-3-7-sonnet@20250219 | `completion('vertex_ai/claude-3-7-sonnet@20250219', messages)` |
-
-### Usage
-
-
-
-
-```python
-from litellm import completion
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
-
-model = "claude-3-sonnet@20240229"
-
-vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
-vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
-
-response = completion(
- model="vertex_ai/" + model,
- messages=[{"role": "user", "content": "hi"}],
- temperature=0.7,
- vertex_ai_project=vertex_ai_project,
- vertex_ai_location=vertex_ai_location,
-)
-print("\nModel Response", response)
-```
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: anthropic-vertex
- litellm_params:
- model: vertex_ai/claude-3-sonnet@20240229
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
- - model_name: anthropic-vertex
- litellm_params:
- model: vertex_ai/claude-3-sonnet@20240229
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
- --header 'Authorization: Bearer sk-1234' \
- --header 'Content-Type: application/json' \
- --data '{
- "model": "anthropic-vertex", # 👈 the 'model_name' in config
- "messages": [
- {
- "role": "user",
- "content": "what llm are you"
- }
- ],
- }'
-```
-
-
-
-
-
-
-### Usage - `thinking` / `reasoning_content`
-
-
-
-
-
-```python
-from litellm import completion
-
-resp = completion(
- model="vertex_ai/claude-3-7-sonnet-20250219",
- messages=[{"role": "user", "content": "What is the capital of France?"}],
- thinking={"type": "enabled", "budget_tokens": 1024},
-)
-
-```
-
-
-
-
-
-1. Setup config.yaml
-
-```yaml
-- model_name: claude-3-7-sonnet-20250219
- litellm_params:
- model: vertex_ai/claude-3-7-sonnet-20250219
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
- -H "Content-Type: application/json" \
- -H "Authorization: Bearer " \
- -d '{
- "model": "claude-3-7-sonnet-20250219",
- "messages": [{"role": "user", "content": "What is the capital of France?"}],
- "thinking": {"type": "enabled", "budget_tokens": 1024}
- }'
-```
-
-
-
-
-
-**Expected Response**
-
-```python
-ModelResponse(
- id='chatcmpl-c542d76d-f675-4e87-8e5f-05855f5d0f5e',
- created=1740470510,
- model='claude-3-7-sonnet-20250219',
- object='chat.completion',
- system_fingerprint=None,
- choices=[
- Choices(
- finish_reason='stop',
- index=0,
- message=Message(
- content="The capital of France is Paris.",
- role='assistant',
- tool_calls=None,
- function_call=None,
- provider_specific_fields={
- 'citations': None,
- 'thinking_blocks': [
- {
- 'type': 'thinking',
- 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.',
- 'signature': 'EuYBCkQYAiJAy6...'
- }
- ]
- }
- ),
- thinking_blocks=[
- {
- 'type': 'thinking',
- 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.',
- 'signature': 'EuYBCkQYAiJAy6AGB...'
- }
- ],
- reasoning_content='The capital of France is Paris. This is a very straightforward factual question.'
- )
- ],
- usage=Usage(
- completion_tokens=68,
- prompt_tokens=42,
- total_tokens=110,
- completion_tokens_details=None,
- prompt_tokens_details=PromptTokensDetailsWrapper(
- audio_tokens=None,
- cached_tokens=0,
- text_tokens=None,
- image_tokens=None
- ),
- cache_creation_input_tokens=0,
- cache_read_input_tokens=0
- )
-)
-```
-
-
-
-## Meta/Llama API
-
-| Model Name | Function Call |
-|------------------|--------------------------------------|
-| meta/llama-3.2-90b-vision-instruct-maas | `completion('vertex_ai/meta/llama-3.2-90b-vision-instruct-maas', messages)` |
-| meta/llama3-8b-instruct-maas | `completion('vertex_ai/meta/llama3-8b-instruct-maas', messages)` |
-| meta/llama3-70b-instruct-maas | `completion('vertex_ai/meta/llama3-70b-instruct-maas', messages)` |
-| meta/llama3-405b-instruct-maas | `completion('vertex_ai/meta/llama3-405b-instruct-maas', messages)` |
-| meta/llama-4-scout-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas', messages)` |
-| meta/llama-4-scout-17-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-128b-16e-instruct-maas', messages)` |
-| meta/llama-4-maverick-17b-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas',messages)` |
-| meta/llama-4-maverick-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas',messages)` |
-
-### Usage
-
-
-
-
-```python
-from litellm import completion
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
-
-model = "meta/llama3-405b-instruct-maas"
-
-vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
-vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
-
-response = completion(
- model="vertex_ai/" + model,
- messages=[{"role": "user", "content": "hi"}],
- vertex_ai_project=vertex_ai_project,
- vertex_ai_location=vertex_ai_location,
-)
-print("\nModel Response", response)
-```
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: anthropic-llama
- litellm_params:
- model: vertex_ai/meta/llama3-405b-instruct-maas
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
- - model_name: anthropic-llama
- litellm_params:
- model: vertex_ai/meta/llama3-405b-instruct-maas
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
- --header 'Authorization: Bearer sk-1234' \
- --header 'Content-Type: application/json' \
- --data '{
- "model": "anthropic-llama", # 👈 the 'model_name' in config
- "messages": [
- {
- "role": "user",
- "content": "what llm are you"
- }
- ],
- }'
-```
-
-
-
-
-## Mistral API
-
-[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137)
-
-| Model Name | Function Call |
-|------------------|--------------------------------------|
-| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` |
-| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` |
-| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` |
-| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` |
-| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` |
-
-### Usage
-
-
-
-
-```python
-from litellm import completion
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
-
-model = "mistral-large@2407"
-
-vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
-vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
-
-response = completion(
- model="vertex_ai/" + model,
- messages=[{"role": "user", "content": "hi"}],
- vertex_ai_project=vertex_ai_project,
- vertex_ai_location=vertex_ai_location,
-)
-print("\nModel Response", response)
-```
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: vertex-mistral
- litellm_params:
- model: vertex_ai/mistral-large@2407
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
- - model_name: vertex-mistral
- litellm_params:
- model: vertex_ai/mistral-large@2407
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
- --header 'Authorization: Bearer sk-1234' \
- --header 'Content-Type: application/json' \
- --data '{
- "model": "vertex-mistral", # 👈 the 'model_name' in config
- "messages": [
- {
- "role": "user",
- "content": "what llm are you"
- }
- ],
- }'
-```
-
-
-
-
-
-### Usage - Codestral FIM
-
-Call Codestral on VertexAI via the OpenAI [`/v1/completion`](https://platform.openai.com/docs/api-reference/completions/create) endpoint for FIM tasks.
-
-Note: You can also call Codestral via `/chat/completion`.
-
-
-
-
-```python
-from litellm import completion
-import os
-
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
-# OR run `!gcloud auth print-access-token` in your terminal
-
-model = "codestral@2405"
-
-vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
-vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
-
-response = text_completion(
- model="vertex_ai/" + model,
- vertex_ai_project=vertex_ai_project,
- vertex_ai_location=vertex_ai_location,
- prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
- suffix="return True", # optional
- temperature=0, # optional
- top_p=1, # optional
- max_tokens=10, # optional
- min_tokens=10, # optional
- seed=10, # optional
- stop=["return"], # optional
-)
-
-print("\nModel Response", response)
-```
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: vertex-codestral
- litellm_params:
- model: vertex_ai/codestral@2405
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
- - model_name: vertex-codestral
- litellm_params:
- model: vertex_ai/codestral@2405
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl -X POST 'http://0.0.0.0:4000/completions' \
- -H 'Authorization: Bearer sk-1234' \
- -H 'Content-Type: application/json' \
- -d '{
- "model": "vertex-codestral", # 👈 the 'model_name' in config
- "prompt": "def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
- "suffix":"return True", # optional
- "temperature":0, # optional
- "top_p":1, # optional
- "max_tokens":10, # optional
- "min_tokens":10, # optional
- "seed":10, # optional
- "stop":["return"], # optional
- }'
-```
-
-
-
-
-
-## AI21 Models
-
-| Model Name | Function Call |
-|------------------|--------------------------------------|
-| jamba-1.5-mini@001 | `completion(model='vertex_ai/jamba-1.5-mini@001', messages)` |
-| jamba-1.5-large@001 | `completion(model='vertex_ai/jamba-1.5-large@001', messages)` |
-
-### Usage
-
-
-
-
-```python
-from litellm import completion
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
-
-model = "meta/jamba-1.5-mini@001"
-
-vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
-vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
-
-response = completion(
- model="vertex_ai/" + model,
- messages=[{"role": "user", "content": "hi"}],
- vertex_ai_project=vertex_ai_project,
- vertex_ai_location=vertex_ai_location,
-)
-print("\nModel Response", response)
-```
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: jamba-1.5-mini
- litellm_params:
- model: vertex_ai/jamba-1.5-mini@001
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
- - model_name: jamba-1.5-large
- litellm_params:
- model: vertex_ai/jamba-1.5-large@001
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-west-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
- --header 'Authorization: Bearer sk-1234' \
- --header 'Content-Type: application/json' \
- --data '{
- "model": "jamba-1.5-large",
- "messages": [
- {
- "role": "user",
- "content": "what llm are you"
- }
- ],
- }'
-```
-
-
-
-
## Gemini Pro
| Model Name | Function Call |
|------------------|--------------------------------------|
-| gemini-pro | `completion('gemini-pro', messages)`, `completion('vertex_ai/gemini-pro', messages)` |
+| gemini-2.5-pro | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` |
+| gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
+| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
## Fine-tuned Models
@@ -1764,123 +1694,10 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
-
-
-## Model Garden
-
-:::tip
-
-All OpenAI compatible models from Vertex Model Garden are supported.
-
-:::
-
-#### Using Model Garden
-
-**Almost all Vertex Model Garden models are OpenAI compatible.**
-
-
-
-
-
-| Property | Details |
-|----------|---------|
-| Provider Route | `vertex_ai/openai/{MODEL_ID}` |
-| Vertex Documentation | [Vertex Model Garden - OpenAI Chat Completions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_gradio_streaming_chat_completions.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
-| Supported Operations | `/chat/completions`, `/embeddings` |
-
-
-
-
-```python
-from litellm import completion
-import os
-
-## set ENV variables
-os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
-os.environ["VERTEXAI_LOCATION"] = "us-central1"
-
-response = completion(
- model="vertex_ai/openai/",
- messages=[{ "content": "Hello, how are you?","role": "user"}]
-)
-```
-
-
-
-
-
-
-**1. Add to config**
-
-```yaml
-model_list:
- - model_name: llama3-1-8b-instruct
- litellm_params:
- model: vertex_ai/openai/5464397967697903616
- vertex_ai_project: "my-test-project"
- vertex_ai_location: "us-east-1"
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
- --header 'Authorization: Bearer sk-1234' \
- --header 'Content-Type: application/json' \
- --data '{
- "model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config
- "messages": [
- {
- "role": "user",
- "content": "what llm are you"
- }
- ],
- }'
-```
-
-
-
-
-
-
-
-
-
-
-
-
-```python
-from litellm import completion
-import os
-
-## set ENV variables
-os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
-os.environ["VERTEXAI_LOCATION"] = "us-central1"
-
-response = completion(
- model="vertex_ai/",
- messages=[{ "content": "Hello, how are you?","role": "user"}]
-)
-```
-
-
-
-
-
-
-
## Gemini Pro Vision
| Model Name | Function Call |
|------------------|--------------------------------------|
-| gemini-pro-vision | `completion('gemini-pro-vision', messages)`, `completion('vertex_ai/gemini-pro-vision', messages)`|
+| gemini-2.5-pro-vision | `completion('gemini-2.5-pro-vision', messages)`, `completion('vertex_ai/gemini-2.5-pro-vision', messages)`|
## Gemini 1.5 Pro (and Vision)
| Model Name | Function Call |
@@ -1894,7 +1711,7 @@ response = completion(
#### Using Gemini Pro Vision
-Call `gemini-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
+Call `gemini-2.5-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
LiteLLM Supports the following image types passed in `url`
- Images with Cloud Storage URIs - gs://cloud-samples-data/generative-ai/image/boats.jpeg
@@ -1912,7 +1729,7 @@ LiteLLM Supports the following image types passed in `url`
import litellm
response = litellm.completion(
- model = "vertex_ai/gemini-pro-vision",
+ model = "vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@@ -1950,7 +1767,7 @@ image_path = "cached_logo.jpg"
# Getting the base64 string
base64_image = encode_image(image_path)
response = litellm.completion(
- model="vertex_ai/gemini-pro-vision",
+ model="vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@@ -2006,7 +1823,7 @@ tools = [
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
- model="vertex_ai/gemini-pro-vision",
+ model="vertex_ai/gemini-2.5-pro-vision",
messages=messages,
tools=tools,
)
@@ -2734,44 +2551,132 @@ print(response)
-## **Image Generation Models**
+## **Gemini TTS (Text-to-Speech) Audio Output**
-Usage
+:::info
+
+LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format.
+
+:::
+
+### Supported Models
+
+LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
+
+### Limitations
+
+:::warning
+
+**Important Limitations**:
+- Gemini TTS models only support the `pcm16` audio format
+- **Streaming support has not been added** to TTS models yet
+- The `modalities` parameter must be set to `['audio']` for TTS requests
+
+:::
+
+### Quick Start
+
+
+
```python
-response = await litellm.aimage_generation(
- prompt="An olympic size swimming pool",
- model="vertex_ai/imagegeneration@006",
- vertex_ai_project="adroit-crow-413218",
- vertex_ai_location="us-central1",
+from litellm import completion
+import json
+
+## GET CREDENTIALS
+file_path = 'path/to/vertex_ai_service_account.json'
+
+# Load the JSON file
+with open(file_path, 'r') as file:
+ vertex_credentials = json.load(file)
+
+# Convert to JSON string
+vertex_credentials_json = json.dumps(vertex_credentials)
+
+response = completion(
+ model="vertex_ai/gemini-2.5-flash-preview-tts",
+ messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+ modalities=["audio"], # Required for TTS models
+ audio={
+ "voice": "Kore",
+ "format": "pcm16" # Required: must be "pcm16"
+ },
+ vertex_credentials=vertex_credentials_json
+)
+
+print(response)
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: gemini-tts-flash
+ litellm_params:
+ model: vertex_ai/gemini-2.5-flash-preview-tts
+ vertex_project: "your-project-id"
+ vertex_location: "us-central1"
+ vertex_credentials: "/path/to/service_account.json"
+ - model_name: gemini-tts-pro
+ litellm_params:
+ model: vertex_ai/gemini-2.5-pro-preview-tts
+ vertex_project: "your-project-id"
+ vertex_location: "us-central1"
+ vertex_credentials: "/path/to/service_account.json"
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Make TTS request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "gemini-tts-flash",
+ "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+ "modalities": ["audio"],
+ "audio": {
+ "voice": "Kore",
+ "format": "pcm16"
+ }
+ }'
+```
+
+
+
+
+### Advanced Usage
+
+You can combine TTS with other Gemini features:
+
+```python
+response = completion(
+ model="vertex_ai/gemini-2.5-pro-preview-tts",
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+ {"role": "user", "content": "Explain quantum computing in simple terms"}
+ ],
+ modalities=["audio"],
+ audio={
+ "voice": "Charon",
+ "format": "pcm16"
+ },
+ temperature=0.7,
+ max_tokens=150,
+ vertex_credentials=vertex_credentials_json
)
```
-**Generating multiple images**
-
-Use the `n` parameter to pass how many images you want generated
-```python
-response = await litellm.aimage_generation(
- prompt="An olympic size swimming pool",
- model="vertex_ai/imagegeneration@006",
- vertex_ai_project="adroit-crow-413218",
- vertex_ai_location="us-central1",
- n=1,
-)
-```
-
-### Supported Image Generation Models
-
-| Model Name | FUsage |
-|------------------------------|--------------------------------------------------------------|
-| `imagen-3.0-generate-001` | `litellm.image_generation('vertex_ai/imagen-3.0-generate-001', prompt)` |
-| `imagen-3.0-fast-generate-001` | `litellm.image_generation('vertex_ai/imagen-3.0-fast-generate-001', prompt)` |
-| `imagegeneration@006` | `litellm.image_generation('vertex_ai/imagegeneration@006', prompt)` |
-| `imagegeneration@005` | `litellm.image_generation('vertex_ai/imagegeneration@005', prompt)` |
-| `imagegeneration@002` | `litellm.image_generation('vertex_ai/imagegeneration@002', prompt)` |
-
-
-
+For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
## **Text to Speech APIs**
@@ -2994,150 +2899,6 @@ print("response from proxy", response)
-## **Batch APIs**
-
-Just add the following Vertex env vars to your environment.
-
-```bash
-# GCS Bucket settings, used to store batch prediction files in
-export GCS_BUCKET_NAME = "litellm-testing-bucket" # the bucket you want to store batch prediction files in
-export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
-
-# Vertex /batch endpoint settings, used for LLM API requests
-export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
-export VERTEXAI_LOCATION="us-central1" # can be any vertex location
-export VERTEXAI_PROJECT="my-test-project"
-```
-
-### Usage
-
-
-#### 1. Create a file of batch requests for vertex
-
-LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**
-
-Each `body` in the file should be an **OpenAI API request**
-
-Create a file called `vertex_batch_completions.jsonl` in the current working directory, the `model` should be the Vertex AI model name
-```
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
-```
-
-
-#### 2. Upload a File of batch requests
-
-For `vertex_ai` litellm will upload the file to the provided `GCS_BUCKET_NAME`
-
-```python
-import os
-oai_client = OpenAI(
- api_key="sk-1234", # litellm proxy API key
- base_url="http://localhost:4000" # litellm proxy base url
-)
-file_name = "vertex_batch_completions.jsonl" #
-_current_dir = os.path.dirname(os.path.abspath(__file__))
-file_path = os.path.join(_current_dir, file_name)
-file_obj = oai_client.files.create(
- file=open(file_path, "rb"),
- purpose="batch",
- extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use vertex_ai for this file upload
-)
-```
-
-**Expected Response**
-
-```json
-{
- "id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
- "bytes": 416,
- "created_at": 1733392026,
- "filename": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
- "object": "file",
- "purpose": "batch",
- "status": "uploaded",
- "status_details": null
-}
-```
-
-
-
-#### 3. Create a batch
-
-```python
-batch_input_file_id = file_obj.id # use `file_obj` from step 2
-create_batch_response = oai_client.batches.create(
- completion_window="24h",
- endpoint="/v1/chat/completions",
- input_file_id=batch_input_file_id, # example input_file_id = "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/c2b1b785-252b-448c-b180-033c4c63b3ce"
- extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
-)
-```
-
-**Expected Response**
-
-```json
-{
- "id": "3814889423749775360",
- "completion_window": "24hrs",
- "created_at": 1733392026,
- "endpoint": "",
- "input_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
- "object": "batch",
- "status": "validating",
- "cancelled_at": null,
- "cancelling_at": null,
- "completed_at": null,
- "error_file_id": null,
- "errors": null,
- "expired_at": null,
- "expires_at": null,
- "failed_at": null,
- "finalizing_at": null,
- "in_progress_at": null,
- "metadata": null,
- "output_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
- "request_counts": null
-}
-```
-
-#### 4. Retrieve a batch
-
-```python
-retrieved_batch = oai_client.batches.retrieve(
- batch_id=create_batch_response.id,
- extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
-)
-```
-
-**Expected Response**
-
-```json
-{
- "id": "3814889423749775360",
- "completion_window": "24hrs",
- "created_at": 1736500100,
- "endpoint": "",
- "input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
- "object": "batch",
- "status": "completed",
- "cancelled_at": null,
- "cancelling_at": null,
- "completed_at": null,
- "error_file_id": null,
- "errors": null,
- "expired_at": null,
- "expires_at": null,
- "failed_at": null,
- "finalizing_at": null,
- "in_progress_at": null,
- "metadata": null,
- "output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
- "request_counts": null
-}
-```
-
-
## **Fine Tuning APIs**
@@ -3243,6 +3004,44 @@ curl http://localhost:4000/v1/fine_tuning/jobs \
+## Labels
+
+
+Google enables you to add custom metadata to its `generateContent` and `streamGenerateContent` calls.
+This mechanism is useful in Vertex AI because it allows costs and usage tracking over multiple
+different applications or users.
+
+
+### Usage
+
+You can use that feature through LiteLLM by sending `labels` or `metadata` field in your requests.
+
+If the client sets the `labels` field in the request to the LiteLLM,
+the LiteLLM will pass the `labels` field to the Vertex AI backend.
+
+If the client sets the `metadata` field in the request to the LiteLLM and the `labels` field is not set,
+the LiteLLM will create the `labels` field filled with `metadata` key/value pairs for all string values and
+pass it to the Vertex AI backend.
+
+
+Here is an example JSON request demonstrating the labels usage:
+
+```json
+{
+ "model": "gemini-2.0-flash-lite",
+ "messages": [
+ { "role": "user", "content": "respond in 20 words. who are you?" }
+ ],
+ "labels": {
+ "client_app": "acme_comp_financial_app",
+ "department": "finance",
+ "project": "acme_ai"
+ }
+}
+```
+
+
+
## Extra
### Using `GOOGLE_APPLICATION_CREDENTIALS`
@@ -3315,7 +3114,3 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv
s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial
-
-
-
-
diff --git a/docs/my-website/docs/providers/vertex_batch.md b/docs/my-website/docs/providers/vertex_batch.md
new file mode 100644
index 00000000000..4eaa0d69d4b
--- /dev/null
+++ b/docs/my-website/docs/providers/vertex_batch.md
@@ -0,0 +1,264 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## **Batch APIs**
+
+Just add the following Vertex env vars to your environment.
+
+```bash
+# GCS Bucket settings, used to store batch prediction files in
+export GCS_BUCKET_NAME="my-batch-bucket" # the bucket you want to store batch prediction files in
+export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
+
+# Vertex /batch endpoint settings, used for LLM API requests
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
+export VERTEXAI_LOCATION="us-central1" # can be any vertex location
+export VERTEXAI_PROJECT="my-project"
+```
+
+### Usage
+
+Follow this complete workflow: create JSONL file → upload file → create batch → retrieve batch status → get file content
+
+#### 1. Create a JSONL file of batch requests
+
+LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**.
+
+Each `body` in the file should be an **OpenAI API request**.
+
+Create a file called `batch_requests.jsonl` with your requests:
+```jsonl
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+```
+
+#### 2. Upload the file
+
+Upload your JSONL file. For `vertex_ai`, the file will be stored in your configured GCS bucket provided by `GCS_BUCKET_NAME`.
+
+
+
+
+```python showLineNumbers title="upload_file.py"
+from openai import OpenAI
+
+oai_client = OpenAI(
+ api_key="sk-1234", # litellm proxy API key
+ base_url="http://localhost:4000" # litellm proxy base url
+)
+
+file_obj = oai_client.files.create(
+ file=open("batch_requests.jsonl", "rb"),
+ purpose="batch",
+ extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"File uploaded with ID: {file_obj.id}")
+```
+
+
+
+
+```bash showLineNumbers title="Upload File"
+curl --request POST \
+ --url http://localhost:4000/v1/files \
+ --header 'Content-Type: multipart/form-data' \
+ --form purpose=batch \
+ --form file=@batch_requests.jsonl \
+ --form custom_llm_provider=vertex_ai
+```
+
+
+
+
+**Expected Response:**
+
+```json
+{
+ "id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+ "bytes": 416,
+ "created_at": 1758303684,
+ "filename": "litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+ "object": "file",
+ "purpose": "batch",
+ "status": "uploaded",
+ "expires_at": null,
+ "status_details": null
+}
+```
+
+#### 3. Create a batch
+
+Create a batch job using the uploaded file ID.
+
+
+
+
+```python showLineNumbers title="create_batch.py"
+batch_input_file_id = file_obj.id # from step 2
+create_batch_response = oai_client.batches.create(
+ completion_window="24h",
+ endpoint="/v1/chat/completions",
+ input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
+ extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"Batch created with ID: {create_batch_response.id}")
+```
+
+
+
+
+```bash showLineNumbers title="Create Batch Request"
+curl --request POST \
+ --url http://localhost:4000/v1/batches \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+ "endpoint": "/v1/chat/completions",
+ "completion_window": "24h",
+ "custom_llm_provider": "vertex_ai"
+}'
+```
+
+
+
+
+**Expected Response:**
+
+```json
+{
+ "id": "7814463557919047680",
+ "completion_window": "24hrs",
+ "created_at": 1758328011,
+ "endpoint": "",
+ "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+ "object": "batch",
+ "status": "validating",
+ "cancelled_at": null,
+ "cancelling_at": null,
+ "completed_at": null,
+ "error_file_id": null,
+ "errors": null,
+ "expired_at": null,
+ "expires_at": null,
+ "failed_at": null,
+ "finalizing_at": null,
+ "in_progress_at": null,
+ "metadata": null,
+ "output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite",
+ "request_counts": null,
+ "usage": null
+}
+```
+
+#### 4. Retrieve batch status
+
+Check the status of your batch job. The batch will progress through states: `validating` → `in_progress` → `completed`.
+
+
+
+
+```python showLineNumbers title="retrieve_batch.py"
+retrieved_batch = oai_client.batches.retrieve(
+ batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
+ extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"Batch status: {retrieved_batch.status}")
+if retrieved_batch.status == "completed":
+ print(f"Output file: {retrieved_batch.output_file_id}")
+```
+
+
+
+
+```bash showLineNumbers title="Retrieve Batch Status"
+curl --request GET \
+ --url 'http://localhost:4000/batches/7814463557919047680?provider=vertex_ai' \
+ --header 'Authorization: Bearer sk-1234'
+```
+
+
+
+
+**Expected Response (when completed):**
+
+```json
+{
+ "id": "7814463557919047680",
+ "completion_window": "24hrs",
+ "created_at": 1758328011,
+ "endpoint": "",
+ "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+ "object": "batch",
+ "status": "completed",
+ "cancelled_at": null,
+ "cancelling_at": null,
+ "completed_at": null,
+ "error_file_id": null,
+ "errors": null,
+ "expired_at": null,
+ "expires_at": null,
+ "failed_at": null,
+ "finalizing_at": null,
+ "in_progress_at": null,
+ "metadata": null,
+ "output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/prediction-model-2025-09-19T21:26:51.569037Z/predictions.jsonl",
+ "request_counts": null,
+ "usage": null
+}
+```
+
+#### 5. Get file content
+
+Once the batch is completed, retrieve the results using the `output_file_id` from the batch response.
+
+**Important:** The `output_file_id` must be URL encoded when used in the request path.
+
+
+
+
+```python showLineNumbers title="get_file_content.py"
+import urllib.parse
+import json
+
+output_file_id = retrieved_batch.output_file_id
+# URL encode the file ID
+encoded_file_id = urllib.parse.quote_plus(output_file_id)
+
+# Get file content
+file_content = oai_client.files.content(
+ file_id=encoded_file_id,
+ extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+# Process the results
+for line in file_content.text.strip().split('\n'):
+ result = json.loads(line)
+ print(f"Request: {result['request']}")
+ print(f"Response: {result['response']}")
+ print("---")
+```
+
+
+
+
+```bash showLineNumbers title="Get File Content"
+# Note: The file ID must be URL encoded
+curl --request GET \
+ --url 'http://localhost:4000/files/gs%253A%252F%252Fmy-batch-bucket%252Flitellm-vertex-files%252Fpublishers%252Fgoogle%252Fmodels%252Fgemini-2.5-flash-lite%252Fprediction-model-2025-09-19T21%253A26%253A51.569037Z%252Fpredictions.jsonl/content?provider=vertex_ai' \
+ --header 'Authorization: Bearer sk-1234'
+```
+
+
+
+
+**Expected Response:**
+
+The response contains JSONL format with one result per line:
+
+```jsonl
+{"status":"","processed_time":"2025-09-19T21:29:47.352+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are a helpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.48079710006713866,"content":{"parts":[{"text":"Hello there! It's nice to meet you"}],"role":"model"},"finishReason":"MAX_TOKENS"}],"createTime":"2025-09-19T21:29:47.484619Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaIvKHdvshMIP_aOtuAg","usageMetadata":{"candidatesTokenCount":10,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":10}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":19,"trafficType":"ON_DEMAND"}}}
+{"status":"","processed_time":"2025-09-19T21:29:47.358+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are an unhelpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.6168075137668185,"content":{"parts":[{"text":"I am unable to assist with this request."}],"role":"model"},"finishReason":"STOP"}],"createTime":"2025-09-19T21:29:47.470889Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaOneHISShMIP28nA8QQ","usageMetadata":{"candidatesTokenCount":9,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":9}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":18,"trafficType":"ON_DEMAND"}}}
+```
diff --git a/docs/my-website/docs/providers/vertex_image.md b/docs/my-website/docs/providers/vertex_image.md
new file mode 100644
index 00000000000..27e584cb222
--- /dev/null
+++ b/docs/my-website/docs/providers/vertex_image.md
@@ -0,0 +1,83 @@
+# Vertex AI Image Generation
+
+Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions.
+
+| Property | Details |
+|----------|---------|
+| Description | Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. |
+| Provider Route on LiteLLM | `vertex_ai/` |
+| Provider Doc | [Google Cloud Vertex AI Image Generation ↗](https://cloud.google.com/vertex-ai/docs/generative-ai/image/generate-images) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Basic Image Generation"
+import litellm
+
+# Generate a single image
+response = await litellm.aimage_generation(
+ prompt="An olympic size swimming pool with crystal clear water and modern architecture",
+ model="vertex_ai/imagen-4.0-generate-001",
+ vertex_ai_project="your-project-id",
+ vertex_ai_location="us-central1",
+)
+
+print(response.data[0].url)
+```
+
+### LiteLLM Proxy
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Vertex AI Image Generation Configuration"
+model_list:
+ - model_name: vertex-imagen
+ litellm_params:
+ model: vertex_ai/imagen-4.0-generate-001
+ vertex_ai_project: "your-project-id"
+ vertex_ai_location: "us-central1"
+ vertex_ai_credentials: "path/to/service-account.json" # Optional if using environment auth
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make requests with OpenAI Python SDK
+
+```python showLineNumbers title="Basic Image Generation via Proxy"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+ base_url="http://localhost:4000", # Your proxy URL
+ api_key="your-proxy-api-key" # Your proxy API key
+)
+
+# Generate image
+response = client.images.generate(
+ model="vertex-imagen",
+ prompt="An olympic size swimming pool with crystal clear water and modern architecture",
+)
+
+print(response.data[0].url)
+```
+
+## Supported Models
+
+
+:::tip
+
+**We support ALL Vertex AI Image Generation models, just set `model=vertex_ai/` as a prefix when sending litellm requests**
+
+:::
+
+LiteLLM supports all Vertex AI Imagen models available through Google Cloud.
+
+For the complete and up-to-date list of supported models, visit: [https://models.litellm.ai/](https://models.litellm.ai/)
+
diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md
new file mode 100644
index 00000000000..856f054b8e6
--- /dev/null
+++ b/docs/my-website/docs/providers/vertex_partner.md
@@ -0,0 +1,904 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+# Vertex AI - Anthropic, DeepSeek, Model Garden
+
+## Supported Partner Providers
+
+| Provider | LiteLLM Route | Vertex Documentation |
+|----------|---------------|---------------|
+| Anthropic (Claude) | `vertex_ai/claude-*` | [Vertex AI - Anthropic Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude) |
+| DeepSeek | `vertex_ai/deepseek-ai/{MODEL}` | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) |
+| Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) |
+| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
+| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
+| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
+| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
+| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
+
+## Vertex AI - Anthropic (Claude)
+
+| Model Name | Function Call |
+|------------------|--------------------------------------|
+| claude-3-opus@20240229 | `completion('vertex_ai/claude-3-opus@20240229', messages)` |
+| claude-3-5-sonnet@20240620 | `completion('vertex_ai/claude-3-5-sonnet@20240620', messages)` |
+| claude-3-sonnet@20240229 | `completion('vertex_ai/claude-3-sonnet@20240229', messages)` |
+| claude-3-haiku@20240307 | `completion('vertex_ai/claude-3-haiku@20240307', messages)` |
+| claude-3-7-sonnet@20250219 | `completion('vertex_ai/claude-3-7-sonnet@20250219', messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "claude-3-sonnet@20240229"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ temperature=0.7,
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: anthropic-vertex
+ litellm_params:
+ model: vertex_ai/claude-3-sonnet@20240229
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: anthropic-vertex
+ litellm_params:
+ model: vertex_ai/claude-3-sonnet@20240229
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "anthropic-vertex", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+
+
+#### Usage - `thinking` / `reasoning_content`
+
+
+
+
+
+```python
+from litellm import completion
+
+resp = completion(
+ model="vertex_ai/claude-3-7-sonnet-20250219",
+ messages=[{"role": "user", "content": "What is the capital of France?"}],
+ thinking={"type": "enabled", "budget_tokens": 1024},
+)
+
+```
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+- model_name: claude-3-7-sonnet-20250219
+ litellm_params:
+ model: vertex_ai/claude-3-7-sonnet-20250219
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "claude-3-7-sonnet-20250219",
+ "messages": [{"role": "user", "content": "What is the capital of France?"}],
+ "thinking": {"type": "enabled", "budget_tokens": 1024}
+ }'
+```
+
+
+
+
+
+**Expected Response**
+
+```python
+ModelResponse(
+ id='chatcmpl-c542d76d-f675-4e87-8e5f-05855f5d0f5e',
+ created=1740470510,
+ model='claude-3-7-sonnet-20250219',
+ object='chat.completion',
+ system_fingerprint=None,
+ choices=[
+ Choices(
+ finish_reason='stop',
+ index=0,
+ message=Message(
+ content="The capital of France is Paris.",
+ role='assistant',
+ tool_calls=None,
+ function_call=None,
+ provider_specific_fields={
+ 'citations': None,
+ 'thinking_blocks': [
+ {
+ 'type': 'thinking',
+ 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.',
+ 'signature': 'EuYBCkQYAiJAy6...'
+ }
+ ]
+ }
+ ),
+ thinking_blocks=[
+ {
+ 'type': 'thinking',
+ 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.',
+ 'signature': 'EuYBCkQYAiJAy6AGB...'
+ }
+ ],
+ reasoning_content='The capital of France is Paris. This is a very straightforward factual question.'
+ )
+ ],
+ usage=Usage(
+ completion_tokens=68,
+ prompt_tokens=42,
+ total_tokens=110,
+ completion_tokens_details=None,
+ prompt_tokens_details=PromptTokensDetailsWrapper(
+ audio_tokens=None,
+ cached_tokens=0,
+ text_tokens=None,
+ image_tokens=None
+ ),
+ cache_creation_input_tokens=0,
+ cache_read_input_tokens=0
+ )
+)
+```
+
+## VertexAI DeepSeek
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/deepseek-ai/{MODEL}` |
+| Vertex Documentation | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) |
+
+#### Usage
+
+**LiteLLM Supports all Vertex AI DeepSeek Models.** Ensure you use the `vertex_ai/deepseek-ai/` prefix for all Vertex AI DeepSeek models.
+
+| Model Name | Usage |
+|------------------|------------------------------|
+| vertex_ai/deepseek-ai/deepseek-r1-0528-maas | `completion('vertex_ai/deepseek-ai/deepseek-r1-0528-maas', messages)` |
+
+
+## VertexAI Meta/Llama API
+
+| Model Name | Function Call |
+|------------------|--------------------------------------|
+| meta/llama-3.2-90b-vision-instruct-maas | `completion('vertex_ai/meta/llama-3.2-90b-vision-instruct-maas', messages)` |
+| meta/llama3-8b-instruct-maas | `completion('vertex_ai/meta/llama3-8b-instruct-maas', messages)` |
+| meta/llama3-70b-instruct-maas | `completion('vertex_ai/meta/llama3-70b-instruct-maas', messages)` |
+| meta/llama3-405b-instruct-maas | `completion('vertex_ai/meta/llama3-405b-instruct-maas', messages)` |
+| meta/llama-4-scout-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas', messages)` |
+| meta/llama-4-scout-17-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-128b-16e-instruct-maas', messages)` |
+| meta/llama-4-maverick-17b-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas',messages)` |
+| meta/llama-4-maverick-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas',messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "meta/llama3-405b-instruct-maas"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: anthropic-llama
+ litellm_params:
+ model: vertex_ai/meta/llama3-405b-instruct-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: anthropic-llama
+ litellm_params:
+ model: vertex_ai/meta/llama3-405b-instruct-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "anthropic-llama", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+## VertexAI Mistral API
+
+[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137)
+
+**LiteLLM Supports all Vertex AI Mistral Models.** Ensure you use the `vertex_ai/mistral-` prefix for all Vertex AI Mistral models.
+
+Overview
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/mistral-{MODEL}` |
+| Vertex Documentation | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
+
+| Model Name | Function Call |
+|------------------|--------------------------------------|
+| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` |
+| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` |
+| mistral-small-2503 | `completion('vertex_ai/mistral-small-2503', messages)` |
+| mistral-large-2411 | `completion('vertex_ai/mistral-large-2411', messages)` |
+| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` |
+| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` |
+| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "mistral-large@2407"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: vertex-mistral
+ litellm_params:
+ model: vertex_ai/mistral-large@2407
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: vertex-mistral
+ litellm_params:
+ model: vertex_ai/mistral-large@2407
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "vertex-mistral", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+
+#### Usage - Codestral FIM
+
+Call Codestral on VertexAI via the OpenAI [`/v1/completion`](https://platform.openai.com/docs/api-reference/completions/create) endpoint for FIM tasks.
+
+Note: You can also call Codestral via `/chat/completion`.
+
+
+
+
+```python
+from litellm import completion
+import os
+
+# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+# OR run `!gcloud auth print-access-token` in your terminal
+
+model = "codestral@2405"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = text_completion(
+ model="vertex_ai/" + model,
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+ prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
+ suffix="return True", # optional
+ temperature=0, # optional
+ top_p=1, # optional
+ max_tokens=10, # optional
+ min_tokens=10, # optional
+ seed=10, # optional
+ stop=["return"], # optional
+)
+
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: vertex-codestral
+ litellm_params:
+ model: vertex_ai/codestral@2405
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: vertex-codestral
+ litellm_params:
+ model: vertex_ai/codestral@2405
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/completions' \
+ -H 'Authorization: Bearer sk-1234' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "vertex-codestral", # 👈 the 'model_name' in config
+ "prompt": "def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
+ "suffix":"return True", # optional
+ "temperature":0, # optional
+ "top_p":1, # optional
+ "max_tokens":10, # optional
+ "min_tokens":10, # optional
+ "seed":10, # optional
+ "stop":["return"], # optional
+ }'
+```
+
+
+
+
+
+## VertexAI AI21 Models
+
+| Model Name | Function Call |
+|------------------|--------------------------------------|
+| jamba-1.5-mini@001 | `completion(model='vertex_ai/jamba-1.5-mini@001', messages)` |
+| jamba-1.5-large@001 | `completion(model='vertex_ai/jamba-1.5-large@001', messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "meta/jamba-1.5-mini@001"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: jamba-1.5-mini
+ litellm_params:
+ model: vertex_ai/jamba-1.5-mini@001
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: jamba-1.5-large
+ litellm_params:
+ model: vertex_ai/jamba-1.5-large@001
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "jamba-1.5-large",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+
+## VertexAI Qwen API
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/qwen/{MODEL}` |
+| Vertex Documentation | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
+
+**LiteLLM Supports all Vertex AI Qwen Models.** Ensure you use the `vertex_ai/qwen/` prefix for all Vertex AI Qwen models.
+
+| Model Name | Usage |
+|------------------|------------------------------|
+| vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | `completion('vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas', messages)` |
+| vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas | `completion('vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas', messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "qwen/qwen3-coder-480b-a35b-instruct-maas"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: vertex-qwen
+ litellm_params:
+ model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+ - model_name: vertex-qwen
+ litellm_params:
+ model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-west-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "vertex-qwen", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+
+## VertexAI GPT-OSS Models
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/openai/{MODEL}` |
+| Vertex Documentation | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
+
+**LiteLLM Supports all Vertex AI GPT-OSS Models.** Ensure you use the `vertex_ai/openai/` prefix for all Vertex AI GPT-OSS models.
+
+| Model Name | Usage |
+|------------------|------------------------------|
+| vertex_ai/openai/gpt-oss-20b-maas | `completion('vertex_ai/openai/gpt-oss-20b-maas', messages)` |
+
+#### Usage
+
+
+
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "openai/gpt-oss-20b-maas"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+ model="vertex_ai/" + model,
+ messages=[{"role": "user", "content": "hi"}],
+ vertex_ai_project=vertex_ai_project,
+ vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: gpt-oss
+ litellm_params:
+ model: vertex_ai/openai/gpt-oss-20b-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-central1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-oss", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+#### Usage - `reasoning_effort`
+
+GPT-OSS models support the `reasoning_effort` parameter for enhanced reasoning capabilities.
+
+
+
+
+```python
+from litellm import completion
+
+response = completion(
+ model="vertex_ai/openai/gpt-oss-20b-maas",
+ messages=[{"role": "user", "content": "Solve this complex problem step by step"}],
+ reasoning_effort="low", # Options: "minimal", "low", "medium", "high"
+ vertex_ai_project="your-vertex-project",
+ vertex_ai_location="us-central1",
+)
+```
+
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+- model_name: gpt-oss
+ litellm_params:
+ model: vertex_ai/openai/gpt-oss-20b-maas
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-central1"
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer " \
+ -d '{
+ "model": "gpt-oss",
+ "messages": [{"role": "user", "content": "Solve this complex problem step by step"}],
+ "reasoning_effort": "low"
+ }'
+```
+
+
+
+
+## Model Garden
+
+:::tip
+
+All OpenAI compatible models from Vertex Model Garden are supported.
+
+:::
+
+#### Using Model Garden
+
+**Almost all Vertex Model Garden models are OpenAI compatible.**
+
+
+
+
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/openai/{MODEL_ID}` |
+| Vertex Documentation | [Model Garden LiteLLM Inference](https://github.com/GoogleCloudPlatform/generative-ai/blob/main/open-models/use-cases/model_garden_litellm_inference.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
+| Supported Operations | `/chat/completions`, `/embeddings` |
+
+
+
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables
+os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = completion(
+ model="vertex_ai/openai/",
+ messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+
+
+
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+ - model_name: llama3-1-8b-instruct
+ litellm_params:
+ model: vertex_ai/openai/5464397967697903616
+ vertex_ai_project: "my-test-project"
+ vertex_ai_location: "us-east-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ }'
+```
+
+
+
+
+
+
+
+
+
+
+
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables
+os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = completion(
+ model="vertex_ai/",
+ messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+
+
+
diff --git a/docs/my-website/docs/providers/vllm.md b/docs/my-website/docs/providers/vllm.md
index 5c8233b0564..1a37f2f10e7 100644
--- a/docs/my-website/docs/providers/vllm.md
+++ b/docs/my-website/docs/providers/vllm.md
@@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM.
| Property | Details |
|-------|-------|
| Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) |
-| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) |
+| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) |
| Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) |
-| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions` |
+| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` |
# Quick Start
@@ -104,6 +104,52 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
+ ## Reasoning Effort
+
+
+
+
+ ```python
+ from litellm import completion
+
+ response = completion(
+ model="hosted_vllm/gpt-oss-120b",
+ messages=[{"role": "user", "content": "whats 2 + 2"}],
+ reasoning_effort="high",
+ api_base="https://hosted-vllm-api.co",
+ )
+ print(response)
+ ```
+
+
+
+ 1. Setup config.yaml
+
+ ```yaml
+ model_list:
+ - model_name: gpt-oss-120b
+ litellm_params:
+ model: hosted_vllm/gpt-oss-120b
+ api_base: https://hosted-vllm-api.co
+ ```
+
+ 2. Start the proxy
+
+ ```bash
+ litellm --config /path/to/config.yaml
+ ```
+
+ 3. Test it!
+
+ ```bash
+ curl http://0.0.0.0:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{"model": "gpt-oss-120b", "messages": [{"role": "user", "content": "whats 2 + 2"}], "reasoning_effort": "high"}'
+ ```
+
+
+
+
## Embeddings
@@ -157,6 +203,110 @@ curl -L -X POST 'http://0.0.0.0:4000/embeddings' \
+## Rerank
+
+
+
+
+```python
+from litellm import rerank
+import os
+
+os.environ["HOSTED_VLLM_API_BASE"] = "http://localhost:8000"
+os.environ["HOSTED_VLLM_API_KEY"] = "" # [optional], if your VLLM server requires an API key
+
+query = "What is the capital of the United States?"
+documents = [
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country.",
+]
+
+response = rerank(
+ model="hosted_vllm/your-rerank-model",
+ query=query,
+ documents=documents,
+ top_n=3,
+)
+print(response)
+```
+
+### Async Usage
+
+```python
+from litellm import arerank
+import os, asyncio
+
+os.environ["HOSTED_VLLM_API_BASE"] = "http://localhost:8000"
+os.environ["HOSTED_VLLM_API_KEY"] = "" # [optional], if your VLLM server requires an API key
+
+async def test_async_rerank():
+ query = "What is the capital of the United States?"
+ documents = [
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country.",
+ ]
+
+ response = await arerank(
+ model="hosted_vllm/your-rerank-model",
+ query=query,
+ documents=documents,
+ top_n=3,
+ )
+ print(response)
+
+asyncio.run(test_async_rerank())
+```
+
+
+
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+ - model_name: my-rerank-model
+ litellm_params:
+ model: hosted_vllm/your-rerank-model # add hosted_vllm/ prefix to route as VLLM provider
+ api_base: http://localhost:8000 # add api base for your VLLM server
+ # api_key: your-api-key # [optional] if your VLLM server requires authentication
+```
+
+2. Start the proxy
+
+```bash
+$ litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/rerank' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "model": "my-rerank-model",
+ "query": "What is the capital of the United States?",
+ "documents": [
+ "Carson City is the capital city of the American state of Nevada.",
+ "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+ "Washington, D.C. is the capital of the United States.",
+ "Capital punishment has existed in the United States since before it was a country."
+ ],
+ "top_n": 3
+}'
+```
+
+[See OpenAI SDK/Langchain/etc. examples](../rerank.md#litellm-proxy-usage)
+
+
+
+
## Send Video URL to VLLM
Example Implementation from VLLM [here](https://github.com/vllm-project/vllm/pull/10020)
diff --git a/docs/my-website/docs/providers/volcano.md b/docs/my-website/docs/providers/volcano.md
index 1742a43d819..efd1e02b60b 100644
--- a/docs/my-website/docs/providers/volcano.md
+++ b/docs/my-website/docs/providers/volcano.md
@@ -3,7 +3,7 @@ https://www.volcengine.com/docs/82379/1263482
:::tip
-**We support ALL Volcengine NIM models, just set `model=volcengine/` as a prefix when sending litellm requests**
+**We support ALL Volcengine models including Chat and Embeddings, just set `model=volcengine/` as a prefix when sending litellm requests**
:::
@@ -11,6 +11,8 @@ https://www.volcengine.com/docs/82379/1263482
```python
# env variable
os.environ['VOLCENGINE_API_KEY']
+# or
+os.environ['ARK_API_KEY']
```
## Sample Usage
@@ -64,9 +66,42 @@ for chunk in response:
print(chunk)
```
+## Sample Usage - Embedding
+```python
+from litellm import embedding
+import os
-## Supported Models - 💥 ALL Volcengine NIM Models Supported!
-We support ALL `volcengine` models, just set `volcengine/` as a prefix when sending completion requests
+os.environ['VOLCENGINE_API_KEY'] = ""
+response = embedding(
+ model="volcengine/doubao-embedding-text-240715",
+ input=["hello world", "good morning"]
+)
+print(response)
+```
+
+### Supported Embedding Models
+- `doubao-embedding-large` (2048 dimensions)
+- `doubao-embedding-large-text-250515` (2048 dimensions)
+- `doubao-embedding-large-text-240915` (4096 dimensions)
+- `doubao-embedding` (2560 dimensions)
+- `doubao-embedding-text-240715` (2560 dimensions)
+
+### Embedding Parameters
+```python
+from litellm import embedding
+
+response = embedding(
+ model="volcengine/doubao-embedding-text-240715",
+ input=["sample text"],
+ encoding_format="float", # optional: "float" (default), "base64"
+ user="user-123", # optional: user identifier for tracking
+)
+```
+
+## Supported Models - 💥 ALL Volcengine Models Supported!
+We support ALL `volcengine` models for both chat completions and embeddings:
+- **Chat Models**: Set `volcengine/` as a prefix when sending completion requests
+- **Embedding Models**: Use the specific model names listed above (e.g., `volcengine/doubao-embedding-text-240715`)
## Sample Usage - LiteLLM Proxy
@@ -74,14 +109,21 @@ We support ALL `volcengine` models, just set `volcengine/` as a
```yaml
model_list:
+ # Chat model
- model_name: volcengine-model
litellm_params:
model: volcengine/
api_key: os.environ/VOLCENGINE_API_KEY
+ # Embedding model
+ - model_name: volcengine-embedding
+ litellm_params:
+ model: volcengine/doubao-embedding-text-240715
+ api_key: os.environ/VOLCENGINE_API_KEY
```
### Send Request
+#### Chat Completion
```shell
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
@@ -95,4 +137,15 @@ curl --location 'http://localhost:4000/chat/completions' \
}
]
}'
+```
+
+#### Embedding
+```shell
+curl --location 'http://localhost:4000/embeddings' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "volcengine-embedding",
+ "input": ["hello world", "good morning"]
+}'
```
\ No newline at end of file
diff --git a/docs/my-website/docs/providers/xinference.md b/docs/my-website/docs/providers/xinference.md
index 3686c02098a..9951a1ee3ab 100644
--- a/docs/my-website/docs/providers/xinference.md
+++ b/docs/my-website/docs/providers/xinference.md
@@ -1,6 +1,17 @@
# Xinference [Xorbits Inference]
https://inference.readthedocs.io/en/latest/index.html
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Xinference is an open-source platform to run inference with any open-source LLMs, image generation models, and more. |
+| Provider Route on LiteLLM | `xinference/` |
+| Link to Provider Doc | [Xinference ↗](https://inference.readthedocs.io/en/latest/index.html) |
+| Supported Operations | [`/embeddings`](#sample-usage---embedding), [`/images/generations`](#image-generation) |
+
+LiteLLM supports Xinference Embedding + Image Generation calls.
+
## API Base, Key
```python
# env variable
@@ -9,7 +20,7 @@ os.environ['XINFERENCE_API_KEY'] = "anything" #[optional] no api key required
```
## Sample Usage - Embedding
-```python
+```python showLineNumbers
from litellm import embedding
import os
@@ -22,7 +33,7 @@ print(response)
```
## Sample Usage `api_base` param
-```python
+```python showLineNumbers
from litellm import embedding
import os
@@ -34,6 +45,94 @@ response = embedding(
print(response)
```
+## Image Generation
+
+### Usage - LiteLLM Python SDK
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+# xinference image generation call
+response = image_generation(
+ model="xinference/stabilityai/stable-diffusion-3.5-large",
+ prompt="A beautiful sunset over a calm ocean",
+ api_base="http://127.0.0.1:9997/v1",
+)
+print(response)
+```
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Setup config.yaml
+
+```yaml showLineNumbers
+model_list:
+ - model_name: xinference-sd
+ litellm_params:
+ model: xinference/stabilityai/stable-diffusion-3.5-large
+ api_base: http://127.0.0.1:9997/v1
+ api_key: anything
+ model_info:
+ mode: image_generation
+
+general_settings:
+ master_key: sk-1234
+```
+
+#### 2. Start the proxy
+
+```bash showLineNumbers
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Test it
+
+```bash showLineNumbers
+curl --location 'http://0.0.0.0:4000/v1/images/generations' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer sk-1234' \
+--data '{
+ "model": "xinference-sd",
+ "prompt": "A beautiful sunset over a calm ocean",
+ "n": 1,
+ "size": "1024x1024",
+ "response_format": "url"
+}'
+```
+
+### Advanced Usage - With Additional Parameters
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['XINFERENCE_API_BASE'] = "http://127.0.0.1:9997/v1"
+
+response = image_generation(
+ model="xinference/stabilityai/stable-diffusion-3.5-large",
+ prompt="A beautiful sunset over a calm ocean",
+ n=1, # number of images
+ size="1024x1024", # image size
+ response_format="b64_json", # return format
+)
+print(response)
+```
+
+### Supported Image Generation Models
+
+Xinference supports various stable diffusion models. Here are some examples:
+
+| Model Name | Function Call |
+|---------------------------------------------------------|----------------------------------------------------------------------------------------------------|
+| stabilityai/stable-diffusion-3.5-large | `image_generation(model="xinference/stabilityai/stable-diffusion-3.5-large", prompt="...")` |
+| stabilityai/stable-diffusion-xl-base-1.0 | `image_generation(model="xinference/stabilityai/stable-diffusion-xl-base-1.0", prompt="...")` |
+| runwayml/stable-diffusion-v1-5 | `image_generation(model="xinference/runwayml/stable-diffusion-v1-5", prompt="...")` |
+
+For a complete list of supported image generation models, see: https://inference.readthedocs.io/en/latest/models/builtin/image/index.html
+
## Supported Models
All models listed here https://inference.readthedocs.io/en/latest/models/builtin/embedding/index.html are supported
diff --git a/docs/my-website/docs/proxy/access_control.md b/docs/my-website/docs/proxy/access_control.md
index 69b8a3ff6de..4ca3eb119d6 100644
--- a/docs/my-website/docs/proxy/access_control.md
+++ b/docs/my-website/docs/proxy/access_control.md
@@ -4,7 +4,7 @@ Role-based access control (RBAC) is based on Organizations, Teams and Internal U
- `Organizations` are the top-level entities that contain Teams.
- `Team` - A Team is a collection of multiple `Internal Users`
-- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM
+- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM. Users can be on multiple teams.
- `Roles` define the permissions of an `Internal User`
- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Keys are tied to a `Internal User` and `Team`
diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md
index a0dde80e9cf..32bf97410cb 100644
--- a/docs/my-website/docs/proxy/admin_ui_sso.md
+++ b/docs/my-website/docs/proxy/admin_ui_sso.md
@@ -4,39 +4,21 @@ import TabItem from '@theme/TabItem';
# ✨ SSO for Admin UI
+:::info
+From v1.76.0, SSO is now Free for up to 5 users.
+:::
+
:::info
✨ SSO is on LiteLLM Enterprise
[Enterprise Pricing](https://www.litellm.ai/#pricing)
-[Get free 7-day trial key](https://www.litellm.ai/#trial)
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
:::
-### SSO for UI
-
-#### Step 1: Set upperbounds for keys
-Control the upperbound that users can use for `max_budget`, `budget_duration` or any `key/generate` param per key.
-
-```yaml
-litellm_settings:
- upperbound_key_generate_params:
- max_budget: 100 # Optional[float], optional): upperbound of $100, for all /key/generate requests
- budget_duration: "10d" # Optional[str], optional): upperbound of 10 days for budget_duration values
- duration: "30d" # Optional[str], optional): upperbound of 30 days for all /key/generate requests
- max_parallel_requests: 1000 # (Optional[int], optional): Max number of requests that can be made in parallel. Defaults to None.
- tpm_limit: 1000 #(Optional[int], optional): Tpm limit. Defaults to None.
- rpm_limit: 1000 #(Optional[int], optional): Rpm limit. Defaults to None.
-
-```
-
-** Expected Behavior **
-
-- Send a `/key/generate` request with `max_budget=200`
-- Key will be created with `max_budget=100` since 100 is the upper bound
-
-#### Step 2: Setup Oauth Client
+### Usage (Google, Microsoft, Okta, etc.)
@@ -50,6 +32,7 @@ GENERIC_AUTHORIZATION_ENDPOINT = "/authorize" # https://dev-2k
GENERIC_TOKEN_ENDPOINT = "/token" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/oauth/token
GENERIC_USERINFO_ENDPOINT = "/userinfo" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/userinfo
GENERIC_CLIENT_STATE = "random-string" # [OPTIONAL] REQUIRED BY OKTA, if not set random state value is generated
+GENERIC_SSO_HEADERS = "Content-Type=application/json, X-Custom-Header=custom-value" # [OPTIONAL] Comma-separated list of additional headers to add to the request - e.g. Content-Type=application/json, etc.
```
You can get your domain specific auth/token/userinfo endpoints at `/.well-known/openid-configuration`
@@ -186,6 +169,10 @@ Set a Proxy Admin when SSO is enabled. Once SSO is enabled, the `user_id` for us
export PROXY_ADMIN_ID="116544810872468347480"
```
+This will update the user role in the `LiteLLM_UserTable` to `proxy_admin`.
+
+If you plan to change this ID, please update the user role via API `/user/update` or UI (Internal Users page).
+
#### Step 3: See all proxy keys
@@ -252,6 +239,13 @@ Example setting a local image (on your container)
```shell
UI_LOGO_PATH="ui_images/logo.jpg"
```
+
+#### Or set your logo directly from Admin UI:
+
+
+
+
+
#### Set Custom Color Theme
- Navigate to [/enterprise/enterprise_ui](https://github.com/BerriAI/litellm/blob/main/enterprise/enterprise_ui/_enterprise_colors.json)
- Inside the `enterprise_ui` directory, rename `_enterprise_colors.json` to `enterprise_colors.json`
@@ -273,3 +267,89 @@ Set your colors to any of the following colors: https://www.tremor.so/docs/layou
```
- Deploy LiteLLM Proxy Server
+## Troubleshooting
+
+### "The 'redirect_uri' parameter must be a Login redirect URI in the client app settings" Error
+
+This error commonly occurs with Okta and other SSO providers when the redirect URI configuration is incorrect.
+
+#### Issue
+```
+Your request resulted in an error. The 'redirect_uri' parameter must be a Login redirect URI in the client app settings
+```
+
+#### Solution
+
+**1. Ensure you have set PROXY_BASE_URL in your .env and it includes protocol**
+
+Make sure your `PROXY_BASE_URL` includes the complete URL with protocol (`http://` or `https://`):
+
+```bash
+# ✅ Correct - includes https://
+PROXY_BASE_URL=https://litellm.platform.com
+
+# ✅ Correct - includes http://
+PROXY_BASE_URL=http://litellm.platform.com
+
+# ❌ Incorrect - missing protocol
+PROXY_BASE_URL=litellm.platform.com
+```
+
+**2. For Okta specifically, ensure GENERIC_CLIENT_STATE is set**
+
+Okta requires the `GENERIC_CLIENT_STATE` parameter:
+
+```bash
+GENERIC_CLIENT_STATE="random-string" # Required for Okta
+```
+
+### Common Configuration Issues
+
+#### Missing Protocol in Base URL
+```bash
+# This will cause redirect_uri errors
+PROXY_BASE_URL=mydomain.com
+
+# Fix: Add the protocol
+PROXY_BASE_URL=https://mydomain.com
+```
+
+### Fallback Login
+
+If you need to access the UI via username/password when SSO is on navigate to `/fallback/login`. This route will allow you to sign in with your username/password credentials.
+
+
+
+
+### Debugging SSO JWT fields
+
+If you need to inspect the JWT fields received from your SSO provider by LiteLLM, follow these instructions. This guide walks you through setting up a debug callback to view the JWT data during the SSO process.
+
+
+
+
+
+1. Add `/sso/debug/callback` as a redirect URL in your SSO provider
+
+ In your SSO provider's settings, add the following URL as a new redirect (callback) URL:
+
+ ```bash showLineNumbers title="Redirect URL"
+ http:///sso/debug/callback
+ ```
+
+
+2. Navigate to the debug login page on your browser
+
+ Navigate to the following URL on your browser:
+
+ ```bash showLineNumbers title="URL to navigate to"
+ https:///sso/debug/login
+ ```
+
+ This will initiate the standard SSO flow. You will be redirected to your SSO provider's login screen, and after successful authentication, you will be redirected back to LiteLLM's debug callback route.
+
+
+3. View the JWT fields
+
+Once redirected, you should see a page called "SSO Debug Information". This page displays the JWT fields received from your SSO provider (as shown in the image above)
+
diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md
index e2f6223c8fb..4cbcd0cffce 100644
--- a/docs/my-website/docs/proxy/alerting.md
+++ b/docs/my-website/docs/proxy/alerting.md
@@ -148,7 +148,7 @@ client = openai.OpenAI(
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
- model="gpt-3.5-turbo",
+ model="gpt-4o",
messages = [],
extra_body={
"metadata": {
diff --git a/docs/my-website/docs/proxy/auto_routing.md b/docs/my-website/docs/proxy/auto_routing.md
new file mode 100644
index 00000000000..7325dc8227e
--- /dev/null
+++ b/docs/my-website/docs/proxy/auto_routing.md
@@ -0,0 +1,221 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Auto Routing
+
+LiteLLM can auto select the best model for a request based on rules you define.
+
+
+
+## LiteLLM Python SDK
+
+Auto routing allows you to define routing rules that automatically select the best model for a request based on the input content. This is useful for directing different types of queries to specialized models.
+
+### Setup
+
+1. **Create a router configuration file** (e.g., `router.json`):
+
+```json
+{
+ "encoder_type": "openai",
+ "encoder_name": "text-embedding-3-large",
+ "routes": [
+ {
+ "name": "litellm-gpt-4.1",
+ "utterances": [
+ "litellm is great"
+ ],
+ "description": "positive affirmation",
+ "function_schemas": null,
+ "llm": null,
+ "score_threshold": 0.5,
+ "metadata": {}
+ },
+ {
+ "name": "litellm-claude-35",
+ "utterances": [
+ "how to code a program in [language]"
+ ],
+ "description": "coding assistant",
+ "function_schemas": null,
+ "llm": null,
+ "score_threshold": 0.5,
+ "metadata": {}
+ }
+ ]
+}
+```
+
+2. **Configure the Router with auto routing models**:
+
+```python
+from litellm import Router
+import os
+
+router = Router(
+ model_list=[
+ # Embedding models for routing
+ {
+ "model_name": "custom-text-embedding-model",
+ "litellm_params": {
+ "model": "text-embedding-3-large",
+ "api_key": os.getenv("OPENAI_API_KEY"),
+ },
+ },
+ # Your target models
+ {
+ "model_name": "litellm-gpt-4.1",
+ "litellm_params": {
+ "model": "gpt-4.1",
+ },
+ "model_info": {"id": "openai-id"},
+ },
+ {
+ "model_name": "litellm-claude-35",
+ "litellm_params": {
+ "model": "claude-3-5-sonnet-latest",
+ },
+ "model_info": {"id": "claude-id"},
+ },
+ # Auto router configuration
+ {
+ "model_name": "auto_router1",
+ "litellm_params": {
+ "model": "auto_router/auto_router_1",
+ "auto_router_config_path": "router.json",
+ "auto_router_default_model": "gpt-4o-mini",
+ "auto_router_embedding_model": "custom-text-embedding-model",
+ },
+ },
+ ],
+)
+```
+
+### Usage
+
+Once configured, use the auto router by calling it with your auto router model name:
+
+```python
+# This request will be routed to gpt-4.1 based on the utterance match
+response = await router.acompletion(
+ model="auto_router1",
+ messages=[{"role": "user", "content": "litellm is great"}],
+)
+
+# This request will be routed to claude-3-5-sonnet-latest for coding queries
+response = await router.acompletion(
+ model="auto_router1",
+ messages=[{"role": "user", "content": "how to code a program in python"}],
+)
+```
+
+### Configuration Parameters
+
+- **auto_router_config_path**: Path to your router.json configuration file
+- **auto_router_default_model**: Fallback model when no route matches
+- **auto_router_embedding_model**: Model used for generating embeddings to match against utterances
+
+### Router Configuration Schema
+
+The `router.json` file supports the following structure:
+
+- **encoder_type**: Type of encoder (e.g., "openai")
+- **encoder_name**: Name of the embedding model
+- **routes**: Array of routing rules with:
+ - **name**: Target model name (must match a model in your model_list)
+ - **utterances**: Example phrases/patterns to match against
+ - **description**: Human-readable description of the route
+ - **score_threshold**: Minimum similarity score to trigger this route (0.0-1.0)
+ - **metadata**: Additional metadata for the route
+
+
+## LiteLLM Proxy Server
+
+### Setup
+
+Navigate to the LiteLLM UI and go to **Models+Endpoints** > **Add Model** > **Auto Router Tab**.
+
+Configure the following required fields:
+
+- **Auto Router Name** - The model name that developers will use when making LLM API requests to LiteLLM
+- **Default Model** - The fallback model used when no route is matched (e.g., if set to "gpt-4o-mini", unmatched requests will be routed to gpt-4o-mini)
+- **Embedding Model** - The model used to generate embeddings for input messages. These embeddings are used to semantically match input against the utterances defined in your routes
+
+#### Route Configuration
+
+
+
+
+
+
+
+Click **Add Route** to create a new routing rule. Each route consists of utterances that are matched against input messages to determine the target model.
+
+Configure each route with:
+
+- **Utterances** - Example phrases that will trigger this route. Use placeholders in brackets for variables:
+
+```json
+"how to code a program in [language]",
+"can you explain this [language] code",
+"can you explain this [language] script",
+"can you convert this [language] code to [target_language]"
+```
+
+- **Description** - A human-readable description of what this route handles
+- **Score Threshold** - The minimum similarity score (0.0-1.0) required to trigger this route
+
+
+### Usage
+
+Once added developers need to select the model=`auto_router1` in the `model` field of the LLM API request.
+
+
+
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="sk-1234", # replace with your LiteLLM API key
+ base_url="http://localhost:4000"
+)
+
+# This request will be auto-routed based on the content
+response = client.chat.completions.create(
+ model="auto_router1",
+ messages=[
+ {
+ "role": "user",
+ "content": "how to code a program in python"
+ }
+ ]
+)
+
+print(response)
+```
+
+
+
+
+```shell
+curl -X POST http://localhost:4000/v1/chat/completions \
+-H "Content-Type: application/json" \
+-H "Authorization: Bearer $LITELLM_API_KEY" \
+-d '{
+ "model": "auto_router1",
+ "messages": [{"role": "user", "content": "how to code a program in python"}]
+}'
+```
+
+
+
+
+
+## How It Works
+
+1. When a request comes in, LiteLLM generates embeddings for the input message
+2. It compares these embeddings against the utterances defined in your routes
+3. If a route's similarity score exceeds the threshold, the request is routed to that model
+4. If no route matches, the request goes to the default model
+
diff --git a/docs/my-website/docs/proxy/billing.md b/docs/my-website/docs/proxy/billing.md
index 902801cd0a2..c1d01467a3c 100644
--- a/docs/my-website/docs/proxy/billing.md
+++ b/docs/my-website/docs/proxy/billing.md
@@ -101,7 +101,7 @@ client = openai.OpenAI(
)
# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+response = client.chat.completions.create(model="gpt-4o", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
@@ -127,7 +127,7 @@ os.environ["OPENAI_API_KEY"] = "sk-tXL0wt5-lOOVK9sfY2UacA" # 👈 Team's Key
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
- model = "gpt-3.5-turbo",
+ model = "gpt-4o",
temperature=0.1,
)
@@ -198,7 +198,7 @@ For:
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
- "model": "gpt-3.5-turbo",
+ "model": "gpt-4o",
"messages": [
{
"role": "user",
@@ -220,7 +220,7 @@ For:
)
# request sent to model set on litellm proxy, `litellm --model`
- response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+ response = client.chat.completions.create(model="gpt-4o", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
@@ -247,7 +247,7 @@ For:
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
- model = "gpt-3.5-turbo",
+ model = "gpt-4o",
temperature=0.1,
extra_body={
"user": "my_customer_id" # 👈 whatever your customer id is
@@ -306,7 +306,7 @@ client = openai.OpenAI(
)
# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+response = client.chat.completions.create(model="gpt-4o", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
diff --git a/docs/my-website/docs/proxy/budget_reset_and_tz.md b/docs/my-website/docs/proxy/budget_reset_and_tz.md
index 541ff6a2f0a..340e33afe18 100644
--- a/docs/my-website/docs/proxy/budget_reset_and_tz.md
+++ b/docs/my-website/docs/proxy/budget_reset_and_tz.md
@@ -29,5 +29,6 @@ Common timezone values:
- `US/Pacific` - Pacific Time
- `Europe/London` - UK Time
- `Asia/Kolkata` - Indian Standard Time (IST)
+- `Asia/Bangkok` - Indochina Time (ICT)
- `Asia/Tokyo` - Japan Standard Time
- `Australia/Sydney` - Australian Eastern Time
diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md
index 84e8c5f8d58..617609cf08a 100644
--- a/docs/my-website/docs/proxy/caching.md
+++ b/docs/my-website/docs/proxy/caching.md
@@ -204,7 +204,71 @@ For quick testing, you can also use REDIS_URL, eg.:
REDIS_URL="rediss://.."
```
-but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between using it vs. redis_host, port, etc.
+but we **don't** recommend using REDIS_URL in prod. We've noticed a performance difference between using it vs. redis_host, port, etc.
+
+#### GCP IAM Authentication
+
+For GCP Memorystore Redis with IAM authentication, install the required dependency:
+
+:::info
+IAM authentication for redis is only supported via GCP and only on Redis Clusters for now.
+:::
+
+```shell
+pip install google-cloud-iam
+```
+
+
+
+
+
+For Redis Cluster with GCP IAM:
+
+```yaml
+litellm_settings:
+ cache: True
+ cache_params:
+ type: redis
+ redis_startup_nodes: [{"host": "10.128.0.2", "port": 6379}, {"host": "10.128.0.2", "port": 11008}]
+ gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
+ ssl: true
+ ssl_cert_reqs: null
+ ssl_check_hostname: false
+```
+
+
+
+
+
+You can configure GCP IAM Redis authentication in your .env:
+
+
+For Redis Cluster:
+
+```env
+REDIS_CLUSTER_NODES='[{"host": "10.128.0.2", "port": 6379}, {"host": "10.128.0.2", "port": 11008}]'
+REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
+REDIS_GCP_SSL_CA_CERTS="./server-ca.pem"
+REDIS_SSL="True"
+REDIS_SSL_CERT_REQS="None"
+REDIS_SSL_CHECK_HOSTNAME="False"
+```
+
+**GCP Authentication Setup**
+
+Make sure your GCP credentials are configured:
+
+```shell
+# Option 1: Service account key file
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
+
+# Option 2: If running on GCP compute instance with service account attached
+# No additional setup needed
+```
+
+
+
+
#### Step 2: Add Redis Credentials to .env
Set either `REDIS_URL` or the `REDIS_HOST` in your os environment, to enable caching.
@@ -895,32 +959,18 @@ curl http://localhost:4000/v1/chat/completions \
+## Redis max_connections
-### Turn on `batch_redis_requests`
-
-**What it does?**
-When a request is made:
-
-- Check if a key starting with `litellm:::` exists in-memory, if no - get the last 100 cached requests for this key and store it
-
-- New requests are stored with this `litellm:..` as the namespace
-
-**Why?**
-Reduce number of redis GET requests. This improved latency by 46% in prod load tests.
-
-**Usage**
+You can set the `max_connections` parameter in your `cache_params` for Redis. This is passed directly to the Redis client and controls the maximum number of simultaneous connections in the pool. If you see errors like `No connection available`, try increasing this value:
```yaml
litellm_settings:
cache: true
cache_params:
type: redis
- ... # remaining redis args (host, port, etc.)
- callbacks: ["batch_redis_requests"] # 👈 KEY CHANGE!
+ max_connections: 100
```
-[**SEE CODE**](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/batch_redis_get.py)
-
## Supported `cache_params` on proxy config.yaml
```yaml
@@ -929,6 +979,7 @@ cache_params:
ttl: Optional[float]
default_in_memory_ttl: Optional[float]
default_in_redis_ttl: Optional[float]
+ max_connections: Optional[Int]
# Type of cache (options: "local", "redis", "s3")
type: s3
@@ -944,6 +995,13 @@ cache_params:
password: secret_password # Redis server password
namespace: Optional[str] = None,
+ # GCP IAM Authentication for Redis
+ gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
+ gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
+ ssl: true # Enable SSL for secure connections
+ ssl_cert_reqs: null # Set to null for self-signed certificates
+ ssl_check_hostname: false # Set to false for self-signed certificates
+
# S3 cache parameters
s3_bucket_name: your_s3_bucket_name # Name of the S3 bucket
diff --git a/docs/my-website/docs/proxy/call_hooks.md b/docs/my-website/docs/proxy/call_hooks.md
index c588ca0d0e6..aef33f8c708 100644
--- a/docs/my-website/docs/proxy/call_hooks.md
+++ b/docs/my-website/docs/proxy/call_hooks.md
@@ -6,6 +6,10 @@ import Image from '@theme/IdealImage';
- Reject data before making llm api calls / before returning the response
- Enforce 'user' param for all openai endpoint calls
+:::tip
+**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`.
+:::
+
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
## Quick Start
@@ -18,7 +22,8 @@ This function is called just before a litellm completion call is made, and allow
from litellm.integrations.custom_logger import CustomLogger
import litellm
from litellm.proxy.proxy_server import UserAPIKeyAuth, DualCache
-from typing import Optional, Literal
+from litellm.types.utils import ModelResponseStream
+from typing import Any, AsyncGenerator, Optional, Literal
# This file includes the custom callbacks for LiteLLM Proxy
# Once defined, these can be passed in proxy_config.yaml
@@ -72,7 +77,7 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
):
pass
- aasync def async_post_call_streaming_iterator_hook(
+ async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
@@ -324,4 +329,4 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"system_fingerprint": null,
"usage": {}
}
-```
\ No newline at end of file
+```
diff --git a/docs/my-website/docs/proxy/cli_sso.md b/docs/my-website/docs/proxy/cli_sso.md
new file mode 100644
index 00000000000..f7669d6a25c
--- /dev/null
+++ b/docs/my-website/docs/proxy/cli_sso.md
@@ -0,0 +1,56 @@
+# CLI Authentication
+
+Use the litellm cli to authenticate to the LiteLLM Gateway. This is great if you're trying to give a large number of developers self-serve access to the LiteLLM Gateway.
+
+
+## Demo
+
+
+
+## Usage
+
+
+1. **Install the CLI**
+
+ If you have [uv](https://github.com/astral-sh/uv) installed, you can try this:
+
+ ```shell
+ uv tool install 'litellm[proxy]'
+ ```
+
+ If that works, you'll see something like this:
+
+ ```shell
+ ...
+ Installed 2 executables: litellm, litellm-proxy
+ ```
+
+ and now you can use the tool by just typing `litellm-proxy` in your terminal:
+
+ ```shell
+ litellm-proxy
+ ```
+
+2. **Set up environment variables**
+
+ ```bash
+ export LITELLM_PROXY_URL=http://localhost:4000
+ ```
+
+ *(Replace with your actual proxy URL)*
+
+3. **Login**
+
+ ```shell
+ litellm-proxy login
+ ```
+
+ This will open a browser window to authenticate. If you have connected LiteLLM Proxy to your SSO provider, you should be able to login with your SSO credentials. Once logged in, you can use the CLI to make requests to the LiteLLM Gateway.
+
+4. **Make a test request to view models**
+
+ ```shell
+ litellm-proxy models list
+ ```
+
+ This will list all the models available to you.
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/clientside_auth.md b/docs/my-website/docs/proxy/clientside_auth.md
index 70424f6d484..c696737adc0 100644
--- a/docs/my-website/docs/proxy/clientside_auth.md
+++ b/docs/my-website/docs/proxy/clientside_auth.md
@@ -1,3 +1,7 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
# Clientside LLM Credentials
diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md
index 03406c9bfa4..c674af1237d 100644
--- a/docs/my-website/docs/proxy/config_settings.md
+++ b/docs/my-website/docs/proxy/config_settings.md
@@ -21,7 +21,7 @@ litellm_settings:
failure_callback: ["sentry"] # list of failure callbacks
callbacks: ["otel"] # list of callbacks - runs on success and failure
service_callbacks: ["datadog", "prometheus"] # logs redis, postgres failures on datadog, prometheus
- turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged.
+ turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data.
redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
@@ -37,7 +37,8 @@ litellm_settings:
content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors
context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors
-
+ # MCP Aliases - Map aliases to MCP server names for easier tool access
+ mcp_aliases: { "github": "github_mcp_server", "zapier": "zapier_mcp_server", "deepwiki": "deepwiki_mcp_server" } # Maps friendly aliases to MCP server names. Only the first alias for each server is used
# Caching settings
cache: true
@@ -49,6 +50,7 @@ litellm_settings:
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
namespace: "litellm.caching.caching" # namespace for redis cache
+ max_connections: 100 # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py.
# Optional - Redis Cluster Settings
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
@@ -57,6 +59,13 @@ litellm_settings:
service_name: "mymaster"
sentinel_nodes: [["localhost", 26379]]
+ # Optional - GCP IAM Authentication for Redis
+ gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
+ gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
+ ssl: true # Enable SSL for secure connections
+ ssl_cert_reqs: null # Set to null for self-signed certificates
+ ssl_check_hostname: false # Set to false for self-signed certificates
+
# Optional - Qdrant Semantic Cache Settings
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
@@ -76,6 +85,7 @@ litellm_settings:
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
+ disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
callback_settings:
@@ -84,6 +94,8 @@ callback_settings:
general_settings:
completion_model: string
+ store_prompts_in_spend_logs: boolean
+ forward_client_headers_to_llm_api: boolean
disable_spend_logs: boolean # turn off writing each transaction to the db
disable_master_key_return: boolean # turn off returning master key on UI (checked on '/user/info' endpoint)
disable_retry_on_max_parallel_request_limit_error: boolean # turn off retries when max parallel request limit is reached
@@ -112,6 +124,35 @@ general_settings:
alerting: ["slack", "email"]
alerting_threshold: 0
use_client_credentials_pass_through_routes: boolean # use client credentials for all pass through routes like "/vertex-ai", /bedrock/. When this is True Virtual Key auth will not be applied on these endpoints
+
+router_settings:
+ routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
+ redis_host: # string
+ redis_password: # string
+ redis_port: # string
+ enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
+ allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
+ cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
+ disable_cooldowns: True # bool - Disable cooldowns for all models
+ enable_tag_filtering: True # bool - Use tag based routing for requests
+ retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
+ "AuthenticationErrorRetries": 3,
+ "TimeoutErrorRetries": 3,
+ "RateLimitErrorRetries": 3,
+ "ContentPolicyViolationErrorRetries": 4,
+ "InternalServerErrorRetries": 4
+ }
+ allowed_fails_policy: {
+ "BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
+ "AuthenticationErrorAllowedFails": 10, # int
+ "TimeoutErrorAllowedFails": 12, # int
+ "RateLimitErrorAllowedFails": 10000, # int
+ "ContentPolicyViolationErrorAllowedFails": 15, # int
+ "InternalServerErrorAllowedFails": 20, # int
+ }
+ content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
+ fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
+
```
### litellm_settings - Reference
@@ -122,10 +163,11 @@ general_settings:
| failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
| callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
| service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) |
-| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) |
+| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) |
| modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
| enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
+| mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) |
| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION |
| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) |
@@ -141,6 +183,8 @@ general_settings:
| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
+| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
+| disable_copilot_system_to_assistant | boolean | If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. Useful for tools (like Claude Code) that send system messages, which Copilot does not support. |
### general_settings - Reference
@@ -180,12 +224,14 @@ general_settings:
| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] |
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
| store_model_in_db | boolean | If true, enables storing model + credential information in the DB. |
+| supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. |
| store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. |
| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
+| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
| allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. |
@@ -211,7 +257,7 @@ general_settings:
| pass_through_endpoints | List[Dict[str, Any]] | Define the pass through endpoints. [Docs](./pass_through) |
| enable_oauth2_proxy_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
| forward_openai_org_id | boolean | If true, forwards the OpenAI Organization ID to the backend LLM call (if it's OpenAI). |
-| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers) to the backend LLM call |
+| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call |
| maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged |
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
### router_settings - Reference
@@ -223,7 +269,7 @@ Most values can also be set via `litellm_settings`. If you see overlapping value
```yaml
router_settings:
- routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
+ routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
redis_host: # string
redis_password: # string
redis_port: # string
@@ -307,6 +353,7 @@ router_settings:
| AGENTOPS_SERVICE_NAME | Service Name for AgentOps logging integration
| AISPEND_ACCOUNT_ID | Account ID for AI Spend
| AISPEND_API_KEY | API Key for AI Spend
+| AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
| ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
| ARIZE_API_KEY | API key for Arize platform integration
| ARIZE_SPACE_KEY | Space key for Arize platform
@@ -318,21 +365,37 @@ router_settings:
| ATHINA_API_KEY | API key for Athina service
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
+| ANTHROPIC_API_KEY | API key for Anthropic service
+| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
+| AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
+| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
| AWS_PROFILE_NAME | AWS CLI profile name to be used
+| AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
| AWS_REGION_NAME | Default AWS region for service interactions
+| AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
| AWS_ROLE_NAME | Role name for AWS IAM usage
+| AWS_S3_BUCKET_NAME | Name of the AWS S3 bucket for file operations
+| AWS_S3_OUTPUT_BUCKET_NAME | Name of the AWS S3 output bucket for batch operations
| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
| AWS_SESSION_NAME | Name for AWS session
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
+| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AUTHORITY_HOST | Azure authority host URL
+| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
| AZURE_CLIENT_ID | Client ID for Azure services
| AZURE_CLIENT_SECRET | Client secret for Azure services
+| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
+| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
+| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
+| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow
| AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token
+| AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY | Cost per GB per day for Azure File Search service
+| AZURE_SCOPE | For EntraID Auth, Scope for Azure services, defaults to "https://cognitiveservices.azure.com/.default"
| AZURE_KEY_VAULT_URI | URI for Azure Key Vault
| AZURE_OPERATION_POLLING_TIMEOUT | Timeout in seconds for Azure operation polling
| AZURE_STORAGE_ACCOUNT_KEY | The Azure Storage Account Key to use for Authentication to Azure Blob Storage logging
@@ -341,18 +404,26 @@ router_settings:
| AZURE_STORAGE_TENANT_ID | The Application Tenant ID to use for Authentication to Azure Blob Storage logging
| AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging
| AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging
+| AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY | Cost per GB per day for Azure Vector Store service
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
| BRAINTRUST_API_KEY | API key for Braintrust integration
+| BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
+| CLOUDZERO_API_KEY | CloudZero API key for authentication
+| CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
+| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
+| CLOUDZERO_MAX_FETCHED_DATA_RECORDS | Maximum number of data records to fetch from CloudZero
+| CLOUDZERO_TIMEZONE | Timezone for date handling (default: UTC)
| CONFIG_FILE_PATH | File path for configuration file
| CONFIDENT_API_KEY | API key for DeepEval integration
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
+| COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
| DATABASE_HOST | Hostname for the database server
| DATABASE_NAME | Name of the database
| DATABASE_PASSWORD | Password for the database user
@@ -379,6 +450,7 @@ router_settings:
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
+| DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%)
@@ -398,15 +470,23 @@ router_settings:
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
+| DEFAULT_NUM_WORKERS_LITELLM_PROXY | Default number of workers for LiteLLM proxy. Default is 4. **We strongly recommend setting NUM Workers to Number of vCPUs available**
| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7
| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03
+| DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET | Default reasoning effort disable thinking budget. Default is 0
| DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET | Default high reasoning effort thinking budget. Default is 4096
| DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET | Default low reasoning effort thinking budget. Default is 1024
| DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET | Default medium reasoning effort thinking budget. Default is 2048
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET | Default minimal reasoning effort thinking budget. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash Lite. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
| DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5
+| DEFAULT_SQS_BATCH_SIZE | Default batch size for SQS logging. Default is 512
+| DEFAULT_SQS_FLUSH_INTERVAL_SECONDS | Default flush interval for SQS logging. Default is 10
| DEFAULT_S3_BATCH_SIZE | Default batch size for S3 logging. Default is 512
| DEFAULT_S3_FLUSH_INTERVAL_SECONDS | Default flush interval for S3 logging. Default is 10
| DEFAULT_SLACK_ALERTING_THRESHOLD | Default threshold for Slack alerting. Default is 300
@@ -415,6 +495,7 @@ router_settings:
| DIRECT_URL | Direct URL for service endpoint
| DISABLE_ADMIN_UI | Toggle to disable the admin UI
| DISABLE_AIOHTTP_TRANSPORT | Flag to disable aiohttp transport. When this is set to True, litellm will use httpx instead of aiohttp. **Default is False**
+| DISABLE_AIOHTTP_TRUST_ENV | Flag to disable aiohttp trust environment. When this is set to True, litellm will not trust the environment for aiohttp eg. `HTTP_PROXY` and `HTTPS_PROXY` environment variables will not be used when this is set to True. **Default is False**
| DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates
| DOCS_DESCRIPTION | Description text for documentation pages
| DOCS_FILTERED | Flag indicating filtered documentation
@@ -422,6 +503,9 @@ router_settings:
| DOCS_URL | The path to the Swagger API documentation. **By default this is "/"**
| EMAIL_LOGO_URL | URL for the logo used in emails
| EMAIL_SUPPORT_CONTACT | Support contact email address
+| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
+| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
+| EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails.
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
@@ -433,6 +517,7 @@ router_settings:
| GALILEO_PASSWORD | Password for Galileo authentication
| GALILEO_PROJECT_ID | Project ID for Galileo usage
| GALILEO_USERNAME | Username for Galileo authentication
+| GOOGLE_SECRET_MANAGER_PROJECT_ID | Project ID for Google Secret Manager
| GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket
| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
@@ -443,6 +528,7 @@ router_settings:
| GENERIC_CLIENT_ID | Client ID for generic OAuth providers
| GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers
| GENERIC_CLIENT_STATE | State parameter for generic client authentication
+| GENERIC_SSO_HEADERS | Comma-separated list of additional headers to add to the request - e.g. Authorization=Bearer ``, Content-Type=application/json, etc.
| GENERIC_INCLUDE_CLIENT_ID | Include client ID in requests for OAuth
| GENERIC_SCOPE | Scope settings for generic OAuth providers
| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
@@ -454,17 +540,24 @@ router_settings:
| GENERIC_USER_PROVIDER_ATTRIBUTE | Attribute specifying the user's provider
| GENERIC_USER_ROLE_ATTRIBUTE | Attribute specifying the user's role
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
+| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
| GALILEO_PROJECT_ID | Project ID for Galileo usage
| GALILEO_USERNAME | Username for Galileo authentication
+| GITHUB_COPILOT_TOKEN_DIR | Directory to store GitHub Copilot token for `github_copilot` llm provider
+| GITHUB_COPILOT_API_KEY_FILE | File to store GitHub Copilot API key for `github_copilot` llm provider
+| GITHUB_COPILOT_ACCESS_TOKEN_FILE | File to store GitHub Copilot access token for `github_copilot` llm provider
| GREENSCALE_API_KEY | API key for Greenscale service
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
| GOOGLE_CLIENT_ID | Client ID for Google OAuth
| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
| GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS
+| GUARDRAILS_AI_API_BASE | Base URL for Guardrails AI API
| HEALTH_CHECK_TIMEOUT_SECONDS | Timeout in seconds for health checks. Default is 60
+| HEROKU_API_BASE | Base URL for Heroku API
+| HEROKU_API_KEY | API key for Heroku services
| HF_API_BASE | Base URL for Hugging Face API
| HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
| HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault)
@@ -491,6 +584,7 @@ router_settings:
| LAGO_API_KEY | API key for accessing Lago services
| LANGFUSE_DEBUG | Toggle debug mode for Langfuse
| LANGFUSE_FLUSH_INTERVAL | Interval for flushing Langfuse logs
+| LANGFUSE_TRACING_ENVIRONMENT | Environment for Langfuse tracing
| LANGFUSE_HOST | Host URL for Langfuse service
| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
| LANGFUSE_RELEASE | Release version of Langfuse integration
@@ -502,10 +596,16 @@ router_settings:
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGTRACE_API_KEY | API key for Langtrace service
+| LASSO_API_BASE | Base URL for Lasso API
+| LASSO_API_KEY | API key for Lasso service
+| LASSO_USER_ID | User ID for Lasso service
+| LASSO_CONVERSATION_ID | Conversation ID for Lasso service
| LENGTH_OF_LITELLM_GENERATED_KEY | Length of keys generated by LiteLLM. Default is 16
+| LEGACY_MULTI_INSTANCE_RATE_LIMITING | Flag to enable legacy multi-instance rate limiting. **Default is False**
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
+| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@@ -515,10 +615,19 @@ router_settings:
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
+| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
+| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
+| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
+| LITELLM_LOGGER_NAME | Name for OTEL logger
+| LITELLM_METER_NAME | Name for OTEL Meter
+| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
+| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
+| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
+| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
@@ -526,6 +635,7 @@ router_settings:
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
+| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100
@@ -542,18 +652,19 @@ router_settings:
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
-| MISTRAL_API_BASE | Base URL for Mistral API
+| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
| MISTRAL_API_KEY | API key for Mistral API
| MICROSOFT_CLIENT_ID | Client ID for Microsoft services
| MICROSOFT_CLIENT_SECRET | Client secret for Microsoft services
| MICROSOFT_TENANT | Tenant ID for Microsoft Azure
| MICROSOFT_SERVICE_PRINCIPAL_ID | Service Principal ID for Microsoft Enterprise Application. (This is an advanced feature if you want litellm to auto-assign members to Litellm Teams based on their Microsoft Entra ID Groups)
-| NO_DOCS | Flag to disable documentation generation
+| NO_DOCS | Flag to disable Swagger UI documentation
+| NO_REDOC | Flag to disable Redoc documentation
| NO_PROXY | List of addresses to bypass proxy
| NON_LLM_CONNECTION_TIMEOUT | Timeout in seconds for non-LLM service connections. Default is 15
| OAUTH_TOKEN_INFO_ENDPOINT | Endpoint for OAuth token info retrieval
| OPENAI_BASE_URL | Base URL for OpenAI API
-| OPENAI_API_BASE | Base URL for OpenAI API
+| OPENAI_API_BASE | Base URL for OpenAI API. Default is https://api.openai.com/
| OPENAI_API_KEY | API key for OpenAI services
| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
| OPENAI_ORGANIZATION | Organization identifier for OpenAI
@@ -569,14 +680,22 @@ router_settings:
| OTEL_EXPORTER | Exporter type for OpenTelemetry
| OTEL_EXPORTER_OTLP_PROTOCOL | Exporter type for OpenTelemetry
| OTEL_HEADERS | Headers for OpenTelemetry requests
+| OTEL_MODEL_ID | Model ID for OpenTelemetry tracing
| OTEL_EXPORTER_OTLP_HEADERS | Headers for OpenTelemetry requests
| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
| PAGERDUTY_API_KEY | API key for PagerDuty Alerting
+| PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service
+| PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service
| PHOENIX_API_KEY | API key for Arize Phoenix
| PHOENIX_COLLECTOR_ENDPOINT | API endpoint for Arize Phoenix
| PHOENIX_COLLECTOR_HTTP_ENDPOINT | API http endpoint for Arize Phoenix
+| PILLAR_API_BASE | Base URL for Pillar API Guardrails
+| PILLAR_API_KEY | API key for Pillar API Guardrails
+| PILLAR_ON_FLAGGED_ACTION | Action to take when content is flagged ('block' or 'monitor')
| POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME`
+| POSTHOG_API_KEY | API key for PostHog analytics integration
+| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -587,10 +706,10 @@ router_settings:
| PROXY_ADMIN_ID | Admin identifier for proxy server
| PROXY_BASE_URL | Base URL for proxy service
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
+| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
| PROXY_LOGOUT_URL | URL for logging out of the proxy service
-| LITELLM_MASTER_KEY | Master key for proxy authentication
| QDRANT_API_BASE | Base URL for Qdrant API
| QDRANT_API_KEY | API key for Qdrant service
| QDRANT_SCALAR_QUANTILE | Scalar quantile for Qdrant operations. Default is 0.99
@@ -601,6 +720,8 @@ router_settings:
| REDIS_PASSWORD | Password for Redis service
| REDIS_PORT | Port number for Redis server
| REDIS_SOCKET_TIMEOUT | Timeout in seconds for Redis socket operations. Default is 0.1
+| REDIS_GCP_SERVICE_ACCOUNT | GCP service account for IAM authentication with Redis. Format: "projects/-/serviceAccounts/name@project.iam.gserviceaccount.com"
+| REDIS_GCP_SSL_CA_CERTS | Path to SSL CA certificate file for secure GCP Memorystore Redis connections
| REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"**
| REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100
| REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64
@@ -608,6 +729,8 @@ router_settings:
| REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
+| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
+| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
| SERVER_ROOT_PATH | Root path for the server application
| SET_VERBOSE | Flag to enable verbose logging
| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000
@@ -621,9 +744,11 @@ router_settings:
| SMTP_TLS | Flag to enable or disable TLS for SMTP connections
| SMTP_USERNAME | Username for SMTP authentication (do not set if SMTP does not require auth)
| SPEND_LOGS_URL | URL for retrieving spend logs
+| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| SSL_CERTIFICATE | Path to the SSL certificate file
| SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1`
| SSL_VERIFY | Flag to enable or disable SSL certificate verification
+| SSL_CERT_FILE | Path to the SSL certificate file for custom CA bundle
| SUPABASE_KEY | API key for Supabase service
| SUPABASE_URL | Base URL for Supabase instance
| STORE_MODEL_IN_DB | If true, enables storing model + credential information in the DB.
@@ -649,4 +774,6 @@ router_settings:
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
| WEBHOOK_URL | URL for receiving webhooks from external services
-| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run
\ No newline at end of file
+| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run |
+| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 |
+| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 |
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md
index db737f75afe..18177b7c4d2 100644
--- a/docs/my-website/docs/proxy/configs.md
+++ b/docs/my-website/docs/proxy/configs.md
@@ -28,22 +28,22 @@ In the config below:
E.g.:
- `model=vllm-models` will route to `openai/facebook/opt-125m`.
-- `model=gpt-3.5-turbo` will load balance between `azure/gpt-turbo-small-eu` and `azure/gpt-turbo-small-ca`
+- `model=gpt-4o` will load balance between `azure/gpt-4o-eu` and `azure/gpt-4o-ca`
```yaml
model_list:
- - model_name: gpt-3.5-turbo ### RECEIVED MODEL NAME ###
+ - model_name: gpt-4o ### RECEIVED MODEL NAME ###
litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
- model: azure/gpt-turbo-small-eu ### MODEL NAME sent to `litellm.completion()` ###
+ model: azure/gpt-4o-eu ### MODEL NAME sent to `litellm.completion()` ###
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
api_key: "os.environ/AZURE_API_KEY_EU" # does os.getenv("AZURE_API_KEY_EU")
rpm: 6 # [OPTIONAL] Rate limit for this deployment: in requests per minute (rpm)
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-instant-v1
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: azure/gpt-turbo-small-ca
+ model: azure/gpt-4o-ca
api_base: https://my-endpoint-canada-berri992.openai.azure.com/
api_key: "os.environ/AZURE_API_KEY_CA"
rpm: 6
@@ -100,9 +100,9 @@ $ litellm --config /path/to/config.yaml --detailed_debug
#### Step 3: Test it
-Sends request to model where `model_name=gpt-3.5-turbo` on config.yaml.
+Sends request to model where `model_name=gpt-4o` on config.yaml.
-If multiple with `model_name=gpt-3.5-turbo` does [Load Balancing](https://docs.litellm.ai/docs/proxy/load_balancing)
+If multiple with `model_name=gpt-4o` does [Load Balancing](https://docs.litellm.ai/docs/proxy/load_balancing)
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
@@ -110,7 +110,7 @@ If multiple with `model_name=gpt-3.5-turbo` does [Load Balancing](https://docs.l
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data ' {
- "model": "gpt-3.5-turbo",
+ "model": "gpt-4o",
"messages": [
{
"role": "user",
@@ -145,9 +145,9 @@ model_list:
api_key: sk-123
api_base: https://openai-gpt-4-test-v-2.openai.azure.com/
temperature: 0.2
- - model_name: openai-gpt-3.5
+ - model_name: openai-gpt-4o
litellm_params:
- model: openai/gpt-3.5-turbo
+ model: openai/gpt-4o
extra_headers: {"AI-Resource Group": "ishaan-resource"}
api_key: sk-123
organization: org-ikDc4ex8NB
@@ -395,9 +395,9 @@ model_list:
model: huggingface/HuggingFaceH4/zephyr-7b-beta
api_base: http://0.0.0.0:8003
rpm: 60000
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: gpt-3.5-turbo
+ model: gpt-4o
api_key:
rpm: 200
- model_name: gpt-3.5-turbo-16k
@@ -409,13 +409,13 @@ model_list:
litellm_settings:
num_retries: 3 # retry call 3 times on each model_name (e.g. zephyr-beta)
request_timeout: 10 # raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
- fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo"]}] # fallback to gpt-3.5-turbo if call fails num_retries
- context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error
+ fallbacks: [{"zephyr-beta": ["gpt-4o"]}] # fallback to gpt-4o if call fails num_retries
+ context_window_fallbacks: [{"zephyr-beta": ["gpt-3.5-turbo-16k"]}, {"gpt-4o": ["gpt-3.5-turbo-16k"]}] # fallback to gpt-3.5-turbo-16k if context window error
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
router_settings: # router_settings are optional
routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
- model_group_alias: {"gpt-4": "gpt-3.5-turbo"} # all requests with `gpt-4` will be routed to models with `gpt-3.5-turbo`
+ model_group_alias: {"gpt-4": "gpt-4o"} # all requests with `gpt-4` will be routed to models with `gpt-4o`
num_retries: 2
timeout: 30 # 30 seconds
redis_host: # set this when using multiple litellm proxy deployments, load balancing state stored in redis
@@ -496,9 +496,9 @@ Supported Environments:
2. For each model set the list of supported environments in `model_info.supported_environments`
```yaml
model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-3.5-turbo-16k
litellm_params:
- model: openai/gpt-3.5-turbo
+ model: openai/gpt-3.5-turbo-16k
api_key: os.environ/OPENAI_API_KEY
model_info:
supported_environments: ["development", "production", "staging"]
@@ -593,15 +593,25 @@ NO_DOCS="True"
in your environment, and restart the proxy.
+### Disable Redoc
+
+To disable the Redoc docs (defaults to `/redoc`), set
+
+```env
+NO_REDOC="True"
+```
+
+in your environment, and restart the proxy.
+
### Use CONFIG_FILE_PATH for proxy (Easier Azure container deployment)
1. Setup config.yaml
```yaml
model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: gpt-3.5-turbo
+ model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
diff --git a/docs/my-website/docs/proxy/control_plane_and_data_plane.md b/docs/my-website/docs/proxy/control_plane_and_data_plane.md
new file mode 100644
index 00000000000..db0b7884c92
--- /dev/null
+++ b/docs/my-website/docs/proxy/control_plane_and_data_plane.md
@@ -0,0 +1,210 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Control Plane for Multi-region Architecture (Enterprise)
+
+Learn how to deploy LiteLLM across multiple regions while maintaining centralized administration and avoiding duplication of management overhead.
+
+:::info
+
+✨ This requires LiteLLM Enterprise features.
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
+
+:::
+
+## Overview
+
+When scaling LiteLLM for production use, you may want to deploy multiple instances across different regions or availability zones while maintaining a single point of administration. This guide covers how to set up a distributed LiteLLM deployment with:
+
+- **Regional Worker Instances**: Handle LLM requests for users in specific regions
+- **Centralized Admin Instance**: Manages configuration, users, keys, and monitoring
+
+## Architecture Pattern: Regional + Admin Instances
+
+### Typical Deployment Scenario
+
+
+
+### Benefits of This Architecture
+
+1. **Reduced Management Overhead**: Only one instance needs admin capabilities
+2. **Regional Performance**: Users get low-latency access from their region
+3. **Centralized Control**: All administration happens from a single interface
+4. **Security**: Limit admin access to designated instances only
+5. **Cost Efficiency**: Avoid duplicating admin infrastructure
+
+## Configuration
+
+### Admin Instance Configuration
+
+The admin instance handles all management operations and provides the UI.
+
+**Environment Variables for Admin Instance:**
+```bash
+# Keep admin capabilities enabled (default behavior)
+# DISABLE_ADMIN_UI=false # Admin UI available
+# DISABLE_ADMIN_ENDPOINTS=false # Management APIs available
+DISABLE_LLM_API_ENDPOINTS=true # LLM APIs disabled
+DATABASE_URL=postgresql://user:pass@global-db:5432/litellm
+LITELLM_MASTER_KEY=your-master-key
+```
+
+### Worker Instance Configuration
+
+Worker instances handle LLM requests but have admin capabilities disabled.
+
+**Environment Variables for Worker Instances:**
+```bash
+# Disable admin capabilities
+DISABLE_ADMIN_UI=true # No admin UI
+DISABLE_ADMIN_ENDPOINTS=true # No management endpoints
+
+DATABASE_URL=postgresql://user:pass@global-db:5432/litellm
+LITELLM_MASTER_KEY=your-master-key
+```
+
+## Environment Variables Reference
+
+### `DISABLE_ADMIN_UI`
+
+Disables the LiteLLM Admin UI interface.
+
+- **Default**: `false`
+- **Worker Instances**: Set to `true`
+- **Admin Instance**: Leave as `false` (or don't set)
+
+```bash
+# Worker instances
+DISABLE_ADMIN_UI=true
+```
+
+**Effect**: When enabled, the web UI at `/ui` becomes unavailable.
+
+### `DISABLE_ADMIN_ENDPOINTS`
+
+:::info
+
+✨ This is an Enterprise feature.
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
+
+:::
+
+Disables all management/admin API endpoints.
+
+- **Default**: `false`
+- **Worker Instances**: Set to `true`
+- **Admin Instance**: Leave as `false` (or don't set)
+
+```bash
+# Worker instances
+DISABLE_ADMIN_ENDPOINTS=true
+```
+
+**Disabled Endpoints Include**:
+- `/key/*` - Key management
+- `/user/*` - User management
+- `/team/*` - Team management
+- `/config/*` - Configuration updates
+- All other administrative endpoints
+
+**Available Endpoints** (when disabled):
+- `/chat/completions` - LLM requests
+- `/v1/*` - OpenAI-compatible APIs
+- `/vertex_ai/*` - Vertex AI pass-through APIs
+- `/bedrock/*` - Bedrock pass-through APIs
+- `/health` - Basic health check
+- `/metrics` - Prometheus metrics
+- All other LLM API endpoints
+
+
+### `DISABLE_LLM_API_ENDPOINTS`
+
+:::info
+
+✨ This is an Enterprise feature.
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
+
+:::
+
+Disables all LLM API endpoints.
+
+- **Default**: `false`
+- **Worker Instances**: Leave as `false` (or don't set)
+- **Admin Instance**: Set to `true`
+
+```bash
+# Admin instance
+DISABLE_LLM_API_ENDPOINTS=true
+```
+
+
+**Disabled Endpoints Include**:
+- `/chat/completions` - LLM requests
+- `/v1/*` - OpenAI-compatible APIs
+- `/vertex_ai/*` - Vertex AI pass-through APIs
+- `/bedrock/*` - Bedrock pass-through APIs
+- All other LLM API endpoints
+
+
+**Available Endpoints** (when disabled):
+- `/key/*` - Key management
+- `/user/*` - User management
+- `/team/*` - Team management
+- `/config/*` - Configuration updates
+- All other administrative endpoints
+
+
+## Usage Patterns
+
+### Client Usage
+
+**For LLM Requests** (use regional endpoints):
+```python
+import openai
+
+# US users
+client_us = openai.OpenAI(
+ base_url="https://us.company.com/v1",
+ api_key="your-litellm-key"
+)
+
+# EU users
+client_eu = openai.OpenAI(
+ base_url="https://eu.company.com/v1",
+ api_key="your-litellm-key"
+)
+
+response = client_us.chat.completions.create(
+ model="gpt-4",
+ messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+**For Administration** (use admin endpoint):
+```python
+import requests
+
+# Create a new API key
+response = requests.post(
+ "https://admin.company.com/key/generate",
+ headers={"Authorization": "Bearer sk-1234"},
+ json={"duration": "30d"}
+)
+```
+
+## Related Documentation
+
+- [Virtual Keys](./virtual_keys.md) - Managing API keys and users
+- [Health Checks](./health.md) - Monitoring instance health
+- [Prometheus Metrics](./logging.md#prometheus-metrics) - Collecting metrics
+- [Production Deployment](./prod.md) - Production best practices
diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md
index 5b17e565a5d..85147e12c66 100644
--- a/docs/my-website/docs/proxy/cost_tracking.md
+++ b/docs/my-website/docs/proxy/cost_tracking.md
@@ -8,18 +8,19 @@ Track spend for keys, users, and teams across 100+ LLMs.
LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
+:::tip Keep Pricing Data Updated
+[Sync model pricing data from GitHub](../sync_models_github.md) to ensure accurate cost tracking.
+:::
+
### How to Track Spend with LiteLLM
**Step 1**
👉 [Setup LiteLLM with a Database](https://docs.litellm.ai/docs/proxy/virtual_keys#setup)
-
**Step2** Send `/chat/completions` request
-
-
```python
@@ -38,7 +39,7 @@ response = client.chat.completions.create(
}
],
user="palantir", # OPTIONAL: pass user to track spend by user
- extra_body={
+ extra_body={
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"] # ENTERPRISE: pass tags to track spend by tags
}
@@ -47,6 +48,7 @@ response = client.chat.completions.create(
print(response)
```
+
@@ -71,6 +73,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}
}'
```
+
@@ -131,7 +134,7 @@ The following spend gets tracked in Table `LiteLLM_SpendLogs`
```json
{
"api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used
- "user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`.
+ "user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`.
"team_id": "e8d1460f-846c-45d7-9b43-55f3cc52ac32", # Team (LiteLLM_TeamTable) that owns `api_key=sk-1234`
"request_tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"],# Tags sent in request
"end_user": "palantir", # Customer - the `user` sent in the request
@@ -152,7 +155,7 @@ Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoin
-### Allowing Non-Proxy Admins to access `/spend` endpoints
+### Allowing Non-Proxy Admins to access `/spend` endpoints
Use this when you want non-proxy admins to access `/spend` endpoints
@@ -162,8 +165,10 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com
:::
-##### Create Key
-Create Key with with `permissions={"get_spend_routes": true}`
+##### Create Key
+
+Create Key with with `permissions={"get_spend_routes": true}`
+
```shell
curl --location 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer sk-1234' \
@@ -176,22 +181,24 @@ curl --location 'http://0.0.0.0:4000/key/generate' \
##### Use generated key on `/spend` endpoints
Access spend Routes with newly generate keys
+
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30' \
-H 'Authorization: Bearer sk-H16BKvrSNConSsBYLGc_7A'
```
-
-
#### Reset Team, API Key Spend - MASTER KEY ONLY
Use `/global/spend/reset` if you want to:
+
- Reset the Spend for all API Keys, Teams. The `spend` for ALL Teams and Keys in `LiteLLM_TeamTable` and `LiteLLM_VerificationToken` will be set to `spend=0`
- LiteLLM will maintain all the logs in `LiteLLMSpendLogs` for Auditing Purposes
-##### Request
+##### Request
+
Only the `LITELLM_MASTER_KEY` you set can access this route
+
```shell
curl -X POST \
'http://localhost:4000/global/spend/reset' \
@@ -205,6 +212,68 @@ curl -X POST \
{"message":"Spend for all API Keys and Teams reset successfully","status":"success"}
```
+## Total spend per user
+
+Assuming you have been issuing keys for end users, and setting their `user_id` on the key, you can check their usage.
+
+```shell title="Total for a user API" showLineNumbers
+curl -L -X GET 'http://localhost:4000/user/info?user_id=jane_smith' \
+-H 'Authorization: Bearer sk-...'
+```
+
+```json title="Total for a user API Response" showLineNumbers
+{
+ "user_id": "jane_smith",
+ "user_info": {
+ "spend": 0.1
+ },
+ "keys": [
+ {
+ "token": "6e952b0efcafbb6350240db25ed534b4ec6011b3e1ba1006eb4f903461fd36f6",
+ "key_name": "sk-...KE_A",
+ "key_alias": "user-01882d6b-e090-776a-a587-21c63e502670-01983ddb-872f-71a3-8b3a-f9452c705483",
+ "soft_budget_cooldown": false,
+ "spend": 0.1,
+ "expires": "2025-07-31T19:14:13.968000+00:00",
+ "models": [],
+ "aliases": {},
+ "config": {},
+ "user_id": "01982d6b-e090-776a-a587-21c63e502660",
+ "team_id": "f2044fde-2293-482f-bf35-a8dab4e85c5f",
+ "permissions": {},
+ "max_parallel_requests": null,
+ "metadata": {},
+ "blocked": null,
+ "tpm_limit": null,
+ "rpm_limit": null,
+ "max_budget": null,
+ "budget_duration": null,
+ "budget_reset_at": null,
+ "allowed_cache_controls": [],
+ "allowed_routes": [],
+ "model_spend": {},
+ "model_max_budget": {},
+ "budget_id": null,
+ "organization_id": null,
+ "object_permission_id": null,
+ "created_at": "2025-07-24T19:14:13.970000Z",
+ "created_by": "582b168f-fc11-4e14-ad6a-cf4bb3656ddc",
+ "updated_at": "2025-07-24T19:14:13.970000Z",
+ "updated_by": "582b168f-fc11-4e14-ad6a-cf4bb3656ddc",
+ "litellm_budget_table": null,
+ "litellm_organization_table": null,
+ "object_permission": null,
+ "team_alias": null
+ }
+ ],
+ "teams": []
+}
+```
+
+**Warning**
+End users can provide the `user` parameter in their request bodies, doing this will increment the cost reported via `/customer/info?end_user_id=self-declared-user`, and not for the user that owns the key as reported by that API. This means users could "avoid" having their spend tracked, through their method.
+This means if you need to track user spend, and are giving end users API keys, you must always set user_id when creating their api keys, and use keys issued for that user every time you're making LLM calls on their behalf in backend services. This will track their spend.
+
## Daily Spend Breakdown API
Retrieve granular daily usage data for a user (by model, provider, and API key) with a single endpoint.
@@ -255,7 +324,198 @@ curl -L -X GET 'http://localhost:4000/user/daily/activity?start_date=2025-03-20&
See our [Swagger API](https://litellm-api.up.railway.app/#/Budget%20%26%20Spend%20Tracking/get_user_daily_activity_user_daily_activity_get) for more details on the `/user/daily/activity` endpoint
-## ✨ (Enterprise) Generate Spend Reports
+## Custom Tags
+
+Requirements:
+
+- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)
+
+**Note:** By default, LiteLLM will track `User-Agent` as a custom tag for cost tracking. This enables viewing usage for tools like Claude Code, Gemini CLI, etc.
+
+
+
+### Client-side spend tag
+
+
+
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {
+ "tags": ["tag1", "tag2", "tag3"]
+ }
+}
+
+'
+```
+
+
+
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/team/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {
+ "tags": ["tag1", "tag2", "tag3"]
+ }
+}
+
+'
+```
+
+
+
+
+Set `extra_body={"metadata": { }}` to `metadata` you want to pass
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+
+response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages = [
+ {
+ "role": "user",
+ "content": "this is a test request, write a short poem"
+ }
+ ],
+ extra_body={
+ "metadata": {
+ "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"] # 👈 Key Change
+ }
+ }
+)
+
+print(response)
+```
+
+
+
+
+
+```js
+const openai = require("openai");
+
+async function runOpenAI() {
+ const client = new openai.OpenAI({
+ apiKey: "sk-1234",
+ baseURL: "http://0.0.0.0:4000",
+ });
+
+ try {
+ const response = await client.chat.completions.create({
+ model: "gpt-3.5-turbo",
+ messages: [
+ {
+ role: "user",
+ content: "this is a test request, write a short poem",
+ },
+ ],
+ metadata: {
+ tags: ["model-anthropic-claude-v2.1", "app-ishaan-prod"], // 👈 Key Change
+ },
+ });
+ console.log(response);
+ } catch (error) {
+ console.log("got this exception from server");
+ console.error(error);
+ }
+}
+
+// Call the asynchronous function
+runOpenAI();
+```
+
+
+
+
+
+Pass `metadata` as part of the request body
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ "metadata": {"tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"]}
+}'
+```
+
+
+
+
+```python
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+ ChatPromptTemplate,
+ HumanMessagePromptTemplate,
+ SystemMessagePromptTemplate,
+)
+from langchain.schema import HumanMessage, SystemMessage
+
+chat = ChatOpenAI(
+ openai_api_base="http://0.0.0.0:4000",
+ model = "gpt-3.5-turbo",
+ temperature=0.1,
+ extra_body={
+ "metadata": {
+ "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"]
+ }
+ }
+)
+
+messages = [
+ SystemMessage(
+ content="You are a helpful assistant that im using to make a test request to."
+ ),
+ HumanMessage(
+ content="test from litellm. tell me why it's amazing in 1 sentence"
+ ),
+]
+response = chat(messages)
+
+print(response)
+```
+
+
+
+
+### Add custom headers to spend tracking
+
+You can add custom headers to the request to track spend and usage.
+
+```yaml
+litellm_settings:
+ extra_spend_tag_headers:
+ - "x-custom-header"
+```
+
+### Disable user-agent tracking
+
+You can disable user-agent tracking by setting `litellm_settings.disable_add_user_agent_to_request_tags` to `true`.
+
+```yaml
+litellm_settings:
+ disable_add_user_agent_to_request_tags: true
+```
+
+## ✨ (Enterprise) Generate Spend Reports
Use this to charge other teams, customers, users
@@ -275,6 +535,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```
#### Example Response
+
@@ -319,7 +580,6 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
]
```
-
@@ -356,6 +616,7 @@ for row in spend_report:
```
Output from script
+
```shell
# Date: 2024-05-11T00:00:00+00:00
# Team: local_test_team
@@ -378,21 +639,19 @@ Output from script
# Metadata: [{'model': 'gpt-3.5-turbo', 'spend': 0.0005715000000000001, 'api_key': 'b94d5e0bc3a71a573917fe1335dc0c14728c7016337451af9714924ff3a729db', 'total_tokens': 423}]
```
-
-
:::info
Customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
-- [LiteLLM API key](virtual_keys.md)
+- [LiteLLM API key](virtual_keys.md)
:::
@@ -400,7 +659,6 @@ Customer [this is `user` passed to `/chat/completions` request](#how-to-track-sp
👉 Key Change: Specify `group_by=customer`
-
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=customer' \
-H 'Authorization: Bearer sk-1234'
@@ -408,7 +666,6 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
#### Example Response
-
```shell
[
{
@@ -449,15 +706,12 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
]
```
-
-
👉 Key Change: Specify `api_key=sk-1234`
-
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&api_key=sk-1234' \
-H 'Authorization: Bearer sk-1234'
@@ -465,7 +719,6 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
#### Example Response
-
```shell
[
{
@@ -501,10 +754,8 @@ Internal User (Key Owner): This is the value of `user_id` passed when calling [`
:::
-
👉 Key Change: Specify `internal_user_id=ishaan`
-
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-12-30&internal_user_id=ishaan' \
-H 'Authorization: Bearer sk-1234'
@@ -512,7 +763,6 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
#### Example Response
-
```shell
[
{
@@ -576,23 +826,340 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
+## 📊 Spend Logs API - Individual Transaction Logs
+
+The `/spend/logs` endpoint now supports a `summarize` parameter to control data format when using date filters.
+
+### Key Parameters
+
+| Parameter | Description |
+| ----------- | -------------------------------------------------------------------------------------------- |
+| `summarize` | **New parameter**: `true` (default) = aggregated data, `false` = individual transaction logs |
+
+### Examples
+
+**Get individual transaction logs:**
+
+```bash
+curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02&summarize=false" \
+-H "Authorization: Bearer sk-1234"
+```
+
+**Get summarized data (default):**
+
+```bash
+curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02" \
+-H "Authorization: Bearer sk-1234"
+```
+
+**Use Cases:**
+
+- `summarize=false`: Analytics dashboards, ETL processes, detailed audit trails
+- `summarize=true`: Daily spending reports, high-level cost tracking (legacy behavior)
## ✨ Custom Spend Log metadata
Log specific key,value pairs as part of the metadata for a spend log
-:::info
+:::info
-Logging specific key,value pairs in spend logs metadata is an enterprise feature. [See here](./enterprise.md#tracking-spend-with-custom-metadata)
+Logging specific key,value pairs in spend logs metadata is an enterprise feature.
:::
+Requirements:
-## ✨ Custom Tags
+- Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)
-:::info
+#### Usage - /chat/completions requests with special spend logs metadata
-Tracking spend with Custom tags is an enterprise feature. [See here](./enterprise.md#tracking-spend-for-custom-tags)
-:::
+
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {
+ "spend_logs_metadata": {
+ "hello": "world"
+ }
+ }
+}
+
+'
+```
+
+
+
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/team/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {
+ "spend_logs_metadata": {
+ "hello": "world"
+ }
+ }
+}
+
+'
+```
+
+
+
+
+
+Set `extra_body={"metadata": { }}` to `metadata` you want to pass
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="anything",
+ base_url="http://0.0.0.0:4000"
+)
+
+# request sent to model set on litellm proxy, `litellm --model`
+response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages = [
+ {
+ "role": "user",
+ "content": "this is a test request, write a short poem"
+ }
+ ],
+ extra_body={
+ "metadata": {
+ "spend_logs_metadata": {
+ "hello": "world"
+ }
+ }
+ }
+)
+
+print(response)
+```
+
+**Using Headers:**
+
+```python
+import openai
+client = openai.OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+# Pass spend logs metadata via headers
+response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages = [
+ {
+ "role": "user",
+ "content": "this is a test request, write a short poem"
+ }
+ ],
+ extra_headers={
+ "x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
+ }
+)
+
+print(response)
+```
+
+
+
+
+
+
+```js
+const openai = require('openai');
+
+async function runOpenAI() {
+ const client = new openai.OpenAI({
+ apiKey: 'sk-1234',
+ baseURL: 'http://0.0.0.0:4000'
+ });
+
+ try {
+ const response = await client.chat.completions.create({
+ model: 'gpt-3.5-turbo',
+ messages: [
+ {
+ role: 'user',
+ content: "this is a test request, write a short poem"
+ },
+ ],
+ metadata: {
+ spend_logs_metadata: { // 👈 Key Change
+ hello: "world"
+ }
+ }
+ });
+ console.log(response);
+ } catch (error) {
+ console.log("got this exception from server");
+ console.error(error);
+ }
+}
+
+// Call the asynchronous function
+runOpenAI();
+```
+
+**Using Headers:**
+
+```js
+const openai = require('openai');
+
+async function runOpenAI() {
+ const client = new openai.OpenAI({
+ apiKey: 'sk-1234',
+ baseURL: 'http://0.0.0.0:4000'
+ });
+
+ try {
+ const response = await client.chat.completions.create({
+ model: 'gpt-3.5-turbo',
+ messages: [
+ {
+ role: 'user',
+ content: "this is a test request, write a short poem"
+ },
+ ]
+ }, {
+ headers: {
+ 'x-litellm-spend-logs-metadata': '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
+ }
+ });
+ console.log(response);
+ } catch (error) {
+ console.log("got this exception from server");
+ console.error(error);
+ }
+}
+
+// Call the asynchronous function
+runOpenAI();
+```
+
+
+
+
+
+Pass `metadata` as part of the request body
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ "metadata": {
+ "spend_logs_metadata": {
+ "hello": "world"
+ }
+ }
+}'
+```
+
+
+
+
+
+Pass `x-litellm-spend-logs-metadata` as a request header with JSON string
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'x-litellm-spend-logs-metadata: {"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' \
+ --data '{
+ "model": "gpt-3.5-turbo",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+```python
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+ ChatPromptTemplate,
+ HumanMessagePromptTemplate,
+ SystemMessagePromptTemplate,
+)
+from langchain.schema import HumanMessage, SystemMessage
+
+chat = ChatOpenAI(
+ openai_api_base="http://0.0.0.0:4000",
+ model = "gpt-3.5-turbo",
+ temperature=0.1,
+ extra_body={
+ "metadata": {
+ "spend_logs_metadata": {
+ "hello": "world"
+ }
+ }
+ }
+)
+
+messages = [
+ SystemMessage(
+ content="You are a helpful assistant that im using to make a test request to."
+ ),
+ HumanMessage(
+ content="test from litellm. tell me why it's amazing in 1 sentence"
+ ),
+]
+response = chat(messages)
+
+print(response)
+```
+
+
+
+
+
+#### Viewing Spend w/ custom metadata
+
+#### `/spend/logs` Request Format
+
+```bash
+curl -X GET "http://0.0.0.0:4000/spend/logs?request_id= UserAPIKeyAuth:
raise Exception
```
+## UserAPIKeyAuth Fields Reference
+
+The `UserAPIKeyAuth` object supports the following fields for comprehensive auth configuration:
+
+### Core Authentication Fields
+```python
+UserAPIKeyAuth(
+ # Basic auth fields
+ api_key: Optional[str] = None, # The API key (will be hashed automatically)
+ token: Optional[str] = None, # Hashed token for internal use
+ key_name: Optional[str] = None, # Human-readable key name
+ key_alias: Optional[str] = None, # Key alias for identification
+
+ # User identification
+ user_id: Optional[str] = None, # Unique user identifier
+ user_email: Optional[str] = None, # User email address
+ user_role: Optional[LitellmUserRoles] = None, # User role (PROXY_ADMIN, INTERNAL_USER, etc.)
+
+ # Team/Organization
+ team_id: Optional[str] = None, # Team identifier
+ team_alias: Optional[str] = None, # Team display name
+ org_id: Optional[str] = None, # Organization identifier
+)
+```
+
+### Budget and Spend Tracking
+```python
+UserAPIKeyAuth(
+ # User budgets
+ max_budget: Optional[float] = None, # Maximum budget for the key
+ spend: float = 0.0, # Current spend amount
+ soft_budget: Optional[float] = None, # Soft budget limit (warnings)
+ model_max_budget: Dict = {}, # Per-model budget limits
+ model_spend: Dict = {}, # Per-model spend tracking
+
+ # Team budgets
+ team_max_budget: Optional[float] = None, # Team's maximum budget
+ team_spend: Optional[float] = None, # Team's current spend
+ team_member_spend: Optional[float] = None, # This user's spend within the team
+
+ # Budget timing
+ budget_duration: Optional[str] = None, # Budget reset period
+ budget_reset_at: Optional[datetime] = None, # When budget resets
+)
+```
+
+### Rate Limiting
+```python
+UserAPIKeyAuth(
+ # User limits
+ tpm_limit: Optional[int] = None, # Tokens per minute limit
+ rpm_limit: Optional[int] = None, # Requests per minute limit
+ user_tpm_limit: Optional[int] = None, # User-specific TPM limit
+ user_rpm_limit: Optional[int] = None, # User-specific RPM limit
+
+ # Team limits
+ team_tpm_limit: Optional[int] = None, # Team TPM limit
+ team_rpm_limit: Optional[int] = None, # Team RPM limit
+ team_member_tpm_limit: Optional[int] = None, # Per-member TPM limit
+ team_member_rpm_limit: Optional[int] = None, # Per-member RPM limit
+
+ # Per-model limits
+ rpm_limit_per_model: Optional[Dict[str, int]] = None, # RPM limits by model
+ tpm_limit_per_model: Optional[Dict[str, int]] = None, # TPM limits by model
+)
+```
+
+### End User Tracking
+```python
+UserAPIKeyAuth(
+ # End user identification and limits
+ end_user_id: Optional[str] = None, # End user identifier
+ end_user_tpm_limit: Optional[int] = None, # End user TPM limit
+ end_user_rpm_limit: Optional[int] = None, # End user RPM limit
+ end_user_max_budget: Optional[float] = None, # End user budget limit
+)
+```
+
+### Model and Route Access
+```python
+UserAPIKeyAuth(
+ # Model access control
+ models: List = [], # Allowed models list
+ team_models: List = [], # Team's allowed models
+ aliases: Dict = {}, # Model aliases
+
+ # Route permissions
+ allowed_routes: Optional[list] = [], # Allowed API routes
+ allowed_cache_controls: Optional[list] = [], # Cache control permissions
+ permissions: Dict = {}, # General permissions
+)
+```
+
+### Advanced Configuration
+```python
+UserAPIKeyAuth(
+ # Request handling
+ max_parallel_requests: Optional[int] = None, # Concurrent request limit
+ allowed_model_region: Optional[AllowedModelRegion] = None, # Geographic restrictions
+
+ # Expiration and status
+ expires: Optional[Union[str, datetime]] = None, # Key expiration
+ blocked: Optional[bool] = None, # Whether key is blocked
+
+ # Metadata and configuration
+ metadata: Dict = {}, # Custom metadata
+ config: Dict = {}, # Configuration settings
+ team_metadata: Optional[Dict] = None, # Team metadata
+
+ # Internal tracking
+ request_route: Optional[str] = None, # Current request route
+ last_refreshed_at: Optional[float] = None, # Cache refresh timestamp
+)
+```
+
+### Complete Example
+
+```python
+from datetime import datetime, timedelta
+from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
+
+async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
+ try:
+ # Example: Comprehensive auth configuration
+ if api_key.startswith("sk-admin-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="admin_user_123",
+ user_email="admin@company.com",
+ user_role=LitellmUserRoles.PROXY_ADMIN,
+ team_id="admin_team",
+ team_alias="Administrative Team",
+ max_budget=1000.0,
+ soft_budget=800.0,
+ tpm_limit=10000,
+ rpm_limit=100,
+ models=["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"],
+ allowed_routes=["/chat/completions", "/embeddings"],
+ expires=datetime.now() + timedelta(days=30),
+ metadata={"department": "engineering", "cost_center": "ai_ops"}
+ )
+ elif api_key.startswith("sk-team-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="team_user_456",
+ user_email="user@company.com",
+ user_role=LitellmUserRoles.INTERNAL_USER,
+ team_id="dev_team",
+ team_alias="Development Team",
+ max_budget=100.0,
+ tpm_limit=1000,
+ rpm_limit=20,
+ models=["gpt-3.5-turbo", "claude-3-haiku"],
+ team_member_tpm_limit=500, # Limit within team
+ end_user_tpm_limit=100, # Per end-user limit
+ metadata={"project": "chatbot_v2"}
+ )
+ else:
+ raise Exception("Invalid API key")
+ except Exception:
+ raise Exception("Authentication failed")
+```
+
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml
@@ -46,3 +209,128 @@ general_settings:
```shell
$ litellm --config /path/to/config.yaml
```
+
+## ✨ Support LiteLLM Virtual Keys + Custom Auth
+
+Supported from v1.72.2+
+
+:::info
+
+✨ Supporting Custom Auth + LiteLLM Virtual Keys is on LiteLLM Enterprise
+
+[Enterprise Pricing](https://www.litellm.ai/#pricing)
+
+[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
+:::
+
+### Usage
+
+1. Setup custom auth file
+
+```python
+"""
+Example custom auth function.
+
+This will allow all keys starting with "my-custom-key" to pass through.
+"""
+from typing import Union
+
+from fastapi import Request
+
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+async def user_api_key_auth(
+ request: Request, api_key: str
+) -> Union[UserAPIKeyAuth, str]:
+ try:
+ if api_key.startswith("my-custom-key"):
+ return "sk-P1zJMdsqCPNN54alZd_ETw"
+ else:
+ raise Exception("Invalid API key")
+ except Exception:
+ raise Exception("Invalid API key")
+
+```
+
+2. Setup config.yaml
+
+Key change set `mode: auto`. This will check both litellm api key auth + custom auth.
+
+```yaml
+model_list:
+ - model_name: "openai-model"
+ litellm_params:
+ model: "gpt-3.5-turbo"
+ api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+ custom_auth: custom_auth_auto.user_api_key_auth
+ custom_auth_settings:
+ mode: "auto" # can be 'on', 'off', 'auto' - 'auto' checks both litellm api key auth + custom auth
+```
+
+Flow:
+1. Checks custom auth first
+2. If custom auth fails, checks litellm api key auth
+3. If both fail, returns 401
+
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-P1zJMdsqCPNN54alZd_ETw' \
+-d '{
+ "model": "openai-model",
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hey! My name is John"
+ }
+ ]
+}'
+```
+
+
+
+
+#### Bubble up custom exceptions
+
+If you want to bubble up custom exceptions, you can do so by raising a `ProxyException`.
+
+```python
+"""
+Example custom auth function.
+
+This will allow all keys starting with "my-custom-key" to pass through.
+"""
+
+from typing import Union
+
+from fastapi import Request
+
+from litellm.proxy._types import UserAPIKeyAuth, ProxyException
+
+
+async def user_api_key_auth(
+ request: Request, api_key: str
+) -> Union[UserAPIKeyAuth, str]:
+ try:
+ if api_key.startswith("my-custom-key"):
+ return "sk-P1zJMdsqCPNN54alZd_ETw"
+ if api_key == "invalid-api-key":
+ # raise a custom exception back to the client
+ raise ProxyException(
+ message="Invalid API key",
+ type="invalid_request_error",
+ param="api_key",
+ code=401,
+ )
+ else:
+ raise Exception("Invalid API key")
+ except Exception:
+ raise Exception("Invalid API key")
+
+```
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md
index e2df7721bfb..fc7312b92ac 100644
--- a/docs/my-website/docs/proxy/custom_pricing.md
+++ b/docs/my-website/docs/proxy/custom_pricing.md
@@ -83,6 +83,24 @@ model_list:
cache_read_input_token_cost: 0.0000006
```
+### Additional Cost Keys
+
+There are other keys you can use to specify costs for different scenarios and modalities:
+
+- `input_cost_per_token_above_200k_tokens` - Cost for input tokens when context exceeds 200k tokens
+- `output_cost_per_token_above_200k_tokens` - Cost for output tokens when context exceeds 200k tokens
+- `cache_creation_input_token_cost_above_200k_tokens` - Cache creation cost for large contexts
+- `cache_read_input_token_cost_above_200k_token` - Cache read cost for large contexts
+- `input_cost_per_image` - Cost per image in multimodal requests
+- `output_cost_per_reasoning_token` - Cost for reasoning tokens (e.g., OpenAI o1 models)
+- `input_cost_per_audio_token` - Cost for audio input tokens
+- `output_cost_per_audio_token` - Cost for audio output tokens
+- `input_cost_per_video_per_second` - Cost per second of video input
+- `input_cost_per_video_per_second_above_128k_tokens` - Video cost for large contexts
+- `input_cost_per_character` - Character-based pricing for some providers
+
+These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
+
## Set 'base_model' for Cost Tracking (e.g. Azure deployments)
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking
diff --git a/docs/my-website/docs/proxy/custom_root_ui.md b/docs/my-website/docs/proxy/custom_root_ui.md
index 82ea7db6b3b..28ef57d81a4 100644
--- a/docs/my-website/docs/proxy/custom_root_ui.md
+++ b/docs/my-website/docs/proxy/custom_root_ui.md
@@ -1,7 +1,20 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
# UI - Custom Root Path
💥 Use this when you want to serve LiteLLM on a custom base url path like `https://localhost:4000/api/v1`
+:::info
+
+Requires v1.72.3 or higher.
+
+:::
+
+Limitations:
+- This does not work in [litellm non-root](./deploy#non-root---without-internet-connection) images, as it requires write access to the UI files.
+
## Usage
### 1. Set `SERVER_ROOT_PATH` in your .env
@@ -20,34 +33,7 @@ litellm proxy --config /path/to/config.yaml
After running the proxy you can access it on `http://0.0.0.0:4000/api/v1/` (since we set `SERVER_ROOT_PATH="/api/v1"`)
-
-### 3. Reserve the `/litellm` path
-
-LiteLLM uses the `/litellm` path to discover the custom root path. So you need to reserve this path in your proxy.
-
-If you are running the UI, it will query the `/litellm/.well-known/litellm-ui-config` endpoint to get the UI configuration.
-
-So you need to reserve the `/litellm` path in your proxy.
-
-You can see the results with:
-
-```bash
-curl http://0.0.0.0:4000/litellm/.well-known/litellm-ui-config
-```
-
-Expected result:
-
-```json
-
-{
- "server_root_path": "/api/v1",
- ...
-}
-
-```
-
-
-### 4. Verify Running on correct path
+### 3. Verify Running on correct path
diff --git a/docs/my-website/docs/proxy/custom_sso.md b/docs/my-website/docs/proxy/custom_sso.md
index a89de0f324f..bbd7f41bee1 100644
--- a/docs/my-website/docs/proxy/custom_sso.md
+++ b/docs/my-website/docs/proxy/custom_sso.md
@@ -1,20 +1,124 @@
-# Event Hook for SSO Login (Custom Handler)
+# ✨ Event Hooks for SSO Login
-Use this if you want to run your own code after a user signs on to the LiteLLM UI using SSO
+:::info
+✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+:::
-## How it works
-- User lands on Admin UI
-- LiteLLM redirects user to your SSO provider
-- Your SSO provider redirects user back to LiteLLM
-- LiteLLM has retrieved user information from your IDP
-- **Your custom SSO handler is called and returns an object of type SSOUserDefinedValues**
+## Overview
+
+LiteLLM provides two different SSO hooks depending on your authentication setup:
+
+| Hook Type | When to Use | What It Does |
+|-----------|-------------|--------------|
+| **Custom UI SSO Sign-in Handler** | You have an OAuth proxy (oauth2-proxy, Gatekeeper, Vouch, etc.) in front of LiteLLM | Parses user info from request headers and signs user into UI |
+| **Custom SSO Handler** | You use direct SSO providers (Google, Microsoft, SAML) and want custom post-auth logic | Runs custom code after standard OAuth flow to set user permissions/teams |
+
+**Quick Decision Guide:**
+- ✅ **Use Custom UI SSO Sign-in Handler** if user authentication happens outside LiteLLM (via headers)
+- ✅ **Use Custom SSO Handler** if you want LiteLLM to handle OAuth flow + run custom logic afterward
+
+---
+
+## Option 1: Custom UI SSO Sign-in Handler
+
+Use this when you have an **OAuth proxy in front of LiteLLM** that has already authenticated the user and passes user information via request headers.
+
+### How it works
+- User lands on Admin UI
+- 👉 **Your custom SSO sign-in handler is called to parse request headers and return user info**
+- LiteLLM has retrieved user information from your custom handler
- User signed in to UI
-## Usage
+### Usage
-#### 1. Create a custom sso handler file.
+#### 1. Create a custom UI SSO handler file
-Make sure the response type follows the `SSOUserDefinedValues` pydantic object. This is used for logging the user into the Admin UI
+This handler parses request headers and returns user information as an OpenID object:
+
+```python
+from fastapi import Request
+from fastapi_sso.sso.base import OpenID
+from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
+
+
+class MyCustomSSOLoginHandler(CustomSSOLoginHandler):
+ """
+ Custom handler for parsing OAuth proxy headers
+
+ Use this when you have an OAuth proxy (like oauth2-proxy, Vouch, etc.)
+ in front of LiteLLM that adds user info to request headers
+ """
+ async def handle_custom_ui_sso_sign_in(
+ self,
+ request: Request,
+ ) -> OpenID:
+ # Parse headers from your OAuth proxy
+ request_headers = dict(request.headers)
+
+ # Extract user info from headers (adjust header names for your proxy)
+ user_id = request_headers.get("x-forwarded-user") or request_headers.get("x-user")
+ user_email = request_headers.get("x-forwarded-email") or request_headers.get("x-email")
+ user_name = request_headers.get("x-forwarded-preferred-username") or request_headers.get("x-preferred-username")
+
+ # Return OpenID object with user information
+ return OpenID(
+ id=user_id or "unknown",
+ email=user_email or "unknown@example.com",
+ first_name=user_name or "Unknown",
+ last_name="User",
+ display_name=user_name or "Unknown User",
+ picture=None,
+ provider="oauth-proxy",
+ )
+
+# Create an instance to be used by LiteLLM
+custom_ui_sso_sign_in_handler = MyCustomSSOLoginHandler()
+```
+
+#### 2. Configure in config.yaml
+
+```yaml
+model_list:
+ - model_name: "openai-model"
+ litellm_params:
+ model: "gpt-3.5-turbo"
+
+general_settings:
+ custom_ui_sso_sign_in_handler: custom_sso_handler.custom_ui_sso_sign_in_handler
+
+litellm_settings:
+ drop_params: True
+ set_verbose: True
+```
+
+#### 3. Start the proxy
+```shell
+$ litellm --config /path/to/config.yaml
+```
+
+#### 4. Navigate to the Admin UI
+
+When a user attempts navigating to the LiteLLM Admin UI, the request will be routed to your custom UI SSO sign-in handler.
+
+---
+
+## Option 2: Custom SSO Handler (Post-Authentication)
+
+Use this if you want to run your own code **after** a user signs on to the LiteLLM UI using standard SSO providers (Google, Microsoft, etc.)
+
+### How it works
+- User lands on Admin UI
+- LiteLLM redirects user to your SSO provider (Google, Microsoft, etc.)
+- Your SSO provider redirects user back to LiteLLM
+- LiteLLM has retrieved user information from your IDP
+- 👉 **Your custom SSO handler is called and returns an object of type SSOUserDefinedValues**
+- User signed in to UI
+
+### Usage
+
+#### 1. Create a custom SSO handler file
+
+Make sure the response type follows the `SSOUserDefinedValues` pydantic object. This is used for logging the user into the Admin UI:
```python
from fastapi import Request
@@ -40,7 +144,7 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
#################################################
- # Run you custom code / logic here
+ # Run your custom code / logic here
# check if user exists in litellm proxy DB
_user_info = await user_info(user_id=userIDPInfo.id)
print("_user_info from litellm DB ", _user_info) # noqa
@@ -58,23 +162,24 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
raise Exception("Failed custom auth")
```
-#### 2. Pass the filepath (relative to the config.yaml)
+#### 2. Configure in config.yaml
-Pass the filepath to the config.yaml
+Pass the filepath to the config.yaml.
e.g. if they're both in the same dir - `./config.yaml` and `./custom_sso.py`, this is what it looks like:
+
```yaml
model_list:
- model_name: "openai-model"
litellm_params:
model: "gpt-3.5-turbo"
+general_settings:
+ custom_sso: custom_sso.custom_sso_handler
+
litellm_settings:
drop_params: True
set_verbose: True
-
-general_settings:
- custom_sso: custom_sso.custom_sso_handler
```
#### 3. Start the proxy
diff --git a/docs/my-website/docs/proxy/customers.md b/docs/my-website/docs/proxy/customers.md
index 2035b24f3a6..ac160d26542 100644
--- a/docs/my-website/docs/proxy/customers.md
+++ b/docs/my-website/docs/proxy/customers.md
@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# 🙋♂️ Customers / End-User Budgets
+# Customers / End-User Budgets
Track spend, set budgets for your customers.
@@ -136,7 +136,7 @@ Create / Update a customer with budget
curl -X POST 'http://0.0.0.0:4000/customer/new'
-H 'Authorization: Bearer sk-1234'
-H 'Content-Type: application/json'
- -D '{
+ -d '{
"user_id" : "my-customer-id",
"max_budget": "0", # 👈 CAN BE FLOAT
}'
diff --git a/docs/my-website/docs/proxy/db_deadlocks.md b/docs/my-website/docs/proxy/db_deadlocks.md
index 0eee928fa64..ef9d31d6232 100644
--- a/docs/my-website/docs/proxy/db_deadlocks.md
+++ b/docs/my-website/docs/proxy/db_deadlocks.md
@@ -84,3 +84,29 @@ LiteLLM emits the following prometheus metrics to monitor the health/status of t
| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory |
| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis |
+
+## Troubleshooting: Redis Connection Errors
+
+You may see errors like:
+
+```
+LiteLLM Redis Caching: async async_increment() - Got exception from REDIS No connection available., Writing value=21
+LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS No connection available., Writing value=None
+```
+
+This means all available Redis connections are in use, and LiteLLM cannot obtain a new connection from the pool. This can happen under high load or with many concurrent proxy requests.
+
+**Solution:**
+
+- Increase the `max_connections` parameter in your Redis config section in `proxy_config.yaml` to allow more simultaneous connections. For example:
+
+```yaml
+litellm_settings:
+ cache: True
+ cache_params:
+ type: redis
+ max_connections: 100 # Increase as needed for your traffic
+```
+
+Adjust this value based on your expected concurrency and Redis server capacity.
+
diff --git a/docs/my-website/docs/proxy/debugging.md b/docs/my-website/docs/proxy/debugging.md
index 5cca6541763..fbcac24a4d6 100644
--- a/docs/my-website/docs/proxy/debugging.md
+++ b/docs/my-website/docs/proxy/debugging.md
@@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs)
**via cli**
-```bash
+```bash showLineNumbers
$ litellm --debug
```
**via env**
-```python
+```python showLineNumbers
os.environ["LITELLM_LOG"] = "INFO"
```
@@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO"
**via cli**
-```bash
+```bash showLineNumbers
$ litellm --detailed_debug
```
**via env**
-```python
+```python showLineNumbers
os.environ["LITELLM_LOG"] = "DEBUG"
```
### Debug Logs
Run the proxy with `--detailed_debug` to view detailed debug logs
-```shell
+```shell showLineNumbers
litellm --config /path/to/config.yaml --detailed_debug
```
When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output
-```shell
+```shell showLineNumbers
POST Request Sent from LiteLLM:
curl -X POST \
https://api.openai.com/v1/chat/completions \
@@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \
-d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}'
```
+## Debug single request
+
+Pass in `litellm_request_debug=True` in the request body
+
+```bash showLineNumbers
+curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+ "model":"fake-openai-endpoint",
+ "messages": [{"role": "user","content": "How many r in the word strawberry?"}],
+ "litellm_request_debug": true
+}'
+```
+
+This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs.
+
+
+```bash showLineNumbers
+INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit)
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 -
+
+POST Request Sent from LiteLLM:
+curl -X POST \
+https://exampleopenaiendpoint-production.up.railway.app/chat/completions \
+-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \
+-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}'
+
+
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE:
+{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}}
+
+
+INFO: 127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK
+
+```
+
+
## JSON LOGS
Set `JSON_LOGS="True"` in your env:
-```bash
+```bash showLineNumbers
export JSON_LOGS="True"
```
**OR**
Set `json_logs: true` in your yaml:
-```yaml
+```yaml showLineNumbers
litellm_settings:
json_logs: true
```
Start proxy
-```bash
+```bash showLineNumbers
$ litellm
```
@@ -80,7 +118,7 @@ The proxy will now all logs in json format.
Turn off fastapi's default 'INFO' logs
1. Turn on 'json logs'
-```yaml
+```yaml showLineNumbers
litellm_settings:
json_logs: true
```
@@ -89,20 +127,20 @@ litellm_settings:
Only get logs if an error occurs.
-```bash
+```bash showLineNumbers
LITELLM_LOG="ERROR"
```
3. Start proxy
-```bash
+```bash showLineNumbers
$ litellm
```
Expected Output:
-```bash
+```bash showLineNumbers
# no info statements
```
@@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the
How to control this?
- Adjust the cooldown time
-```yaml
+```yaml showLineNumbers
router_settings:
cooldown_time: 0 # 👈 KEY CHANGE
```
- Disable Cooldowns [NOT RECOMMENDED]
-```yaml
+```yaml showLineNumbers
router_settings:
disable_cooldowns: True
```
diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md
index 35174114255..d731c0a3c1d 100644
--- a/docs/my-website/docs/proxy/deploy.md
+++ b/docs/my-website/docs/proxy/deploy.md
@@ -12,10 +12,8 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
-git clone https://github.com/BerriAI/litellm
-
-# Go to folder
-cd litellm
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
@@ -29,7 +27,7 @@ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
-docker-compose up
+docker compose up
```
@@ -41,12 +39,12 @@ Example `litellm_config.yaml`
```yaml
model_list:
- - model_name: azure-gpt-3.5
+ - model_name: azure-gpt-4o
litellm_params:
model: azure/
api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE")
api_key: os.environ/AZURE_API_KEY # runs os.getenv("AZURE_API_KEY")
- api_version: "2023-07-01-preview"
+ api_version: "2025-01-01-preview"
```
@@ -59,7 +57,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
- ghcr.io/berriai/litellm:main-latest \
+ ghcr.io/berriai/litellm:main-stable \
--config /app/config.yaml --detailed_debug
```
@@ -67,13 +65,13 @@ Get Latest Image 👉 [here](https://github.com/berriai/litellm/pkgs/container/l
#### Step 3. TEST Request
- Pass `model=azure-gpt-3.5` this was set on step 1
+ Pass `model=azure-gpt-4o` this was set on step 1
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
- "model": "azure-gpt-3.5",
+ "model": "azure-gpt-4o",
"messages": [
{
"role": "user",
@@ -89,12 +87,12 @@ See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli):
Here's how you can run the docker image and pass your config to `litellm`
```shell
-docker run ghcr.io/berriai/litellm:main-latest --config your_config.yaml
+docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
```
Here's how you can run the docker image and start litellm on port 8002 with `num_workers=8`
```shell
-docker run ghcr.io/berriai/litellm:main-latest --port 8002 --num_workers 8
+docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8
```
@@ -102,7 +100,7 @@ docker run ghcr.io/berriai/litellm:main-latest --port 8002 --num_workers 8
```shell
# Use the provided base image
-FROM ghcr.io/berriai/litellm:main-latest
+FROM ghcr.io/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@@ -127,6 +125,8 @@ CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
Follow these instructions to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps.
+**Note:** You'll need to copy the `schema.prisma` file from the [litellm repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) to your build directory alongside the Dockerfile and requirements.txt.
+
Dockerfile
```shell
@@ -149,6 +149,12 @@ COPY requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
+# Copy Prisma schema file
+COPY schema.prisma .
+
+# Generate prisma client
+RUN prisma generate
+
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
@@ -205,9 +211,9 @@ metadata:
data:
config.yaml: |
model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: azure/gpt-turbo-small-ca
+ model: azure/gpt-4o-ca
api_base: https://my-endpoint-canada-berri992.openai.azure.com/
api_key: os.environ/CA_AZURE_OPENAI_API_KEY
---
@@ -236,7 +242,10 @@ spec:
spec:
containers:
- name: litellm
- image: ghcr.io/berriai/litellm:main-latest # it is recommended to fix a version generally
+ image: ghcr.io/berriai/litellm:main-stable # it is recommended to fix a version generally
+ args:
+ - "--config"
+ - "/app/proxy_server_config.yaml"
ports:
- containerPort: 4000
volumeMounts:
@@ -253,7 +262,7 @@ spec:
```
:::info
-To avoid issues with predictability, difficulties in rollback, and inconsistent environments, use versioning or SHA digests (for example, `litellm:main-v1.30.3` or `litellm@sha256:12345abcdef...`) instead of `litellm:main-latest`.
+To avoid issues with predictability, difficulties in rollback, and inconsistent environments, use versioning or SHA digests (for example, `litellm:main-v1.30.3` or `litellm@sha256:12345abcdef...`) instead of `litellm:main-stable`.
:::
@@ -331,7 +340,7 @@ Requirements:
We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
```shell
-docker pull ghcr.io/berriai/litellm-database:main-latest
+docker pull ghcr.io/berriai/litellm-database:main-stable
```
```shell
@@ -342,7 +351,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
- ghcr.io/berriai/litellm-database:main-latest \
+ ghcr.io/berriai/litellm-database:main-stable \
--config /app/config.yaml --detailed_debug
```
@@ -370,7 +379,7 @@ spec:
spec:
containers:
- name: litellm-container
- image: ghcr.io/berriai/litellm:main-latest
+ image: ghcr.io/berriai/litellm:main-stable
imagePullPolicy: Always
env:
- name: AZURE_API_KEY
@@ -386,7 +395,8 @@ spec:
- "/app/proxy_config.yaml" # Update the path to mount the config file
volumeMounts: # Define volume mount for proxy_config.yaml
- name: config-volume
- mountPath: /app
+ mountPath: /app/proxy_config.yaml
+ subPath: config.yaml # Specify the field under data of the ConfigMap litellm-config
readOnly: true
livenessProbe:
httpGet:
@@ -544,15 +554,15 @@ LiteLLM Proxy supports sharing rpm/tpm shared across multiple litellm instances,
```yaml
model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
model: azure/
api_base:
api_key:
rpm: 6 # Rate limit for this deployment: in requests per minute (rpm)
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: azure/gpt-turbo-small-ca
+ model: azure/gpt-4o-ca
api_base: https://my-endpoint-canada-berri992.openai.azure.com/
api_key:
rpm: 6
@@ -565,7 +575,7 @@ router_settings:
Start docker container with config
```shell
-docker run ghcr.io/berriai/litellm:main-latest --config your_config.yaml
+docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
```
### Deploy with Database + Redis
@@ -576,15 +586,15 @@ LiteLLM Proxy supports sharing rpm/tpm shared across multiple litellm instances,
```yaml
model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
model: azure/
api_base:
api_key:
rpm: 6 # Rate limit for this deployment: in requests per minute (rpm)
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
litellm_params:
- model: azure/gpt-turbo-small-ca
+ model: azure/gpt-4o-ca
api_base: https://my-endpoint-canada-berri992.openai.azure.com/
api_key:
rpm: 6
@@ -600,7 +610,7 @@ Start `litellm-database`docker container with config
docker run --name litellm-proxy \
-e DATABASE_URL=postgresql://:@:/ \
-p 4000:4000 \
-ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml
+ghcr.io/berriai/litellm-database:main-stable --config your_config.yaml
```
### (Non Root) - without Internet Connection
@@ -629,7 +639,7 @@ Use this, If you need to set ssl certificates for your on prem litellm proxy
Pass `ssl_keyfile_path` (Path to the SSL keyfile) and `ssl_certfile_path` (Path to the SSL certfile) when starting litellm proxy
```shell
-docker run ghcr.io/berriai/litellm:main-latest \
+docker run ghcr.io/berriai/litellm:main-stable \
--ssl_keyfile_path ssl_test/keyfile.key \
--ssl_certfile_path ssl_test/certfile.crt
```
@@ -644,7 +654,7 @@ Step 1. Build your custom docker image with hypercorn
```shell
# Use the provided base image
-FROM ghcr.io/berriai/litellm:main-latest
+FROM ghcr.io/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@@ -683,7 +693,48 @@ docker run \
--run_hypercorn
```
-### 4. config.yaml file on s3, GCS Bucket Object/url
+### 4. Keepalive Timeout
+
+Defaults to 5 seconds. Between requests, connections must receive new data within this period or be disconnected.
+
+
+Usage Example:
+In this example, we set the keepalive timeout to 75 seconds.
+
+```shell showLineNumbers title="docker run"
+docker run ghcr.io/berriai/litellm:main-stable \
+ --keepalive_timeout 75
+```
+
+Or set via environment variable:
+In this example, we set the keepalive timeout to 75 seconds.
+
+```shell showLineNumbers title="Environment Variable"
+export KEEPALIVE_TIMEOUT=75
+docker run ghcr.io/berriai/litellm:main-stable
+```
+
+
+### Restart Workers After N Requests
+
+Use this to mitigate memory growth by recycling workers after a fixed number of requests. When set, each worker restarts after completing the specified number of requests. Defaults to disabled when unset.
+
+Usage Examples:
+
+```shell showLineNumbers title="docker run (CLI flag)"
+docker run ghcr.io/berriai/litellm:main-stable \
+ --max_requests_before_restart 10000
+```
+
+Or set via environment variable:
+
+```shell showLineNumbers title="Environment Variable"
+export MAX_REQUESTS_BEFORE_RESTART=10000
+docker run ghcr.io/berriai/litellm:main-stable
+```
+
+
+### 5. config.yaml file on s3, GCS Bucket Object/url
Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc)
@@ -708,7 +759,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \
-e LITELLM_CONFIG_BUCKET_TYPE="gcs" \
-p 4000:4000 \
- ghcr.io/berriai/litellm-database:main-latest --detailed_debug
+ ghcr.io/berriai/litellm-database:main-stable --detailed_debug
```
@@ -729,7 +780,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_NAME= \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \
-p 4000:4000 \
- ghcr.io/berriai/litellm-database:main-latest
+ ghcr.io/berriai/litellm-database:main-stable
```
@@ -822,7 +873,7 @@ Run the following command, replacing `` with the value you copied
docker run --name litellm-proxy \
-e DATABASE_URL= \
-p 4000:4000 \
- ghcr.io/berriai/litellm-database:main-latest
+ ghcr.io/berriai/litellm-database:main-stable
```
#### 4. Access the Application:
@@ -849,7 +900,7 @@ https://litellm-7yjrj3ha2q-uc.a.run.app is our example proxy, substitute it with
curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
- "model": "gpt-3.5-turbo",
+ "model": "gpt-4o",
"messages": [{"role": "user", "content": "Say this is a test!"}],
"temperature": 0.7
}'
@@ -901,7 +952,7 @@ services:
context: .
args:
target: runtime
- image: ghcr.io/berriai/litellm:main-latest
+ image: ghcr.io/berriai/litellm:main-stable
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
volumes:
@@ -976,5 +1027,13 @@ User-agent: *
Disallow: /
```
+## Deployment FAQ
+
+**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?**
+
+A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments.
+**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?**
+
+A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md
index 3a48f8a28f4..f3da18065ec 100644
--- a/docs/my-website/docs/proxy/docker_quick_start.md
+++ b/docs/my-website/docs/proxy/docker_quick_start.md
@@ -2,7 +2,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
-# Getting Started - E2E Tutorial
+# E2E Tutorial
End-to-End tutorial for LiteLLM Proxy to:
- Add an Azure OpenAI model
@@ -13,7 +13,7 @@ End-to-End tutorial for LiteLLM Proxy to:
## Pre-Requisites
-- Install LiteLLM Docker Image ** OR ** LiteLLM CLI (pip package)
+- Install LiteLLM Docker Image **OR** LiteLLM CLI (pip package)
@@ -35,6 +35,30 @@ $ pip install 'litellm[proxy]'
+
+
+Use this docker compose to spin up the proxy with a postgres database running locally.
+
+```bash
+# Get the docker compose file
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
+
+# Add the master key - you can change this after setup
+echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
+
+# Add the litellm salt key - you cannot change this after adding a model
+# It is used to encrypt / decrypt your LLM API Key credentials
+# We recommend - https://1password.com/password-generator/
+# password generator to get a random hash for litellm salt key
+echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
+
+source .env
+
+# Start
+docker compose up
+```
+
+
## 1. Add a model
@@ -43,6 +67,8 @@ Control LiteLLM Proxy with a config.yaml file.
Setup your config.yaml with your azure model.
+Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route).
+
```yaml
model_list:
- model_name: gpt-4o
@@ -252,15 +278,15 @@ See All General Settings [here](http://localhost:3000/docs/proxy/configs#all-set
- **Description**:
- Set a `master key`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`).
- **Usage**:
- - ** Set on config.yaml** set your master key under `general_settings:master_key`, example -
+ - **Set on config.yaml** set your master key under `general_settings:master_key`, example -
`master_key: sk-1234`
- - ** Set env variable** set `LITELLM_MASTER_KEY`
+ - **Set env variable** set `LITELLM_MASTER_KEY`
2. **`database_url`** (str)
- **Description**:
- Set a `database_url`, this is the connection to your Postgres DB, which is used by litellm for generating keys, users, teams.
- **Usage**:
- - ** Set on config.yaml** set your master key under `general_settings:database_url`, example -
+ - **Set on config.yaml** set your `database_url` under `general_settings:database_url`, example -
`database_url: "postgresql://..."`
- Set `DATABASE_URL=postgresql://:@:/` in your env
@@ -478,6 +504,7 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
+- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
diff --git a/docs/my-website/docs/proxy/dynamic_logging.md b/docs/my-website/docs/proxy/dynamic_logging.md
new file mode 100644
index 00000000000..3bc9f72b033
--- /dev/null
+++ b/docs/my-website/docs/proxy/dynamic_logging.md
@@ -0,0 +1,214 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+# Dynamic Callback Management
+
+:::info
+
+✨ This is an enterprise feature.
+
+[Get started with LiteLLM Enterprise](https://www.litellm.ai/enterprise)
+
+:::
+
+LiteLLM's dynamic callback management enables teams to control logging behavior on a per-request basis without requiring central infrastructure changes. This is essential for organizations managing large-scale service ecosystems where:
+
+- **Teams manage their own compliance** - Services can handle sensitive data appropriately without central oversight
+- **Decentralized responsibility** - Each team controls their data handling while using shared infrastructure
+
+You can disable callbacks by passing the `x-litellm-disable-callbacks` header with your requests, giving teams granular control over where their data is logged.
+
+## Getting Started: List and Disable Callbacks
+
+Managing callbacks is a two-step process:
+
+1. **First, list your active callbacks** to see what's currently enabled
+2. **Then, disable specific callbacks** as needed for your requests
+
+
+
+## 1. List Active Callbacks
+
+Start by viewing all currently enabled callbacks on your proxy to see what's available to disable.
+
+#### Request
+
+```bash
+curl -X 'GET' \
+ 'http://localhost:4000/callbacks/list' \
+ -H 'accept: application/json' \
+ -H 'x-litellm-api-key: sk-1234'
+```
+
+#### Response
+
+```json
+{
+ "success": [
+ "deployment_callback_on_success",
+ "sync_deployment_callback_on_success"
+ ],
+ "failure": [
+ "async_deployment_callback_on_failure",
+ "deployment_callback_on_failure"
+ ],
+ "success_and_failure": [
+ "langfuse",
+ "datadog"
+ ]
+}
+```
+
+#### Response Fields
+
+The response contains three arrays that categorize your active callbacks:
+- **`success`** - Callbacks that only execute when requests complete successfully. These callbacks receive data from successful LLM responses.
+- **`failure`** - Callbacks that only execute when requests fail or encounter errors. These callbacks receive error information and failed request data.
+- **`success_and_failure`** - Callbacks that execute for both successful and failed requests. These are typically logging/observability tools that need to capture all request data regardless of outcome.
+
+---
+
+## 2. Disable Callbacks
+
+Now that you know which callbacks are active, you can selectively disable them using the `x-litellm-disable-callbacks` header. You can reference any callback name from the list response above.
+
+### Disable a Single Callback
+
+Use the `x-litellm-disable-callbacks` header to disable specific callbacks for individual requests.
+
+
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'x-litellm-disable-callbacks: langfuse' \
+ --data '{
+ "model": "claude-sonnet-4-20250514",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-sonnet-4-20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ extra_headers={
+ "x-litellm-disable-callbacks": "langfuse"
+ }
+)
+
+print(response)
+```
+
+
+
+
+### Disable Multiple Callbacks
+
+You can disable multiple callbacks by providing a comma-separated list in the header. Use any combination of callback names from your `/callbacks/list` response.
+
+
+
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+ --header 'Content-Type: application/json' \
+ --header 'Authorization: Bearer sk-1234' \
+ --header 'x-litellm-disable-callbacks: langfuse,datadog,prometheus' \
+ --data '{
+ "model": "claude-sonnet-4-20250514",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ]
+}'
+```
+
+
+
+
+```python
+import openai
+
+client = openai.OpenAI(
+ api_key="sk-1234",
+ base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+ model="claude-sonnet-4-20250514",
+ messages=[
+ {
+ "role": "user",
+ "content": "what llm are you"
+ }
+ ],
+ extra_headers={
+ "x-litellm-disable-callbacks": "langfuse,datadog,prometheus"
+ }
+)
+
+print(response)
+```
+
+
+
+
+## Header Format and Case Sensitivity
+
+### Expected Header Format
+
+The `x-litellm-disable-callbacks` header accepts callback names in the following formats (use the exact names returned by `/callbacks/list`):
+
+- **Single callback**: `x-litellm-disable-callbacks: langfuse`
+- **Multiple callbacks**: `x-litellm-disable-callbacks: langfuse,datadog,prometheus`
+
+When specifying multiple callbacks, use comma-separated values without spaces around the commas.
+
+### Case Sensitivity
+
+**Callback name checks are case insensitive.** This means all of the following are equivalent:
+
+```bash
+# These are all equivalent
+x-litellm-disable-callbacks: langfuse
+x-litellm-disable-callbacks: LANGFUSE
+x-litellm-disable-callbacks: LangFuse
+x-litellm-disable-callbacks: langFUSE
+```
+
+This applies to both single and multiple callback specifications:
+
+```bash
+# Case insensitive for multiple callbacks
+x-litellm-disable-callbacks: LANGFUSE,datadog,PROMETHEUS
+x-litellm-disable-callbacks: langfuse,DATADOG,prometheus
+```
+
+
diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md
new file mode 100644
index 00000000000..06d49dfaf0f
--- /dev/null
+++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md
@@ -0,0 +1,258 @@
+
+# Dynamic TPM/RPM Allocation
+
+Prevent projects from gobbling too much tpm/rpm.
+
+Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
+
+## Quick Start Usage
+
+1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: my-fake-model
+ litellm_params:
+ model: gpt-3.5-turbo
+ api_key: my-fake-key
+ mock_response: hello-world
+ tpm: 60
+
+litellm_settings:
+ callbacks: ["dynamic_rate_limiter_v3"]
+
+general_settings:
+ master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
+ database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python showLineNumbers title="test.py"
+"""
+- Run 2 concurrent teams calling same model
+- model has 60 TPM
+- Mock response returns 30 total tokens / request
+- Each team will only be able to make 1 request per minute
+"""
+
+import requests
+from openai import OpenAI, RateLimitError
+
+def create_key(api_key: str, base_url: str):
+ response = requests.post(
+ url="{}/key/generate".format(base_url),
+ json={},
+ headers={
+ "Authorization": "Bearer {}".format(api_key)
+ }
+ )
+
+ _response = response.json()
+
+ return _response["key"]
+
+key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+# call proxy with key 1 - works
+openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
+
+response = openai_client_1.chat.completions.with_raw_response.create(
+ model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
+)
+
+print("Headers for call 1 - {}".format(response.headers))
+_response = response.parse()
+print("Total tokens for call - {}".format(_response.usage.total_tokens))
+
+
+# call proxy with key 2 - works
+openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
+
+response = openai_client_2.chat.completions.with_raw_response.create(
+ model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
+)
+
+print("Headers for call 2 - {}".format(response.headers))
+_response = response.parse()
+print("Total tokens for call - {}".format(_response.usage.total_tokens))
+# call proxy with key 2 - fails
+try:
+ openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
+ raise Exception("This should have failed!")
+except RateLimitError as e:
+ print("This was rate limited b/c - {}".format(str(e)))
+
+```
+
+**Expected Response**
+
+```
+This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key= over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
+```
+
+
+## [BETA] Set Priority / Reserve Quota
+
+Reserve TPM/RPM capacity for different environments or use cases. This ensures critical production workloads always have guaranteed capacity, while development or lower-priority tasks use remaining quota.
+
+**Use Cases:**
+- Production vs Development environments
+- Real-time applications vs batch processing
+- Critical services vs experimental features
+
+:::tip
+
+Reserving TPM/RPM on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
+:::
+
+### How Priority Reservation Works
+
+Priority reservation allocates a percentage of your model's total TPM/RPM to specific priority levels. Keys with higher priority get guaranteed access to their reserved quota first.
+
+**Example Scenario:**
+- Model has 10 RPM total capacity
+- Priority reservation: `{"prod": 0.9, "dev": 0.1}`
+- Result: Production keys get 9 RPM guaranteed, Development keys get 1 RPM guaranteed
+
+### Configuration
+
+#### 1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: "gpt-3.5-turbo"
+ api_key: os.environ/OPENAI_API_KEY
+ rpm: 10 # Total model capacity
+
+litellm_settings:
+ callbacks: ["dynamic_rate_limiter_v3"]
+ priority_reservation:
+ "prod": 0.9 # 90% reserved for production (9 RPM)
+ "dev": 0.1 # 10% reserved for development (1 RPM)
+ priority_reservation_settings:
+ default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata
+ saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit
+
+general_settings:
+ master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
+ database_url: postgres://.. # OR set `DATABASE_URL=".."` in your.env
+```
+
+**Configuration Details:**
+
+`priority_reservation`: Dict[str, float]
+- **Key (str)**: Priority level name (can be any string like "prod", "dev", "critical", etc.)
+- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
+- **Note**: Values should sum to 1.0 or less
+
+`priority_reservation_settings`: Object (Optional)
+- **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5)
+- **saturation_threshold (float)**: Saturation level (0.0 to 1.0) at which strict priority enforcement begins for a model. Saturation is calculated as `max(current_rpm/max_rpm, current_tpm/max_tpm)`. Below this threshold, generous mode allows priority borrowing from unused capacity. Above this threshold, strict mode enforces normalized priority limits.
+ - Example: When model usage is low, keys can use more than their allocated share. When model usage is high, keys are strictly limited to their allocated share.
+
+**Start Proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+#### 2. Create Keys with Priority Levels
+
+**Production Key:**
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {"priority": "prod"}
+}'
+```
+
+**Development Key:**
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+ "metadata": {"priority": "dev"}
+}'
+```
+
+**Key Without Priority (uses default_priority weight):**
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{}'
+```
+
+**Expected Response for both:**
+```json
+{
+ "key": "sk-...",
+ "metadata": {"priority": "prod"}, // or "dev"
+ ...
+}
+```
+
+#### 3. Test Priority Allocation
+
+**Test Production Key (should get 9 RPM):**
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+ -H 'Content-Type: application/json' \
+ -H 'Authorization: Bearer sk-prod-key' \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "Hello from prod"}]
+ }'
+```
+
+**Test Development Key (should get 1 RPM):**
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+ -H 'Content-Type: application/json' \
+ -H 'Authorization: Bearer sk-dev-key' \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "Hello from dev"}]
+ }'
+```
+
+### Expected Behavior
+
+With the configuration above:
+
+1. **Production keys** can make up to 9 requests per minute (90% of 10 RPM)
+2. **Development keys** can make up to 1 request per minute (10% of 10 RPM)
+3. **Keys without explicit priority** get the default_priority weight (0 = 0%), which allocates 0 requests per minute (0% of 10 RPM)
+4. Named priorities in `priority_reservation` and keys with `default_priority` operate independently
+
+**Rate Limit Error Example:**
+```json
+{
+ "error": {
+ "message": "Key=sk-dev-... over available RPM=0. Model RPM=10, Reserved RPM for priority 'dev'=1, Active keys=1",
+ "type": "rate_limit_exceeded",
+ "code": 429
+ }
+}
+```
+
+### Demo Video
+
+This video walks through setting up dynamic rate limiting with priority reservation and locust tests to validate the behavior.
+
+
+
diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md
index 4eb35367dbe..9cd027da7f6 100644
--- a/docs/my-website/docs/proxy/email.md
+++ b/docs/my-website/docs/proxy/email.md
@@ -124,9 +124,7 @@ On the Create Key Modal, Select Advanced Settings > Set Send Email to True.
/>
-
-
-## Customizing Email Branding
+## Email Customization
:::info
@@ -134,13 +132,96 @@ Customizing Email Branding is an Enterprise Feature [Get in touch with us for a
:::
-LiteLLM allows you to customize the:
-- Logo on the Email
-- Email support contact
+LiteLLM allows you to customize various aspects of your email notifications. Below is a complete reference of all customizable fields:
-Set the following in your env to customize your emails
+| Field | Environment Variable | Type | Default Value | Example | Description |
+|-------|-------------------|------|---------------|---------|-------------|
+| Logo URL | `EMAIL_LOGO_URL` | string | LiteLLM logo | `"https://your-company.com/logo.png"` | Public URL to your company logo |
+| Support Contact | `EMAIL_SUPPORT_CONTACT` | string | support@berri.ai | `"support@your-company.com"` | Email address for user support |
+| Email Signature | `EMAIL_SIGNATURE` | string (HTML) | Standard LiteLLM footer | `"
"` | HTML-formatted footer for all emails |
+| Invitation Subject | `EMAIL_SUBJECT_INVITATION` | string | "LiteLLM: New User Invitation" | `"Welcome to Your Company!"` | Subject line for invitation emails |
+| Key Creation Subject | `EMAIL_SUBJECT_KEY_CREATED` | string | "LiteLLM: API Key Created" | `"Your New API Key is Ready"` | Subject line for key creation emails |
-```shell
-EMAIL_LOGO_URL="https://litellm-listing.s3.amazonaws.com/litellm_logo.png" # public url to your logo
-EMAIL_SUPPORT_CONTACT="support@berri.ai" # Your company support email
+
+## HTML Support in Email Signature
+
+The `EMAIL_SIGNATURE` field supports HTML formatting for rich, branded email footers. Here's an example of what you can include:
+
+```html
+
" # Custom HTML footer/signature
+
+# Email Subject Lines
+EMAIL_SUBJECT_INVITATION="Welcome to Your Company!" # Subject for invitation emails
+EMAIL_SUBJECT_KEY_CREATED="Your API Key is Ready" # Subject for key creation emails
+```
+
+## HTML Support in Email Signature
+
+The `EMAIL_SIGNATURE` environment variable supports HTML formatting, allowing you to create rich, branded email footers. You can include:
+
+- Text formatting (bold, italic, etc.)
+- Line breaks using ` `
+- Links using ``
+- Paragraphs using `
+
LiteLLM manages:
@@ -34,7 +37,7 @@ LiteLLM manages:
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)