scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge with main

Browse source
This commit is contained in:
yuneng-jiang 2025-11-20 20:24:03 -08:00
commit 31535ff4b6
639 changed files with 40789 additions and 11086 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

509
.circleci/config.yml
View file

@ -24,6 +24,39 @@ commands:
cd enterprise
python -m pip install -e .
cd ..
setup_litellm_test_deps:
steps:
- checkout
- setup_google_dns
- restore_cache:
keys:
- v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
- v2-litellm-deps-
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ~/.cache/pip
key: v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
jobs:
# Add Windows testing job
@ -668,13 +701,16 @@ jobs:
paths:
- litellm_security_tests_coverage.xml
- litellm_security_tests_coverage
litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
# Split proxy unit tests into 3 jobs for faster execution and better debugging
# test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker
litellm_proxy_unit_testing_key_generation:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- checkout
- setup_google_dns
@ -699,6 +735,114 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "pytest-timeout==2.2.0"
pip install "pytest-forked==1.6.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-genai==1.22.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "apscheduler==3.10.4"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
pip install "pydantic==2.10.2"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "pytest-postgresql==7.0.1"
pip install "fakeredis==2.28.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ./venv
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
set +e
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
set -e
- run:
name: Run key generation tests (no parallel execution to avoid event loop issues)
command: |
pwd
ls
# Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker
python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml
mv .coverage litellm_proxy_unit_tests_key_generation_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_unit_tests_key_generation_coverage.xml
- litellm_proxy_unit_tests_key_generation_coverage
litellm_proxy_unit_testing_part1:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- checkout
- setup_google_dns
- run:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
- run:
name: Install PostgreSQL
command: |
sudo apt-get update
sudo apt-get install -y postgresql-14 postgresql-contrib-14
- restore_cache:
keys:
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "pytest-timeout==2.2.0"
pip install "pytest-forked==1.6.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
@ -752,28 +896,132 @@ jobs:
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
set -e
# Run pytest and generate JUnit XML report
- run:
name: Run tests
name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job)
command: |
pwd
ls
python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
# Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues)
python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_unit_tests_coverage.xml
mv .coverage litellm_proxy_unit_tests_coverage
# Store test results
mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml
mv .coverage litellm_proxy_unit_tests_part1_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_unit_tests_coverage.xml
- litellm_proxy_unit_tests_coverage
- litellm_proxy_unit_tests_part1_coverage.xml
- litellm_proxy_unit_tests_part1_coverage
litellm_proxy_unit_testing_part2:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- checkout
- setup_google_dns
- run:
name: Show git commit hash
command: |
echo "Git commit hash: $CIRCLE_SHA1"
- run:
name: Install PostgreSQL
command: |
sudo apt-get update
sudo apt-get install -y postgresql-14 postgresql-contrib-14
- restore_cache:
keys:
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "pytest-timeout==2.2.0"
pip install "pytest-forked==1.6.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-genai==1.22.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "apscheduler==3.10.4"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
pip install "pydantic==2.10.2"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "pytest-postgresql==7.0.1"
pip install "fakeredis==2.28.1"
pip install "pytest-xdist==3.6.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ./venv
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Run prisma ./docker/entrypoint.sh
command: |
set +e
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
set -e
- run:
name: Run proxy unit tests (part 2 - remaining tests)
command: |
pwd
ls
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml
mv .coverage litellm_proxy_unit_tests_part2_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_proxy_unit_tests_part2_coverage.xml
- litellm_proxy_unit_tests_part2_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- image: cimg/python:3.13.1
@ -1128,59 +1376,88 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_tests:
# Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
litellm_mapped_tests_proxy:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- checkout
- setup_google_dns
- setup_litellm_test_deps
- run:
name: Install Dependencies
name: Run proxy tests
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
# Run pytest and generate JUnit XML report
- run:
name: Run litellm tests
command: |
pwd
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mapped_tests_coverage.xml
mv .coverage litellm_mapped_tests_coverage
# Store test results
mv coverage.xml litellm_proxy_tests_coverage.xml
mv .coverage litellm_proxy_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mapped_tests_coverage.xml
- litellm_mapped_tests_coverage
- litellm_proxy_tests_coverage.xml
- litellm_proxy_tests_coverage
litellm_mapped_tests_llms:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run LLM provider tests
command: |
python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_llms_tests_coverage.xml
mv .coverage litellm_llms_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_llms_tests_coverage.xml
- litellm_llms_tests_coverage
litellm_mapped_tests_core:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: xlarge
steps:
- setup_litellm_test_deps
- run:
name: Run core tests
command: |
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_core_tests_coverage.xml
mv .coverage litellm_core_tests_coverage
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_core_tests_coverage.xml
- litellm_core_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
@ -1447,7 +1724,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1914,14 +2191,14 @@ jobs:
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
name: Install Python 3.10
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -2695,19 +2972,22 @@ jobs:
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
name: Install Python 3.10
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
@ -2736,6 +3016,8 @@ jobs:
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
pip install "fastuuid==0.13.5"
pip install -r requirements.txt
- run:
name: Install dockerize
command: |
@ -2848,6 +3130,9 @@ jobs:
- run:
name: Run tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pwd
ls
python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
@ -2878,7 +3163,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3054,7 +3339,7 @@ jobs:
python -m build
twine upload --verbose dist/*
e2e_ui_testing:
ui_build:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
@ -3081,6 +3366,48 @@ jobs:
# Now source the build script
source ./build_ui.sh
- persist_to_workspace:
root: .
paths:
- litellm/proxy/_experimental/out
ui_unit_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Run UI unit tests (Vitest)
command: |
# Use Node 20 (several deps require >=20)
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
--coverage.reporter=lcov \
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Upgrade Docker to v24.x (API 1.44+)
command: |
@ -3126,24 +3453,6 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Run UI unit tests (Vitest)
command: |
# Use Node 20 (several deps require >=20)
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
--coverage.reporter=lcov \
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
- run:
name: Build Docker image
@ -3300,7 +3609,19 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_proxy_unit_testing:
- litellm_proxy_unit_testing_key_generation:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_proxy_unit_testing_part1:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_proxy_unit_testing_part2:
filters:
branches:
only:
@ -3336,6 +3657,20 @@ workflows:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_unit_tests:
requires:
- ui_build
filters:
branches:
only:
- main
- /litellm_.*/
- auth_ui_unit_tests:
filters:
branches:
@ -3343,6 +3678,8 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
requires:
- ui_build
filters:
branches:
only:
@ -3444,7 +3781,19 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_tests:
- litellm_mapped_tests_proxy:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_llms:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests_core:
filters:
branches:
only:
@ -3495,7 +3844,9 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
@ -3506,7 +3857,9 @@ workflows:
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- litellm_proxy_unit_testing
- litellm_proxy_unit_testing_key_generation
- litellm_proxy_unit_testing_part1
- litellm_proxy_unit_testing_part2
- litellm_security_tests
- langfuse_logging_unit_tests
- local_testing
@ -3560,7 +3913,9 @@ workflows:
- llm_responses_api_testing
- ocr_testing
- search_testing
- litellm_mapped_tests
- litellm_mapped_tests_proxy
- litellm_mapped_tests_llms
- litellm_mapped_tests_core
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
@ -3576,7 +3931,9 @@ workflows:
- auth_ui_unit_tests
- db_migration_disable_update_check
- e2e_ui_testing
- litellm_proxy_unit_testing
- litellm_proxy_unit_testing_key_generation
- litellm_proxy_unit_testing_part1
- litellm_proxy_unit_testing_part2
- litellm_security_tests
- installing_litellm_on_python
- installing_litellm_on_python_3_13

4
AGENTS.md
View file

@ -94,6 +94,10 @@ LiteLLM supports MCP for agent workflows:
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
## RUNNING SCRIPTS
Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
## TESTING CONSIDERATIONS
1. **Provider Tests**: Test against real provider APIs when possible

3
CLAUDE.md
View file

@ -25,6 +25,9 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `poetry run python script.py` - Run Python scripts (use for non-test files)
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:

6
Makefile
View file

@ -34,13 +34,13 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
pip install openai==1.99.5
pip install openai==2.8.0
poetry install --with dev
pip install openai==1.99.5
pip install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
pip install openai==1.99.5
pip install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"

261
VERTEX_ENV_SETUP.md
View file

@ -1,261 +0,0 @@
# Vertex AI Environment Variables Setup Guide
## Overview
LiteLLM can load Vertex AI credentials from environment variables instead of storing them in config files. This is more secure and easier to manage for local development.
## Environment Variables
LiteLLM looks for these environment variables (in order of precedence):
### 1. **DEFAULT_VERTEXAI_PROJECT** (Required)
Your GCP project ID that has Vertex AI enabled.
```bash
export DEFAULT_VERTEXAI_PROJECT="my-gcp-project-id"
```
### 2. **DEFAULT_VERTEXAI_LOCATION** (Required)
The region/location for Vertex AI services.
```bash
export DEFAULT_VERTEXAI_LOCATION="global"
# or
export DEFAULT_VERTEXAI_LOCATION="us-central1"
```
Common locations:
- `global` - For Discovery Engine and global services
- `us-central1` - US Central region
- `us-east1` - US East region
- `europe-west1` - Europe West region
- `asia-southeast1` - Asia Southeast region
### 3. **DEFAULT_GOOGLE_APPLICATION_CREDENTIALS** (Required)
Path to your service account JSON key file.
```bash
export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
```
### 4. **GOOGLE_APPLICATION_CREDENTIALS** (Fallback)
Standard Google Cloud environment variable (used as fallback).
```bash
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
```
## Quick Setup
### Option 1: Interactive Script
```bash
chmod +x setup_vertex_env.sh
source setup_vertex_env.sh
```
### Option 2: Manual Setup
1. **Set environment variables** (for current session):
```bash
export DEFAULT_VERTEXAI_PROJECT="your-project-id"
export DEFAULT_VERTEXAI_LOCATION="global"
export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
```
2. **Make them persistent** (add to `~/.zshrc` or `~/.bashrc`):
```bash
echo 'export DEFAULT_VERTEXAI_PROJECT="your-project-id"' >> ~/.zshrc
echo 'export DEFAULT_VERTEXAI_LOCATION="global"' >> ~/.zshrc
echo 'export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
echo 'export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
```
3. **Reload your shell**:
```bash
source ~/.zshrc
```
## Service Account Setup
### 1. Create a Service Account
```bash
gcloud iam service-accounts create litellm-vertex-sa \
--display-name="LiteLLM Vertex AI Service Account"
```
### 2. Grant Necessary Permissions
For Discovery Engine (vector stores):
```bash
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/discoveryengine.viewer"
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/discoveryengine.dataStoreEditor"
```
For general Vertex AI:
```bash
gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
--member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
--role="roles/aiplatform.user"
```
### 3. Create and Download Key
```bash
gcloud iam service-accounts keys create ~/service-account-key.json \
--iam-account=litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com
```
## Verify Setup
### Check Environment Variables
```bash
python3 << 'EOF'
import os
print("✓ Environment Variables:")
print(f" DEFAULT_VERTEXAI_PROJECT: {os.getenv('DEFAULT_VERTEXAI_PROJECT')}")
print(f" DEFAULT_VERTEXAI_LOCATION: {os.getenv('DEFAULT_VERTEXAI_LOCATION')}")
print(f" DEFAULT_GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')}")
print(f" GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}")
# Check if credentials file exists
creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
if creds_path and os.path.exists(creds_path):
print(f"\n✅ Credentials file found at: {creds_path}")
else:
print(f"\n❌ Credentials file NOT found at: {creds_path}")
EOF
```
### Test Authentication
```bash
python3 << 'EOF'
import os
import json
from google.oauth2 import service_account
from google.auth.transport.requests import Request
creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
project = os.getenv('DEFAULT_VERTEXAI_PROJECT')
try:
# Load credentials
credentials = service_account.Credentials.from_service_account_file(
creds_path,
scopes=['https://www.googleapis.com/auth/cloud-platform']
)
# Get access token
credentials.refresh(Request())
print("✅ Authentication successful!")
print(f" Project: {project}")
print(f" Service Account: {credentials.service_account_email}")
print(f" Token expiry: {credentials.expiry}")
except Exception as e:
print(f"❌ Authentication failed: {e}")
EOF
```
## Using with Vector Store Passthrough
Once your environment is set up, the vector store passthrough will work in two ways:
### 1. **With Vector Store Config** (Priority 1)
If you have a vector store configured with its own credentials in `litellm_params`, those will be used first:
```yaml
vector_stores:
- vector_store_id: test-store-123
custom_llm_provider: vertex_ai
litellm_params:
vertex_project: "specific-project"
vertex_location: "us-central1"
vertex_credentials: "{...}" # Inline credentials
```
### 2. **Environment Variables Fallback** (Priority 2)
If the vector store doesn't have explicit credentials, it falls back to your environment variables:
```yaml
vector_stores:
- vector_store_id: test-store-123
custom_llm_provider: vertex_ai
# No litellm_params - will use DEFAULT_VERTEXAI_PROJECT, DEFAULT_VERTEXAI_LOCATION, etc.
```
### 3. **Model Config Fallback** (Priority 3)
If neither above work, it looks for credentials in your model configuration.
## Troubleshooting
### "No credentials found"
Check that all environment variables are set:
```bash
env | grep -E "(DEFAULT_VERTEXAI|GOOGLE_APPLICATION_CREDENTIALS)"
```
### "Authentication failed"
Verify your service account key is valid:
```bash
cat $DEFAULT_GOOGLE_APPLICATION_CREDENTIALS | python3 -m json.tool
```
### "Permission denied"
Ensure your service account has the necessary roles:
```bash
gcloud projects get-iam-policy YOUR_PROJECT_ID \
--flatten="bindings[].members" \
--filter="bindings.members:serviceAccount:litellm-vertex-sa@*"
```
### Different Credentials for Different Projects
If you need to use different credentials for different vector stores, configure them explicitly in the vector store config rather than relying on environment variables.
## Start LiteLLM Proxy
Once your environment is configured:
```bash
# Start the proxy (it will automatically load env vars)
litellm --config proxy_server_config.yaml
# Or with debug logging
export LITELLM_LOG=DEBUG
litellm --config proxy_server_config.yaml
```
You should see logs like:
```
Vertex: Loading vertex credentials from /path/to/service-account.json
Found credentials for vertex_ai_default
```
## Test the Endpoint
```bash
curl -X POST http://0.0.0.0:4000/vertex_ai/discovery/v1/projects/fake-project/locations/global/dataStores/test-store-123/servingConfigs/default_config:search \
-H 'Authorization: Bearer YOUR_LITELLM_API_KEY' \
-H 'Content-Type: application/json' \
-d '{"query": "test query"}'
```
The proxy will use your environment credentials to make the request to Vertex AI!

12
cookbook/LiteLLM_CometAPI.ipynb vendored
View file

@ -28,7 +28,7 @@
"Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n",
"Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n",
"Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n",
"Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
"Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n",
"Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n",
"Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n",
@ -50,11 +50,11 @@
"Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n",
"Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n",
"Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n",
"Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n",
"Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n",

1
cookbook/misc/update_json_caching.py
View file

@ -10,7 +10,6 @@ models_to_update = [
"gpt-4o-2024-05-13",
"text-embedding-3-small",
"text-embedding-3-large",
"text-embedding-ada-002-v2",
"ft:gpt-4o-2024-08-06",
"ft:gpt-4o-mini-2024-07-18",
"ft:gpt-3.5-turbo",

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.7
version: 0.4.8
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

3
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -22,6 +22,9 @@ spec:
metadata:
labels:
{{- include "litellm.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}

7
docs/my-website/.trivyignore Normal file
View file

@ -0,0 +1,7 @@
# js-yaml CVE-2025-64718
# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1
# via npm overrides in package.json. Trivy incorrectly reports this based on
# dependency requirements in the lockfile, but the actual installed version is 4.1.1.
# Verified with: npm list js-yaml
CVE-2025-64718

24
docs/my-website/blog/authors.yml Normal file
View file

@ -0,0 +1,24 @@
litellm:
name: LiteLLM Team
title: LiteLLM Core Team
url: https://github.com/BerriAI/litellm
image_url: https://github.com/BerriAI.png
krrish:
name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
ishaan:
name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
# Alias for typo in name
ishaan-alt:
name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg

700
docs/my-website/blog/gemini_3/index.md Normal file
View file

@ -0,0 +1,700 @@
---
slug: gemini_3
title: "DAY 0 Support: Gemini 3 on LiteLLM"
date: 2025-11-19T10:00:00
authors:
- name: Sameer Kankute
title: "SWE @ LiteLLM (LLM Translation)"
url: https://in.linkedin.com/in/sameer-kankute
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::info
This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK.
:::
## Quick Start
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "Hello!"}],
reasoning_effort="low"
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Add to config.yaml:**
```yaml
model_list:
- model_name: gemini-3-pro-preview
litellm_params:
model: gemini/gemini-3-pro-preview
api_key: os.environ/GEMINI_API_KEY
```
**2. Start proxy:**
```bash
litellm --config /path/to/config.yaml
```
**3. Make request:**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "Hello!"}],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
Both endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
## Thought Signatures
#### What are Thought Signatures?
Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling.
#### How Thought Signatures Work
1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response
2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls
3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini
## Example: Multi-Turn Function Calling
#### Streaming with Thought Signatures
When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved:
<Tabs>
<TabItem value="streaming" label="Streaming SDK">
```python
import os
import litellm
from litellm import completion
os.environ["GEMINI_API_KEY"] = "your-api-key"
MODEL = "gemini/gemini-3-pro-preview"
messages = [
{"role": "system", "content": "You are a helpful assistant. Use the calculate tool."},
{"role": "user", "content": "What is 2+2?"},
]
tools = [{
"type": "function",
"function": {
"name": "calculate",
"description": "Calculate a mathematical expression",
"parameters": {
"type": "object",
"properties": {"expression": {"type": "string"}},
"required": ["expression"],
},
},
}]
print("Step 1: Sending request with stream=True...")
response = completion(
model=MODEL,
messages=messages,
stream=True,
tools=tools,
reasoning_effort="low"
)
# Collect all chunks
chunks = []
for part in response:
chunks.append(part)
# Reconstruct message using stream_chunk_builder
# Thought signatures are now preserved automatically!
full_response = litellm.stream_chunk_builder(chunks, messages=messages)
print(f"Full response: {full_response}")
assistant_msg = full_response.choices[0].message
# ✅ Thought signature is now preserved in provider_specific_fields
if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields:
thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature")
print(f"Thought signature preserved: {thought_sig is not None}")
# Append assistant message (includes thought signatures automatically)
messages.append(assistant_msg)
# Mock tool execution
messages.append({
"role": "tool",
"content": "4",
"tool_call_id": assistant_msg.tool_calls[0].id
})
print("\nStep 2: Sending tool result back to model...")
response_2 = completion(
model=MODEL,
messages=messages,
stream=True,
tools=tools,
reasoning_effort="low"
)
for part in response_2:
if part.choices[0].delta.content:
print(part.choices[0].delta.content, end="")
print() # New line
```
**Key Points:**
- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures
- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history
- ✅ Multi-turn conversations work seamlessly with streaming
</TabItem>
<TabItem value="sdk" label="Non-Streaming SDK">
```python
from openai import OpenAI
import json
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
# Define tools
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
# Step 1: Initial request
messages = [{"role": "user", "content": "What's the weather in Tokyo?"}]
response = client.chat.completions.create(
model="gemini-3-pro-preview",
messages=messages,
tools=tools,
reasoning_effort="low"
)
# Step 2: Append assistant message (thought signatures automatically preserved)
messages.append(response.choices[0].message)
# Step 3: Execute tool and append result
for tool_call in response.choices[0].message.tool_calls:
if tool_call.function.name == "get_weather":
result = {"temperature": 30, "unit": "celsius"}
messages.append({
"role": "tool",
"content": json.dumps(result),
"tool_call_id": tool_call.id
})
# Step 4: Follow-up request (thought signatures automatically included)
response2 = client.chat.completions.create(
model="gemini-3-pro-preview",
messages=messages,
tools=tools,
reasoning_effort="low"
)
print(response2.choices[0].message.content)
```
**Key Points:**
- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature`
- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved
- ✅ You don't need to manually extract or manage thought signatures
</TabItem>
<TabItem value="proxy" label="cURL">
```bash
# Step 1: Initial request
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [
{"role": "user", "content": "What'\''s the weather in Tokyo?"}
],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
],
"reasoning_effort": "low"
}'
```
**Response includes thought signature:**
```json
{
"choices": [{
"message": {
"role": "assistant",
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Tokyo\"}"
},
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
}
}]
}
}]
}
```
```bash
# Step 2: Follow-up request (include assistant message with thought signature)
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [
{"role": "user", "content": "What'\''s the weather in Tokyo?"},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Tokyo\"}"
},
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
}
}]
},
{
"role": "tool",
"content": "{\"temperature\": 30, \"unit\": \"celsius\"}",
"tool_call_id": "call_abc123"
}
],
"tools": [...],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
#### Important Notes on Thought Signatures
1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them.
2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature.
3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved.
4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning.
## Conversation History: Switching from Non-Gemini-3 Models
#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history?
**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed.
#### How It Works
When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM:
1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures
2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility
3. **Maintains conversation flow**: Your conversation history continues to work seamlessly
#### Example: Switching Models Mid-Conversation
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from openai import OpenAI
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
# Step 1: Start with gemini-2.5-flash (no thought signatures)
messages = [{"role": "user", "content": "What's the weather?"}]
response1 = client.chat.completions.create(
model="gemini-2.5-flash",
messages=messages,
tools=[...],
reasoning_effort="low"
)
# Append assistant message (no tool call thought signature from gemini-2.5-flash)
messages.append(response1.choices[0].message)
# Step 2: Switch to gemini-3-pro-preview
# LiteLLM automatically adds dummy thought signature to the previous assistant message
response2 = client.chat.completions.create(
model="gemini-3-pro-preview", # 👈 Switched model
messages=messages, # 👈 Same conversation history
tools=[...],
reasoning_effort="low"
)
# ✅ Works seamlessly! No errors, no breaking changes
print(response2.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="cURL">
```bash
# Step 1: Start with gemini-2.5-flash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-2.5-flash",
"messages": [{"role": "user", "content": "What'\''s the weather?"}],
"tools": [...],
"reasoning_effort": "low"
}'
# Step 2: Switch to gemini-3-pro-preview with same conversation history
# LiteLLM automatically handles the missing thought signature
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview", # 👈 Switched model
"messages": [
{"role": "user", "content": "What'\''s the weather?"},
{
"role": "assistant",
"tool_calls": [...] # 👈 No thought_signature from gemini-2.5-flash
}
],
"tools": [...],
"reasoning_effort": "low"
}'
# ✅ Works! LiteLLM adds dummy signature automatically
```
</TabItem>
</Tabs>
#### Dummy Signature Details
The dummy signature used is: `base64("skip_thought_signature_validator")`
This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to:
- Accept the conversation history without validation errors
- Continue the conversation seamlessly
- Maintain context across model switches
## Thinking Level Parameter
#### How `reasoning_effort` Maps to `thinking_level`
For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter:
| `reasoning_effort` | `thinking_level` | Notes |
|-------------------|------------------|-------|
| `"minimal"` | `"low"` | Maps to low thinking level |
| `"low"` | `"low"` | Default for most use cases |
| `"medium"` | `"high"` | Medium not available yet, maps to high |
| `"high"` | `"high"` | Maximum reasoning depth |
| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking |
| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking |
#### Default Behavior
If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs.
### Example Usage
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from litellm import completion
# Low thinking level (faster, lower cost)
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "What's the weather?"}],
reasoning_effort="low" # Maps to thinking_level="low"
)
# High thinking level (deeper reasoning, higher cost)
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "Solve this complex math problem step by step."}],
reasoning_effort="high" # Maps to thinking_level="high"
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```bash
# Low thinking level
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "What'\''s the weather?"}],
"reasoning_effort": "low"
}'
# High thinking level
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "Solve this complex problem."}],
"reasoning_effort": "high"
}'
```
</TabItem>
</Tabs>
## Important Notes
1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`.
2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause:
- Infinite loops
- Degraded reasoning performance
- Failure on complex tasks
3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance.
## Using with Claude Code CLI
You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows.
### Setup
**1. Add Gemini 3 Pro Preview to your `config.yaml`:**
```yaml
model_list:
- model_name: gemini-3-pro-preview
litellm_params:
model: gemini/gemini-3-pro-preview
api_key: os.environ/GEMINI_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
**2. Set environment variables:**
```bash
export GEMINI_API_KEY="your-gemini-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
**3. Start LiteLLM Proxy:**
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
**4. Configure Claude Code to use LiteLLM Proxy:**
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
**5. Use Gemini 3 Pro Preview with Claude Code:**
```bash
# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy
claude --model gemini-3-pro-preview
```
### Example Usage
Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface:
```bash
$ claude --model gemini-3-pro-preview
> Explain how thought signatures work in multi-turn conversations.
# Gemini 3 Pro Preview responds through Claude Code interface
```
### Benefits
- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface
- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy
- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging
- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models
- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code
### Troubleshooting
**Claude Code not finding the model:**
- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview`
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy
**Authentication errors:**
- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
- Ensure `GEMINI_API_KEY` is set correctly
- Check LiteLLM proxy logs for detailed error messages
## Best Practices
#### 1. Always Include Thought Signatures in Conversation History
When building multi-turn conversations with function calling:
✅ **Do:**
```python
# Append the full assistant message (includes thought signatures)
messages.append(response.choices[0].message)
```
❌ **Don't:**
```python
# Don't manually construct assistant messages without thought signatures
messages.append({
"role": "assistant",
"tool_calls": [...] # Missing thought signatures!
})
```
#### 2. Use Appropriate Thinking Levels
- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization
- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning
#### 3. Keep Temperature at Default
For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues.
#### 4. Handle Model Switches Gracefully
When switching from non-Gemini-3 to Gemini-3:
- ✅ LiteLLM automatically handles missing thought signatures
- ✅ No manual intervention needed
- ✅ Conversation history continues seamlessly
## Troubleshooting
#### Issue: Missing Thought Signatures
**Symptom**: Error when including assistant messages in conversation history
**Solution**: Ensure you're appending the full assistant message from the response:
```python
messages.append(response.choices[0].message) # ✅ Includes thought signatures
```
#### Issue: Conversation Breaks When Switching Models
**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview
**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version.
#### Issue: Infinite Loops or Poor Performance
**Symptom**: Model gets stuck or produces poor results
**Solution**:
- Ensure `temperature=1.0` (default for Gemini 3)
- Check that `reasoning_effort` is set appropriately
- Verify you're using the correct model name: `gemini/gemini-3-pro-preview`
## Additional Resources
- [Gemini Provider Documentation](../gemini.md)
- [Thought Signatures Guide](../gemini.md#thought-signatures)
- [Reasoning Content Documentation](../../reasoning_content.md)
- [Function Calling Guide](../../function_calling.md)

52
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
View file

@ -37,57 +37,7 @@ Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization).
`my_guardrail.py`:
```python
import os
from typing import Optional, List
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class MyGuardrail(CustomGuardrail):
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(default_on=True)
async def apply_guardrail(
self,
text: str,
language: Optional[str] = None,
entities: Optional[List[PiiEntityType]] = None,
request_data: Optional[dict] = None,
) -> str:
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
response.raise_for_status()
return response.json()
```
Follow from [Custom Guardrail](../proxy/guardrails/custom_guardrail#custom-guardrail) tutorial.
### Create the Init File

23
docs/my-website/docs/benchmarks.md
View file

@ -125,18 +125,23 @@ class MyUser(HttpUser):
## LiteLLM vs Portkey Performance Comparison
**Test Configuration**: 4 CPUs, 8 GB RAM per instance | Load: 1k concurrent users, 500 ramp-up
**Versions:** Portkey **v1.14.0** | LiteLLM **v1.79.1-stable**
**Test Duration:** 5 minutes
### Multi-Instance (4×) Performance
| Metric | Portkey (no DB) | LiteLLM (with DB) |
| ------------------- | --------------- | ----------------- |
| **Total Requests** | 293,796 | 312,405 |
| **Failed Requests** | 0 | 0 |
| **Median Latency** | 100 ms | 100 ms |
| **p95 Latency** | 230 ms | 150 ms |
| **p99 Latency** | 500 ms | 240 ms |
| **Average Latency** | 123 ms | 111 ms |
| **Current RPS** | 1,170.9 | 1,170 |
| Metric | Portkey (no DB) | LiteLLM (with DB) | Comment |
| ------------------- | --------------- | ----------------- | -------------- |
| **Total Requests** | 293,796 | 312,405 | LiteLLM higher |
| **Failed Requests** | 0 | 0 | Same |
| **Median Latency** | 100 ms | 100 ms | Same |
| **p95 Latency** | 230 ms | 150 ms | LiteLLM lower |
| **p99 Latency** | 500 ms | 240 ms | LiteLLM lower |
| **Average Latency** | 123 ms | 111 ms | LiteLLM lower |
| **Current RPS** | 1,170.9 | 1,170 | Same |
*Lower is better for latency metrics; higher is better for requests and RPS.*
### Technical Insights

2
docs/my-website/docs/embedding/supported_embedding.md
View file

@ -196,7 +196,7 @@ input=["good morning from litellm"]
]
}
],
"model": "text-embedding-ada-002-v2",
"model": "text-embedding-ada-002",
"usage": {
"prompt_tokens": 10,
"total_tokens": 10

7
docs/my-website/docs/enterprise.md
View file

@ -3,7 +3,8 @@ import Image from '@theme/IdealImage';
# Enterprise
:::info
✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
- ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs.
:::
For companies that need SSO, user management and professional support for LiteLLM Proxy
@ -16,7 +17,7 @@ Get free 7-day trial key [here](https://www.litellm.ai/enterprise#trial)
Includes all enterprise features.
<Image img={require('../img/enterprise_vs_oss.png')} />
<Image img={require('../img/enterprise_vs_oss_2.png')} />
[**Procurement available via AWS / Azure Marketplace**](./data_security.md#legalcompliance-faqs)
@ -40,7 +41,7 @@ Self-Managed Enterprise deployments require our team to understand your exact ne
### How does deployment with Enterprise License work?
You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) and get an enterprise license key to add to your environment to unlock additional functionality (SSO, Prometheus metrics, etc.).
You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) and get an enterprise license key to add to your environment to unlock additional functionality (SSO, etc.).
```env
LITELLM_LICENSE="eyJ..."

15
docs/my-website/docs/extras/contributing_code.md
View file

@ -107,3 +107,18 @@ docker run \
litellm_test_image \
--config /app/config.yaml --detailed_debug
```
### Running LiteLLM Proxy Locally
1. cd into the `proxy/` directory
```
cd litellm/litellm/proxy
```
2. Run the proxy
```shell
python3 proxy_cli.py --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```

98
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -197,6 +197,53 @@ for idx, image_obj in enumerate(response.data):
f.write(base64.b64decode(image_obj.b64_json))
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
#### Basic Image Edit (Gemini)
```python showLineNumbers title="Vertex AI Gemini Image Edit"
import os
import litellm
# Set Vertex AI credentials
os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json"
response = litellm.image_edit(
model="vertex_ai/gemini-2.5-flash",
image=open("original_image.png", "rb"),
prompt="Add neon lights in the background",
size="1024x1024",
)
print(response)
```
#### Image Edit with Imagen (Supports Masks)
```python showLineNumbers title="Vertex AI Imagen Image Edit"
import os
import litellm
# Set Vertex AI credentials
os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json"
# Imagen supports mask for inpainting
response = litellm.image_edit(
model="vertex_ai/imagen-3.0-capability-001",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"), # Optional: for inpainting
prompt="Turn this into watercolor style scenery",
n=2, # Number of variations
size="1024x1024",
)
print(response)
```
</TabItem>
</Tabs>
@ -302,6 +349,55 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-F "size=1024x1024"
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
1. Add Vertex AI image edit models to your `config.yaml`:
```yaml showLineNumbers title="Vertex AI Proxy Configuration"
model_list:
- model_name: vertex-gemini-image-edit
litellm_params:
model: vertex_ai/gemini-2.5-flash
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
- model_name: vertex-imagen-image-edit
litellm_params:
model: vertex_ai/imagen-3.0-capability-001
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="Vertex AI Gemini Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=vertex-gemini-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Add neon lights in the background" \
-F "size=1024x1024"
```
4. Imagen image edit with mask:
```bash showLineNumbers title="Vertex AI Imagen Proxy Image Edit with Mask"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=vertex-imagen-image-edit" \
-F "image=@original_image.png" \
-F "mask=@mask_image.png" \
-F "prompt=Turn this into watercolor style scenery" \
-F "n=2" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

86
docs/my-website/docs/mcp.md
View file

@ -211,11 +211,12 @@ mcp_servers:
oauth2_example:
url: "https://my-mcp-server.com/mcp"
auth_type: "oauth2" # 👈 KEY CHANGE
authorization_url: "https://my-mcp-server.com/oauth/authorize" # optional for client-credentials
token_url: "https://my-mcp-server.com/oauth/token" # required
authorization_url: "https://my-mcp-server.com/oauth/authorize" # optional override
token_url: "https://my-mcp-server.com/oauth/token" # optional override
registration_url: "https://my-mcp-server.com/oauth/register" # optional override
client_id: os.environ/OAUTH_CLIENT_ID
client_secret: os.environ/OAUTH_CLIENT_SECRET
scopes: ["tool.read", "tool.write"] # optional
scopes: ["tool.read", "tool.write"] # optional override
bearer_example:
url: "https://my-mcp-server.com/mcp"
@ -325,6 +326,10 @@ mcp_servers:
| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
| `auth_value` | No | Authentication value (required if `auth_type` is set) |
| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
| `description` | No | Optional description for the MCP server |
| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
@ -1216,7 +1221,6 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
This configuration is currently available on the config.yaml, with UI support coming soon.
```yaml
@ -1224,19 +1228,77 @@ mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
authorization_url: https://github.com/login/oauth/authorize
token_url: https://github.com/login/oauth/access_token
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
scopes: ["public_repo", "user:email"]
```
**Note**
In the future, users will only need to specify the `url` of the MCP server.
LiteLLM will automatically resolve the corresponding `authorization_url`, `token_url`, and `registration_url` based on the MCP server metadata (e.g., `.well-known/oauth-authorization-server` or `oauth-protected-resource`).
[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
### How It Works
```mermaid
sequenceDiagram
participant Browser as User-Agent (Browser)
participant Client as Client
participant LiteLLM as LiteLLM Proxy
participant MCP as MCP Server (Resource Server)
participant Auth as Authorization Server
Note over Client,LiteLLM: Step 1 – Resource discovery
Client->>LiteLLM: GET /.well-known/oauth-protected-resource/{mcp_server_name}/mcp
LiteLLM->>Client: Return resource metadata
Note over Client,LiteLLM: Step 2 – Authorization server discovery
Client->>LiteLLM: GET /.well-known/oauth-authorization-server/{mcp_server_name}
LiteLLM->>Client: Return authorization server metadata
Note over Client,Auth: Step 3 – Dynamic client registration
Client->>LiteLLM: POST /{mcp_server_name}/register
LiteLLM->>Auth: Forward registration request
Auth->>LiteLLM: Issue client credentials
LiteLLM->>Client: Return client credentials
Note over Client,Browser: Step 4 – User authorization (PKCE)
Client->>Browser: Open authorization URL + code_challenge + resource
Browser->>Auth: Authorization request
Note over Auth: User authorizes
Auth->>Browser: Redirect with authorization code
Browser->>LiteLLM: Callback to LiteLLM with code
LiteLLM->>Browser: Redirect back with authorization code
Browser->>Client: Callback with authorization code
Note over Client,Auth: Step 5 – Token exchange
Client->>LiteLLM: Token request + code_verifier + resource
LiteLLM->>Auth: Forward token request
Auth->>LiteLLM: Access (and refresh) token
LiteLLM->>Client: Return tokens
Note over Client,MCP: Step 6 – Authenticated MCP call
Client->>LiteLLM: MCP request with access token + LiteLLM API key
LiteLLM->>MCP: MCP request with Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: Return MCP response
```
**Participants**
- **Client** – The MCP-capable AI agent (e.g., Claude Code, Cursor, or another IDE/agent) that initiates OAuth discovery, authorization, and tool invocations on behalf of the user.
- **LiteLLM Proxy** – Mediates all OAuth discovery, registration, token exchange, and MCP traffic while protecting stored credentials.
- **Authorization Server** – Issues OAuth 2.0 tokens via dynamic client registration, PKCE authorization, and token endpoints.
- **MCP Server (Resource Server)** – The protected MCP endpoint that receives LiteLLM’s authenticated JSON-RPC requests.
- **User-Agent (Browser)** – Temporarily involved so the end user can grant consent during the authorization step.
**Flow Steps**
1. **Resource Discovery**: The client fetches MCP resource metadata from LiteLLM’s `.well-known/oauth-protected-resource` endpoint to understand scopes and capabilities.
2. **Authorization Server Discovery**: The client retrieves the OAuth server metadata (token endpoint, authorization endpoint, supported PKCE methods) through LiteLLM’s `.well-known/oauth-authorization-server` endpoint.
3. **Dynamic Client Registration**: The client registers through LiteLLM, which forwards the request to the authorization server (RFC 7591). If the provider doesn’t support dynamic registration, you can pre-store `client_id`/`client_secret` in LiteLLM (e.g., GitHub MCP) and the flow proceeds the same way.
4. **User Authorization**: The client launches a browser session (with code challenge and resource hints). The user approves access, the authorization server sends the code through LiteLLM back to the client.
5. **Token Exchange**: The client calls LiteLLM with the authorization code, code verifier, and resource. LiteLLM exchanges them with the authorization server and returns the issued access/refresh tokens.
6. **MCP Invocation**: With a valid token, the client sends the MCP JSON-RPC request (plus LiteLLM API key) to LiteLLM, which forwards it to the MCP server and relays the tool response.
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
@ -1887,4 +1949,4 @@ async with stdio_client(server_params) as (read, write):
```
</TabItem>
</Tabs>
</Tabs>

24
docs/my-website/docs/providers/anthropic.md
View file

@ -953,6 +953,30 @@ except Exception as e:
s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this!
### Context Management (Beta)
Anthropic’s [context editing](https://docs.claude.com/en/docs/build-with-claude/context-editing) API lets you automatically clear older tool results or thinking blocks. LiteLLM now forwards the native `context_management` payload when you call Anthropic models, and automatically attaches the required `context-management-2025-06-27` beta header.
```python
from litellm import completion
response = completion(
model="anthropic/claude-sonnet-4-20250514",
messages=[{"role": "user", "content": "Summarize the latest tool results"}],
context_management={
"edits": [
{
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": 30000},
"keep": {"type": "tool_uses", "value": 3},
"clear_at_least": {"type": "input_tokens", "value": 5000},
"exclude_tools": ["web_search"],
}
]
},
)
```
### Anthropic Hosted Tools (Computer, Text Editor, Web Search, Memory)

83
docs/my-website/docs/providers/azure_ai_speech.md
View file

@ -136,6 +136,89 @@ response = speech(
| `wav` | riff-24khz-16bit-mono-pcm | 24kHz |
| `pcm` | raw-24khz-16bit-mono-pcm | 24kHz |
## Passing Raw SSML
LiteLLM automatically detects when your `input` contains SSML (by checking for `<speak>` tags) and passes it through to Azure without any transformation. This gives you complete control over speech synthesis.
**When to use raw SSML:**
- Using the `<lang>` element with multilingual voices to translate text (e.g., English text → Spanish speech)
- Complex SSML structures with multiple voices or prosody changes
- Fine-grained control over pronunciation, breaks, emphasis, and other speech features
### LiteLLM SDK
```python showLineNumbers title="Raw SSML for Multilingual Translation"
from litellm import speech
# Use <lang> element to convert English text to Spanish speech
# The <lang> element forces the output language regardless of input text language
language_code = "es-ES"
text = "Hello, how are you today?" # English text
voice = "en-US-AvaMultilingualNeural"
ssml = f"""<speak version="1.0"
xmlns="http://www.w3.org/2001/10/synthesis"
xmlns:mstts="http://www.w3.org/2001/mstts"
xml:lang="{language_code}">
<voice name="{voice}">
<lang xml:lang="{language_code}">{text}</lang>
</voice>
</speak>"""
response = speech(
model="azure/speech/azure-tts",
voice=voice,
input=ssml, # LiteLLM auto-detects SSML and sends as-is
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
response.stream_to_file("speech.mp3")
```
```python showLineNumbers title="Raw SSML with Complex Features"
from litellm import speech
# Complex SSML with multiple prosody adjustments
ssml = """<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis'
xmlns:mstts='https://www.w3.org/2001/mstts' xml:lang='en-US'>
<voice name='en-US-JennyNeural'>
<mstts:express-as style='cheerful' styledegree='2'>
<prosody rate='+20%' pitch='high'>
Welcome to our service!
</prosody>
</mstts:express-as>
<break time='500ms'/>
<prosody rate='-10%'>
How can I help you today?
</prosody>
</voice>
</speak>"""
response = speech(
model="azure/speech/azure-tts",
voice="en-US-JennyNeural",
input=ssml, # LiteLLM detects <speak> and passes through unchanged
api_base="https://eastus.tts.speech.microsoft.com",
api_key=os.environ["AZURE_TTS_API_KEY"],
)
response.stream_to_file("speech.mp3")
```
### LiteLLM Proxy
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "azure-speech",
"voice": "en-US-AvaMultilingualNeural",
"input": "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" xmlns:mstts=\"http://www.w3.org/2001/mstts\" xml:lang=\"es-ES\"><voice name=\"en-US-AvaMultilingualNeural\"><lang xml:lang=\"es-ES\">Hello, how are you today?</lang></voice></speak>"
}' \
--output speech.mp3
```
## Sending Azure-Specific Params
Azure AI Speech supports advanced SSML features through optional parameters:

4
docs/my-website/docs/providers/fal_ai.md
View file

@ -31,10 +31,14 @@ Get your API key from [fal.ai](https://fal.ai/).
| Model Name | Description | Documentation |
|------------|-------------|---------------|
| `fal_ai/fal-ai/flux-pro/v1.1` | FLUX Pro v1.1 - Balanced speed and quality | [Docs ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1) |
| `fal_ai/flux/schnell` | Flux Schnell - Low-latency generation with `image_size` support | [Docs ↗](https://fal.ai/models/fal-ai/flux/schnell) |
| `fal_ai/fal-ai/bytedance/seedream/v3/text-to-image` | ByteDance Seedream v3 - Text-to-image with `image_size` control | [Docs ↗](https://fal.ai/models/fal-ai/bytedance/seedream/v3/text-to-image) |
| `fal_ai/fal-ai/bytedance/dreamina/v3.1/text-to-image` | ByteDance Dreamina v3.1 - Text-to-image with `image_size` control | [Docs ↗](https://fal.ai/models/fal-ai/bytedance/dreamina/v3.1/text-to-image) |
| `fal_ai/fal-ai/flux-pro/v1.1-ultra` | FLUX Pro v1.1 Ultra - High-quality image generation | [Docs ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra) |
| `fal_ai/fal-ai/imagen4/preview` | Google's Imagen 4 - Highest quality model | [Docs ↗](https://fal.ai/models/fal-ai/imagen4/preview) |
| `fal_ai/fal-ai/recraft/v3/text-to-image` | Recraft v3 - Multiple style options | [Docs ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image) |
| `fal_ai/fal-ai/ideogram/v3` | Ideogram v3 - Lettering-first creative model (Balanced: $0.06/image) | [Docs ↗](https://fal.ai/models/fal-ai/ideogram/v3) |
| `fal_ai/fal-ai/stable-diffusion-v35-medium` | Stable Diffusion v3.5 Medium | [Docs ↗](https://fal.ai/models/fal-ai/stable-diffusion-v35-medium) |
| `fal_ai/bria/text-to-image/3.2` | Bria 3.2 - Commercial-grade generation | [Docs ↗](https://fal.ai/models/bria/text-to-image/3.2) |

409
docs/my-website/docs/providers/gemini.md
View file

@ -70,7 +70,11 @@ LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter.
Note: Reasoning cannot be turned off on Gemini 2.5 Pro models.
:::
**Mapping**
:::tip Gemini 3 Models
For **Gemini 3+ models** (e.g., `gemini-3-pro-preview`), LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter instead of `thinking_budget`. The `thinking_level` parameter uses `"low"` or `"high"` values for better control over reasoning depth.
:::
**Mapping for Gemini 2.5 and earlier models**
| reasoning_effort | thinking | Notes |
| ---------------- | -------- | ----- |
@ -80,6 +84,17 @@ Note: Reasoning cannot be turned off on Gemini 2.5 Pro models.
| "medium" | "budget_tokens": 2048 | |
| "high" | "budget_tokens": 4096 | |
**Mapping for Gemini 3+ models**
| reasoning_effort | thinking_level | Notes |
| ---------------- | -------------- | ----- |
| "minimal" | "low" | Minimizes latency and cost |
| "low" | "low" | Best for simple instruction following or chat |
| "medium" | "high" | Maps to high (medium not yet available) |
| "high" | "high" | Maximizes reasoning depth |
| "disable" | "low" | Cannot fully disable thinking in Gemini 3 |
| "none" | "low" | Cannot fully disable thinking in Gemini 3 |
<Tabs>
<TabItem value="sdk" label="SDK">
@ -137,6 +152,59 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
### Gemini 3+ Models - `thinking_level` Parameter
For Gemini 3+ models (e.g., `gemini-3-pro-preview`), you can use the new `thinking_level` parameter directly:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
# Use thinking_level for Gemini 3 models
resp = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "Solve this complex math problem step by step."}],
reasoning_effort="high", # Options: "low" or "high"
)
# Low thinking level for faster, simpler tasks
resp = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "What is the weather today?"}],
reasoning_effort="low", # Minimizes latency and cost
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "Solve this complex problem."}],
"reasoning_effort": "high"
}'
```
</TabItem>
</Tabs>
:::warning
**Temperature Recommendation for Gemini 3 Models**
For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause:
- Infinite loops
- Degraded reasoning performance
- Failure on complex tasks
LiteLLM will automatically set `temperature=1.0` if not specified for Gemini 3+ models.
:::
**Expected Response**
@ -951,6 +1019,295 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
## Thought Signatures
Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry.
Thought signatures are particularly important for multi-turn function calling scenarios where the model needs to maintain context across multiple tool invocations.
### How Thought Signatures Work
- **Function calls with signatures**: When Gemini returns a function call, it includes a `thought_signature` in the response
- **Preservation**: LiteLLM automatically extracts and stores thought signatures in `provider_specific_fields` of tool calls
- **Return in conversation history**: When you include the assistant's message with tool calls in subsequent requests, LiteLLM automatically preserves and returns the thought signatures to Gemini
- **Parallel function calls**: Only the first function call in a parallel set has a thought signature
- **Sequential function calls**: Each function call in a multi-step sequence has its own signature
### Enabling Thought Signatures
To enable thought signatures, you need to enable thinking/reasoning:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "What's the weather in Tokyo?"}],
tools=[...],
reasoning_effort="low", # Enable thinking to get thought signatures
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-2.5-flash",
"messages": [{"role": "user", "content": "What'\''s the weather in Tokyo?"}],
"tools": [...],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
### Multi-Turn Function Calling with Thought Signatures
When building conversation history for multi-turn function calling, you must include the thought signatures from previous responses. LiteLLM handles this automatically when you append the full assistant message to your conversation history.
<Tabs>
<TabItem value="sdk" label="OpenAI Client">
```python
from openai import OpenAI
import json
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
def get_current_temperature(location: str) -> dict:
"""Gets the current weather temperature for a given location."""
return {"temperature": 30, "unit": "celsius"}
def set_thermostat_temperature(temperature: int) -> dict:
"""Sets the thermostat to a desired temperature."""
return {"status": "success"}
get_weather_declaration = {
"name": "get_current_temperature",
"description": "Gets the current weather temperature for a given location.",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
"required": ["location"],
},
}
set_thermostat_declaration = {
"name": "set_thermostat_temperature",
"description": "Sets the thermostat to a desired temperature.",
"parameters": {
"type": "object",
"properties": {"temperature": {"type": "integer"}},
"required": ["temperature"],
},
}
# Initial request
messages = [
{"role": "user", "content": "If it's too hot or too cold in London, set the thermostat to a comfortable level."}
]
response = client.chat.completions.create(
model="gemini-2.5-flash",
messages=messages,
tools=[get_weather_declaration, set_thermostat_declaration],
reasoning_effort="low"
)
# Append the assistant's message (includes thought signatures automatically)
messages.append(response.choices[0].message)
# Execute tool calls and append results
for tool_call in response.choices[0].message.tool_calls:
if tool_call.function.name == "get_current_temperature":
result = get_current_temperature(**json.loads(tool_call.function.arguments))
messages.append({
"role": "tool",
"content": json.dumps(result),
"tool_call_id": tool_call.id
})
# Second request - thought signatures are automatically preserved
response2 = client.chat.completions.create(
model="gemini-2.5-flash",
messages=messages,
tools=[get_weather_declaration, set_thermostat_declaration],
reasoning_effort="low"
)
print(response2.choices[0].message.content)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash
# Step 1: Initial request
curl --location 'http://localhost:4000/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gemini-2.5-flash",
"messages": [
{
"role": "user",
"content": "If it'\''s too hot or too cold in London, set the thermostat to a comfortable level."
}
],
"tools": [
{
"type": "function",
"function": {
"name": "get_current_temperature",
"description": "Gets the current weather temperature for a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
},
{
"type": "function",
"function": {
"name": "set_thermostat_temperature",
"description": "Sets the thermostat to a desired temperature.",
"parameters": {
"type": "object",
"properties": {
"temperature": {"type": "integer"}
},
"required": ["temperature"]
}
}
}
],
"tool_choice": "auto",
"reasoning_effort": "low"
}'
```
The response will include tool calls with thought signatures in `provider_specific_fields`:
```json
{
"choices": [{
"message": {
"role": "assistant",
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_current_temperature",
"arguments": "{\"location\": \"London\"}"
},
"index": 0,
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ...=="
}
}]
}
}]
}
```
```bash
# Step 2: Follow-up request with tool response
# Include the assistant message from Step 1 (with thought signatures in provider_specific_fields)
curl --location 'http://localhost:4000/v1/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "gemini-2.5-flash",
"messages": [
{
"role": "user",
"content": "If it'\''s too hot or too cold in London, set the thermostat to a comfortable level."
},
{
"role": "assistant",
"content": null,
"tool_calls": [
{
"id": "call_c130b9f8c2c042e9b65e39a88245",
"type": "function",
"function": {
"name": "get_current_temperature",
"arguments": "{\"location\": \"London\"}"
},
"index": 0,
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ...=="
}
}
]
},
{
"role": "tool",
"content": "{\"temperature\": 30, \"unit\": \"celsius\"}",
"tool_call_id": "call_c130b9f8c2c042e9b65e39a88245"
}
],
"tools": [
{
"type": "function",
"function": {
"name": "get_current_temperature",
"description": "Gets the current weather temperature for a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
},
{
"type": "function",
"function": {
"name": "set_thermostat_temperature",
"description": "Sets the thermostat to a desired temperature.",
"parameters": {
"type": "object",
"properties": {
"temperature": {"type": "integer"}
},
"required": ["temperature"]
}
}
}
],
"tool_choice": "auto",
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
### Important Notes
1. **Automatic Handling**: LiteLLM automatically extracts thought signatures from Gemini responses and preserves them when you include assistant messages in conversation history. You don't need to manually extract or manage them.
2. **Parallel Function Calls**: When the model makes parallel function calls, only the first function call will have a thought signature. Subsequent parallel calls won't have signatures.
3. **Sequential Function Calls**: In multi-step function calling scenarios, each step's first function call will have its own thought signature that must be preserved.
4. **Required for Context**: Thought signatures are essential for maintaining reasoning context across multi-turn conversations with function calling. Without them, the model may lose context of its previous reasoning.
5. **Format**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls in the response, and are automatically included when you append the assistant message to your conversation history.
## JSON Mode
<Tabs>
@ -1022,6 +1379,56 @@ LiteLLM Supports the following image types passed in `url`
- Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg
- Image in local storage - ./localimage.jpeg
## Image Resolution Control (Gemini 3+)
For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images in your request.
**Supported `detail` values:**
- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
**Usage Example:**
```python
from litellm import completion
messages = [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/chart.png",
"detail": "high" # High resolution for detailed chart analysis
}
},
{
"type": "text",
"text": "Analyze this chart"
},
{
"type": "image_url",
"image_url": {
"url": "https://example.com/icon.png",
"detail": "low" # Low resolution for simple icon
}
}
]
}
]
response = completion(
model="gemini/gemini-3-pro-preview",
messages=messages,
)
```
:::info
**Per-Part Resolution:** Each image in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature is only available for Gemini 3+ models.
:::
## Sample Usage
```python
import os

121
docs/my-website/docs/providers/oci.md
View file

@ -58,12 +58,11 @@ This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrast
## Usage
<Tabs>
<TabItem value="manual" label="Manual Credentials">
<TabItem value="manual" label="Manual Credentials" default>
Input the parameters obtained from the OCI signing key creation process into the `completion` function:
```python
import os
from litellm import completion
messages = [{"role": "user", "content": "Hey! how's it going?"}]
@ -86,7 +85,7 @@ print(response)
```
</TabItem>
<TabItem value="oci-sdk" label="OCI SDK Signer" default>
<TabItem value="oci-sdk" label="OCI SDK Signer">
Use the OCI SDK `Signer` for authentication:
@ -153,7 +152,6 @@ For applications running on OCI compute instances:
from litellm import completion
from oci.auth.signers import InstancePrincipalsSecurityTokenSigner
oci.auth.signers.get_oke_workload_identity_resource_principal_signer()
# Use instance principal authentication
signer = InstancePrincipalsSecurityTokenSigner()
@ -168,7 +166,7 @@ response = completion(
print(response)
```
**Use workload identity authentication**
**Workload Identity Authentication**
For applications running in Oracle Kubernetes Engine (OKE):
@ -176,7 +174,7 @@ For applications running in Oracle Kubernetes Engine (OKE):
from litellm import completion
from oci.auth.signers import get_oke_workload_identity_resource_principal_signer
# Use instance principal authentication
# Use workload identity authentication
signer = get_oke_workload_identity_resource_principal_signer()
messages = [{"role": "user", "content": "Hey! how's it going?"}]
@ -196,10 +194,9 @@ print(response)
Just set `stream=True` when calling completion.
<Tabs>
<TabItem value="manual-stream" label="Manual Credentials">
<TabItem value="manual-stream" label="Manual Credentials" default>
```python
import os
from litellm import completion
messages = [{"role": "user", "content": "Hey! how's it going?"}]
@ -224,7 +221,7 @@ for chunk in response:
```
</TabItem>
<TabItem value="oci-sdk-stream" label="OCI SDK Signer" default>
<TabItem value="oci-sdk-stream" label="OCI SDK Signer">
```python
from litellm import completion
@ -258,7 +255,27 @@ for chunk in response:
### Using Cohere Models
<Tabs>
<TabItem value="cohere-sdk" label="OCI SDK Signer" default>
<TabItem value="cohere-manual" label="Manual Credentials" default>
```python
from litellm import completion
messages = [{"role": "user", "content": "Explain quantum computing"}]
response = completion(
model="oci/cohere.command-latest",
messages=messages,
oci_region="us-chicago-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```
</TabItem>
<TabItem value="cohere-sdk" label="OCI SDK Signer">
```python
from litellm import completion
@ -283,19 +300,28 @@ print(response)
```
</TabItem>
<TabItem value="cohere-manual" label="Manual Credentials">
</Tabs>
## Using Dedicated Endpoints
OCI supports dedicated endpoints for hosting models. Use the `oci_serving_mode="DEDICATED"` parameter along with `oci_endpoint_id` to specify the endpoint ID.
<Tabs>
<TabItem value="dedicated-manual" label="Manual Credentials" default>
```python
from litellm import completion
messages = [{"role": "user", "content": "Explain quantum computing"}]
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/cohere.command-latest",
model="oci/xai.grok-4", # Must match the model type hosted on the endpoint
messages=messages,
oci_region="us-chicago-1",
oci_region=<your_oci_region>,
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
@ -303,4 +329,69 @@ print(response)
```
</TabItem>
</Tabs>
<TabItem value="dedicated-sdk" label="OCI SDK Signer">
```python
from litellm import completion
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(
model="oci/xai.grok-4", # Must match the model type hosted on the endpoint
messages=messages,
oci_signer=signer,
oci_region="us-chicago-1",
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your dedicated endpoint OCID
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
</Tabs>
**Important:** When using `oci_serving_mode="DEDICATED"`:
- The `model` parameter **must match the type of model hosted on your dedicated endpoint** (e.g., use `"oci/cohere.command-latest"` for Cohere models, `"oci/xai.grok-4"` for Grok models)
- The model name determines the API format and vendor-specific handling (Cohere vs Generic)
- The `oci_endpoint_id` parameter specifies your dedicated endpoint's OCID
- If `oci_endpoint_id` is not provided, the `model` parameter will be used as the endpoint ID (for backward compatibility)
**Example with Cohere Dedicated Endpoint:**
```python
# For a dedicated endpoint hosting a Cohere model
response = completion(
model="oci/cohere.command-latest", # Use Cohere model name to get Cohere API format
messages=messages,
oci_region="us-chicago-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...", # Your Cohere endpoint OCID
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
```
## Optional Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `oci_region` | string | `us-ashburn-1` | OCI region where the GenAI service is deployed |
| `oci_serving_mode` | string | `ON_DEMAND` | Service mode: `ON_DEMAND` for managed models or `DEDICATED` for dedicated endpoints |
| `oci_endpoint_id` | string | Same as `model` | (For DEDICATED mode) The OCID of your dedicated endpoint |
| `oci_compartment_id` | string | **Required** | The OCID of the OCI compartment containing your resources |
| `oci_user` | string | - | (Manual auth) The OCID of the OCI user |
| `oci_fingerprint` | string | - | (Manual auth) The fingerprint of the API signing key |
| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy |
| `oci_key` | string | - | (Manual auth) The private key content as a string |
| `oci_key_file` | string | - | (Manual auth) Path to the private key file |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |

54
docs/my-website/docs/providers/openai.md
View file

@ -176,6 +176,9 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5-mini-2025-08-07 | `response = completion(model="gpt-5-mini-2025-08-07", messages=messages)` |
| gpt-5-nano-2025-08-07 | `response = completion(model="gpt-5-nano-2025-08-07", messages=messages)` |
| gpt-5-pro | `response = completion(model="gpt-5-pro", messages=messages)` |
| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |
| gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` |
| gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` |
| gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` |
| gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` |
| gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` |
@ -477,6 +480,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
| `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` |
| `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex-mini` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5-pro` | `high` | `high` only |
**Note:**
@ -486,6 +491,55 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements.
### Verbosity Control for GPT-5 Models
The `verbosity` parameter controls the length and detail of responses from GPT-5 family models. It accepts three values: `"low"`, `"medium"`, or `"high"`.
**Supported models:** `gpt-5`, `gpt-5.1`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro`
**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`) do **not** support the `verbosity` parameter.
**Use cases:**
- **`"low"`**: Best for concise answers or simple code generation (e.g., SQL queries)
- **`"medium"`**: Default - balanced output length
- **`"high"`**: Use when you need thorough explanations or extensive code refactoring
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Low verbosity - concise responses
response = litellm.completion(
model="gpt-5.1",
messages=[{"role": "user", "content": "Write a function to reverse a string"}],
verbosity="low"
)
# High verbosity - detailed responses
response = litellm.completion(
model="gpt-5.1",
messages=[{"role": "user", "content": "Explain how neural networks work"}],
verbosity="high"
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-5.1",
"messages": [{"role": "user", "content": "Write a function to reverse a string"}],
"verbosity": "low"
}'
```
</TabItem>
</Tabs>
## OpenAI Chat Completion to Responses API Bridge
Call any Responses API model from OpenAI's `/chat/completions` endpoint.

49
docs/my-website/docs/providers/snowflake.md
View file

@ -3,20 +3,15 @@ import TabItem from '@theme/TabItem';
# Snowflake
| Property | Details |
|-------|-------|
| Description | The Snowflake Cortex LLM REST API lets you access the COMPLETE function via HTTP POST requests|
| Provider Route on LiteLLM | `snowflake/` |
| Link to Provider Doc | [Snowflake ↗](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api) |
| Base URL | `https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete` |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions` |
| Property | Details |
|----------------------------|-----------------------------------------------------------------------------------------------------------|
| Description | The Snowflake Cortex LLM REST API lets you access the COMPLETE and EMBED functions via HTTP POST requests |
| Provider Route on LiteLLM | `snowflake/` |
| Link to Provider Doc | [Snowflake ↗](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-llm-rest-api) |
| Base URLs | `https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:complete`,`https://{account-id}.snowflakecomputing.com/api/v2/cortex/inference:embed`|
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings` |
Currently, Snowflake's REST API does not have an endpoint for `snowflake-arctic-embed` embedding models. If you want to use these embedding models with Litellm, you can call them through our Hugging Face provider.
Find the Arctic Embed models [here](https://huggingface.co/collections/Snowflake/arctic-embed-661fd57d50fab5fc314e4c18) on Hugging Face.
## Supported OpenAI Parameters
```
"temperature",
@ -29,6 +24,9 @@ Find the Arctic Embed models [here](https://huggingface.co/collections/Snowflake
Snowflake does have API keys. Instead, you access the Snowflake API with your JWT token and account identifier.
It is also possible to use [programmatic access tokens](https://docs.snowflake.com/en/user-guide/programmatic-access-tokens) (PAT). It can be defined by using 'pat/' prefix
```python
import os
os.environ["SNOWFLAKE_JWT"] = "YOUR JWT"
@ -37,17 +35,38 @@ os.environ["SNOWFLAKE_ACCOUNT_ID"] = "YOUR ACCOUNT IDENTIFIER"
## Usage
```python
from litellm import completion
from litellm import completion, embedding
## set ENV variables
os.environ["SNOWFLAKE_JWT"] = "YOUR JWT"
os.environ["SNOWFLAKE_JWT"] = "JWT_TOKEN"
os.environ["SNOWFLAKE_ACCOUNT_ID"] = "YOUR ACCOUNT IDENTIFIER"
# Snowflake call
# Snowflake completion call
response = completion(
model="snowflake/mistral-7b",
messages = [{ "content": "Hello, how are you?","role": "user"}]
)
# Snowflake embedding call
response = embedding(
model="snowflake/mistral-7b",
input = ["My text"]
)
# Pass`api_key` and `account_id` as parameters
response = completion(
model="snowflake/mistral-7b",
messages = [{ "content": "Hello, how are you?","role": "user"}],
account_id="AAAA-BBBB",
api_key="JWT_TOKEN"
)
# using PAT
response = completion(
model="snowflake/mistral-7b",
messages = [{ "content": "Hello, how are you?","role": "user"}],
api_key="pat/PAT_TOKEN"
)
```
## Usage with LiteLLM Proxy

51
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -380,3 +380,54 @@ If you need to inspect the JWT fields received from your SSO provider by LiteLLM
Once redirected, you should see a page called "SSO Debug Information". This page displays the JWT fields received from your SSO provider (as shown in the image above)
## Advanced
### Manage User Roles via Azure App Roles
Centralize role management by defining user permissions in Azure Entra ID. LiteLLM will automatically assign roles based on your Azure configuration when users sign in—no need to manually manage roles in LiteLLM.
#### Step 1: Create App Roles on Azure App Registration
1. Navigate to your App Registration on https://portal.azure.com/
2. Go to **App roles** > **Create app role**
3. Configure the app role using one of the [supported LiteLLM roles](./access_control.md#global-proxy-roles):
- **Display name**: Admin Viewer (or your preferred display name)
- **Value**: `proxy_admin_viewer` (must match one of the LiteLLM role values exactly)
4. Click **Apply** to save the role
5. Repeat for each LiteLLM role you want to use
**Supported LiteLLM role values** (see [full role documentation](./access_control.md#global-proxy-roles)):
- `proxy_admin` - Full admin access
- `proxy_admin_viewer` - Read-only admin access
- `internal_user` - Can create/view/delete own keys
- `internal_user_viewer` - Can view own keys (read-only)
<Image img={require('../../img/app_roles.png')} style={{ width: '900px', height: 'auto' }} />
---
#### Step 2: Assign Users to App Roles
1. Navigate to **Enterprise Applications** on https://portal.azure.com/
2. Select your LiteLLM application
3. Go to **Users and groups** > **Add user/group**
4. Select the user
5. Under **Select a role**, choose the app role you created (e.g., `proxy_admin_viewer`)
6. Click **Assign** to save
<Image img={require('../../img/app_role2.png')} style={{ width: '900px', height: 'auto' }} />
---
#### Step 3: Sign in and verify
1. Sign in to the LiteLLM UI via SSO
2. LiteLLM will automatically extract the app role from the JWT token
3. The user will be assigned the corresponding role (you can verify this in the UI by checking the user profile dropdown)
<Image img={require('../../img/app_role3.png')} style={{ width: '900px', height: 'auto' }} />
**Note:** The role from Entra ID will take precedence over any existing role in the LiteLLM database. This ensures your SSO provider is the authoritative source for user roles.

240
docs/my-website/docs/proxy/ai_hub.md Normal file
View file

@ -0,0 +1,240 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# AI Hub
Share models and agents with your organization. Show developers what's available without needing to rebuild them.
This feature is **available in v1.74.3-stable and above**.
## Overview
Admin can select models/agents to expose on public AI hub → Users go to the public url and see what's available.
<Image img={require('../../img/final_public_model_hub_view.png')} />
## Models
### How to use
#### 1. Go to the Admin UI
Navigate to the Model Hub page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=model-hub-table`)
<Image img={require('../../img/model_hub_admin_view.png')} />
#### 2. Select the models you want to expose
Click on `Select Models to Make Public` and select the models you want to expose.
<Image img={require('../../img/make_public_modal.png')} />
#### 3. Confirm the changes
<Image img={require('../../img/make_public_modal_confirmation.png')} />
#### 4. Success!
Go to the public url (`PROXY_BASE_URL/ui/model_hub_table`) and see available models.
<Image img={require('../../img/final_public_model_hub_view.png')} />
### API Endpoints
- `GET /public/model_hub` – returns the list of public model groups. Requires a valid user API key.
- `GET /public/model_hub/info` – returns metadata (docs title, version, useful links) for the public model hub.
## Agents
:::info
Agents are only available in v1.79.4-stable and above.
:::
Share pre-built agents (A2A spec) across your organization. Users can discover and use agents without rebuilding them.
[**Demo Video**](https://drive.google.com/file/d/1r-_Rtiu04RW5Fwwu3_eshtA1oZtC3_DH/view?usp=sharing)
### 1. Create an agent
Create an agent that follows the [A2A spec](https://a2a.dev/).
<Tabs>
<TabItem value="ui" label="UI">
<Image img={require('../../img/add_agent.png')} />
</TabItem>
<TabItem value="api" label="API">
```bash
curl -X POST 'http://0.0.0.0:4000/v1/agents' \
--header 'Authorization: Bearer <your-master-key>' \
--header 'Content-Type: application/json' \
--data '{
"agent_name": "hello-world-agent",
"agent_card_params": {
"protocolVersion": "1.0",
"name": "Hello World Agent",
"description": "Just a hello world agent",
"url": "http://localhost:9999/",
"version": "1.0.0",
"defaultInputModes": ["text"],
"defaultOutputModes": ["text"],
"capabilities": {
"streaming": true
},
"skills": [
{
"id": "hello_world",
"name": "Returns hello world",
"description": "just returns hello world",
"tags": ["hello world"],
"examples": ["hi", "hello world"]
}
]
}
}'
```
**Expected Response**
```json
{
"agent_id": "123e4567-e89b-12d3-a456-426614174000",
"agent_name": "hello-world-agent",
"agent_card_params": {
"protocolVersion": "1.0",
"name": "Hello World Agent",
"description": "Just a hello world agent",
"url": "http://localhost:9999/",
"version": "1.0.0",
"defaultInputModes": ["text"],
"defaultOutputModes": ["text"],
"capabilities": {
"streaming": true
},
"skills": [
{
"id": "hello_world",
"name": "Returns hello world",
"description": "just returns hello world",
"tags": ["hello world"],
"examples": ["hi", "hello world"]
}
]
},
"created_at": "2025-11-15T10:30:00Z",
"created_by": "user123"
}
```
</TabItem>
</Tabs>
### 2. Make agent public
Make the agent discoverable on the AI Hub.
<Tabs>
<TabItem value="ui" label="UI">
Navigate to the Agents Tab on the AI Hub page
<Image img={require('../../img/ai_hub_with_agents.png')} />
Select the agents you want to make public and click on `Make Public` button.
<Image img={require('../../img/make_agents_public.png')} />
</TabItem>
<TabItem value="api" label="API">
**Option 1: Make single agent public**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/agents/123e4567-e89b-12d3-a456-426614174000/make_public' \
--header 'Authorization: Bearer <your-master-key>' \
--header 'Content-Type: application/json'
```
**Option 2: Make multiple agents public**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/agents/make_public' \
--header 'Authorization: Bearer <your-master-key>' \
--header 'Content-Type: application/json' \
--data '{
"agent_ids": [
"123e4567-e89b-12d3-a456-426614174000",
"123e4567-e89b-12d3-a456-426614174001"
]
}'
```
**Expected Response**
```json
{
"message": "Successfully updated public agent groups",
"public_agent_groups": [
"123e4567-e89b-12d3-a456-426614174000"
],
"updated_by": "user123"
}
```
</TabItem>
</Tabs>
### 3. View public agents
Users can now discover the agent via the public endpoint.
<Tabs>
<TabItem value="ui" label="UI">
<Image img={require('../../img/public_agent_hub.png')} />
</TabItem>
<TabItem value="api" label="API">
```bash
curl -X GET 'http://0.0.0.0:4000/public/agent_hub' \
--header 'Authorization: Bearer <user-api-key>'
```
**Expected Response**
```json
[
{
"protocolVersion": "1.0",
"name": "Hello World Agent",
"description": "Just a hello world agent",
"url": "http://localhost:9999/",
"version": "1.0.0",
"defaultInputModes": ["text"],
"defaultOutputModes": ["text"],
"capabilities": {
"streaming": true
},
"skills": [
{
"id": "hello_world",
"name": "Returns hello world",
"description": "just returns hello world",
"tags": ["hello world"],
"examples": ["hi", "hello world"]
}
]
}
]
```
</TabItem>
</Tabs>

22
docs/my-website/docs/proxy/cli_sso.md
View file

@ -9,6 +9,26 @@ Use the litellm cli to authenticate to the LiteLLM Gateway. This is great if you
## Usage
### Prerequisites - Start LiteLLM Proxy with Beta Flag
:::warning[Beta Feature - Required]
CLI SSO Authentication is currently in beta. You must set this environment variable **when starting up your LiteLLM Proxy**:
```bash
export EXPERIMENTAL_UI_LOGIN="True"
litellm --config config.yaml
```
Or add it to your proxy startup command:
```bash
EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
```
:::
### Steps
1. **Install the CLI**
@ -33,6 +53,8 @@ Use the litellm cli to authenticate to the LiteLLM Gateway. This is great if you
2. **Set up environment variables**
On your local machine, set the proxy URL:
```bash
export LITELLM_PROXY_URL=http://localhost:4000
```

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -655,6 +655,7 @@ router_settings:
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).

4
docs/my-website/docs/proxy/control_plane_and_data_plane.md
View file

@ -163,6 +163,10 @@ DISABLE_LLM_API_ENDPOINTS=true
- `/config/*` - Configuration updates
- All other administrative endpoints
### `LITELLM_UI_API_DOC_BASE_URL`
Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy.
## Usage Patterns

4
docs/my-website/docs/proxy/db_info.md
View file

@ -46,8 +46,8 @@ You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/ma
| Table Name | Description | Row Insert Frequency |
|------------|-------------|---------------------|
| LiteLLM_SpendLogs | Detailed logs of all API requests. Records token usage, spend, and timing information. Tracks which models and keys were used. | **High - every LLM API request - Success or Failure** |
| LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** |
| LiteLLM_SpendLogs | Detailed logs of all API requests. Records token usage, spend, and timing information. Tracks which models and keys were used. | **Medium - this is a batch process that runs on an interval.** |
| LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - Runs on every change to an entity** |
## Disable `LiteLLM_SpendLogs`

60
docs/my-website/docs/proxy/dynamic_logging.md
View file

@ -211,4 +211,64 @@ x-litellm-disable-callbacks: LANGFUSE,datadog,PROMETHEUS
x-litellm-disable-callbacks: langfuse,DATADOG,prometheus
```
---
## Disabling Dynamic Callback Management (Enterprise)
Some organizations have compliance requirements where **all requests must be logged under all circumstances**. For these cases, you can disable dynamic callback management entirely to ensure users cannot disable any logging callbacks.
### Use Case
This is designed for enterprise scenarios where:
- **Compliance requirements** mandate that all API requests must be logged
- **Audit trails** must be complete with no gaps
- **Security policies** require all traffic to be monitored
- **No exceptions** can be made for callback disabling
### How to Disable
Set `allow_dynamic_callback_disabling` to `false` in your config.yaml:
```yaml showLineNumbers title="config.yaml"
litellm_settings:
allow_dynamic_callback_disabling: false
```
### Effect
When disabled:
- The `x-litellm-disable-callbacks` header will be **ignored**
- All configured callbacks will **always execute** for every request
- Users cannot bypass logging through headers or request metadata
- All requests are guaranteed to be logged per your proxy configuration
### Example: Compliance Logging Setup
Here's a complete example for an organization requiring guaranteed logging:
```yaml showLineNumbers title="config.yaml"
# config.yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["langfuse", "datadog", "s3"]
# Disable dynamic callback disabling for compliance
allow_dynamic_callback_disabling: false
```
With this configuration:
- All requests will be logged to Langfuse, Datadog, and S3
- Users cannot disable any of these callbacks via headers
- Complete audit trail is guaranteed for compliance requirements
:::info
**Default Behavior**: Dynamic callback disabling is **enabled by default** (`allow_dynamic_callback_disabling: true`). You must explicitly set it to `false` to enforce guaranteed logging.
:::

12
docs/my-website/docs/proxy/enterprise.md
View file

@ -32,13 +32,9 @@ Features:
- ✅ [Set Model budgets for Virtual Keys](./users#-virtual-key-model-specific)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](./proxy/bucket#🪣-logging-gcs-s3-buckets)
- ✅ [`/spend/report` API endpoint](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend)
- **Prometheus Metrics**
- ✅ [Prometheus Metrics - Num Requests, failures, LLM Provider Outages](prometheus)
- ✅ [`x-ratelimit-remaining-requests`, `x-ratelimit-remaining-tokens` for LLM APIs on Prometheus](prometheus#✨-enterprise-llm-remaining-requests-and-remaining-tokens)
- **Control Guardrails per API Key**
- **Control Guardrails per API Key/Team**
- **Custom Branding**
- ✅ [Custom Branding + Routes on Swagger Docs](#swagger-docs---custom-routes--branding)
- ✅ [Public Model Hub](#public-model-hub)
- ✅ [Custom Email Branding](./email.md#customizing-email-branding)
@ -905,9 +901,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
'
```
## Public Model Hub
## Public AI Hub
Share a public page of available models for users
Share a public page of available models and agents for users
[Learn more](./ai_hub.md)
<Image img={require('../../img/model_hub.png')} style={{ width: '900px', height: 'auto' }}/>

492
docs/my-website/docs/proxy/guardrails/custom_guardrail.md
View file

@ -4,151 +4,86 @@ import TabItem from '@theme/TabItem';
# Custom Guardrail
Use this is you want to write code to run a custom guardrail
Use this if you want to write code to run a custom guardrail
## Quick Start
### 1. Write a `CustomGuardrail` Class
A CustomGuardrail has 4 methods to enforce guardrails
- `async_pre_call_hook` - (Optional) modify input or reject request before making LLM API call
- `async_moderation_hook` - (Optional) reject request, runs while making LLM API call (help to lower latency)
- `async_post_call_success_hook`- (Optional) apply guardrail on input/output, runs after making LLM API call
- `async_post_call_streaming_iterator_hook` - (Optional) pass the entire stream to the guardrail
**[See detailed spec of methods here](#customguardrail-methods)**
The simplest way to create a custom guardrail is by implementing the `apply_guardrail` method. This method is called to check text content and can block requests by raising an exception.
**Example `CustomGuardrail` Class**
Create a new file called `custom_guardrail.py` and add this code to it
Create a new file called `custom_guardrail.py` and add this code to it:
```python
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
import os
from typing import Optional, List
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import ModelResponseStream
from litellm.types.guardrails import PiiEntityType
from litellm._logging import verbose_proxy_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
class myCustomGuardrail(CustomGuardrail):
def __init__(
self,
**kwargs,
):
# store kwargs as optional_params
self.optional_params = kwargs
def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs):
self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY")
self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com")
super().__init__(**kwargs)
async def async_pre_call_hook(
async def apply_guardrail(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: Literal[
"completion",
"text_completion",
"embeddings",
"image_generation",
"moderation",
"audio_transcription",
"pass_through_endpoint",
"rerank"
],
) -> Optional[Union[Exception, str, dict]]:
text: str, # IMPORTANT: This is the text to check against your guardrail rules. It's extracted from the request or response across all LLM call types.
language: Optional[str] = None, # ignore
entities: Optional[List[PiiEntityType]] = None, # ignore
request_data: Optional[dict] = None, # ignore
) -> str:
"""
Runs before the LLM API call
Runs on only Input
Use this if you want to MODIFY the input
Check text content against your guardrail rules.
Raise an exception to block the request.
Return the text (optionally modified) to allow it through.
"""
result = await self._check_with_api(text, request_data)
if result.get("action") == "BLOCK":
raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}")
return text
# In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
_content = _content.replace("litellm", "********")
message["content"] = _content
verbose_proxy_logger.debug(
"async_pre_call_hook: Message after masking %s", _messages
async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict:
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
}
response = await async_client.post(
f"{self.api_base}/check",
headers=headers,
json={"text": text},
timeout=5,
)
return data
async def async_moderation_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"],
):
"""
Runs in parallel to LLM API call
Runs on only Input
This can NOT modify the input, only used to reject or accept a call before going to LLM API
"""
# this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call
# In this guardrail, if a user inputs `litellm` we will mask it.
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
raise ValueError("Guardrail failed words - `litellm` detected")
async def async_post_call_success_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response,
):
"""
Runs on response from LLM API call
It can be used to reject a response
If a response contains the word "coffee" -> we will raise an exception
"""
verbose_proxy_logger.debug("async_pre_call_hook response: %s", response)
if isinstance(response, litellm.ModelResponse):
for choice in response.choices:
if isinstance(choice, litellm.Choices):
verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice)
if (
choice.message.content
and isinstance(choice.message.content, str)
and "coffee" in choice.message.content
):
raise ValueError("Guardrail failed Coffee Detected")
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_data: dict,
) -> AsyncGenerator[ModelResponseStream, None]:
"""
Passes the entire stream to the guardrail
This is useful for guardrails that need to see the entire response, such as PII masking.
See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168
Triggered by mode: 'post_call'
"""
async for item in response:
yield item
response.raise_for_status()
return response.json()
```
:::tip Advanced: Using Individual Event Hooks
If you need more fine-grained control, you can implement individual event hooks instead of (or in addition to) `apply_guardrail`:
- `async_pre_call_hook` - Modify input or reject request before making LLM API call
- `async_moderation_hook` - Reject request, runs in parallel with LLM API call (helps lower latency)
- `async_post_call_success_hook` - Apply guardrail on input/output, runs after making LLM API call
- `async_post_call_streaming_iterator_hook` - Pass the entire stream to the guardrail
**[See examples of individual event hooks here](#advanced-individual-event-hooks)** | **[See detailed spec of methods here](#customguardrail-methods)**
:::
### 2. Pass your custom guardrail class in LiteLLM `config.yaml`
In the config below, we point the guardrail to our custom guardrail by setting `guardrail: custom_guardrail.myCustomGuardrail`
@ -166,9 +101,32 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "custom-pre-guard"
- guardrail_name: "my-custom-guardrail"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change
mode: "during_call" # runs apply_guardrail method
api_key: os.environ/MY_GUARDRAIL_API_KEY
api_base: https://api.myguardrail.com
```
:::info Mode Options
- `during_call` - Default mode, runs `apply_guardrail` method (or `async_moderation_hook` if using individual hooks)
- `pre_call` - Runs `async_pre_call_hook` for input modification
- `post_call` - Runs `async_post_call_success_hook` for output validation
:::
<details>
<summary>Advanced: Multiple modes with individual event hooks</summary>
If you're using individual event hooks, you can configure multiple guardrails with different modes:
```yaml
guardrails:
- guardrail_name: "custom-pre-guard"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail
mode: "pre_call" # runs async_pre_call_hook
- guardrail_name: "custom-during-guard"
litellm_params:
@ -180,6 +138,8 @@ guardrails:
mode: "post_call" # runs async_post_call_success_hook
```
</details>
### 3. Start LiteLLM Gateway
<Tabs>
@ -218,15 +178,76 @@ litellm --config config.yaml --detailed_debug
### 4. Test it
#### Test `"custom-pre-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Blocked Request" value = "blocked">
This request will be blocked if it violates your guardrail policy:
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"messages": [
{
"role": "user",
"content": "Content that violates policy"
}
],
"guardrails": ["my-custom-guardrail"]
}'
```
Expected response when blocked:
```json
{
"error": {
"message": "Content blocked: Policy violation",
"type": "None",
"param": "None",
"code": "500"
}
}
```
</TabItem>
<TabItem label="Successful Call" value = "allowed">
This request passes the guardrail:
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4",
"messages": [
{"role": "user", "content": "What is the weather like today?"}
],
"guardrails": ["my-custom-guardrail"]
}'
```
</TabItem>
</Tabs>
<details>
<summary>Advanced: Testing individual event hooks</summary>
If you're using individual event hooks, you can test each mode separately:
#### Test `"custom-pre-guard"`
<Tabs>
<TabItem label="Modify input" value = "not-allowed">
Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#1-write-a-customguardrail-class)
Expect this to mask the word `litellm` before sending the request to the LLM API. [This runs the `async_pre_call_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -244,37 +265,6 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after pre-guard
```json
{
"id": "chatcmpl-9zREDkBIG20RJB4pMlyutmi1hXQWc",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "It looks like you've chosen a string of asterisks. This could be a way to censor or hide certain text. However, without more context, I can't provide a specific word or phrase. If there's something specific you'd like me to say or if you need help with a topic, feel free to let me know!",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"created": 1724429701,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion",
"system_fingerprint": "fp_3aa7262c27",
"usage": {
"completion_tokens": 65,
"prompt_tokens": 14,
"total_tokens": 79
},
"service_tier": null
}
```
</TabItem>
<TabItem label="Successful Call " value = "allowed">
@ -282,7 +272,7 @@ Expected response after pre-guard
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
@ -294,20 +284,14 @@ curl -i http://localhost:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Test `"custom-during-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Unsuccessful call" value = "not-allowed">
Expect this to fail since since `litellm` is in the message content. [This runs the `async_moderation_hook`](#1-write-a-customguardrail-class)
Expect this to fail since `litellm` is in the message content. [This runs the `async_moderation_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -325,7 +309,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after running during-guard
Expected response:
```json
{
@ -345,7 +329,7 @@ Expected response after running during-guard
```shell
curl -i http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
@ -357,21 +341,14 @@ curl -i http://localhost:4000/v1/chat/completions \
</TabItem>
</Tabs>
#### Test `"custom-post-guard"`
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
<Tabs>
<TabItem label="Unsuccessful call" value = "not-allowed">
Expect this to fail since since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#1-write-a-customguardrail-class)
Expect this to fail since `coffee` will be in the response content. [This runs the `async_post_call_success_hook`](#advanced-individual-event-hooks)
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
@ -389,7 +366,7 @@ curl -i -X POST http://localhost:4000/v1/chat/completions \
}'
```
Expected response after running during-guard
Expected response:
```json
{
@ -407,7 +384,7 @@ Expected response after running during-guard
<TabItem label="Successful Call " value = "allowed">
```shell
curl -i -X POST http://localhost:4000/v1/chat/completions \
curl -i -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
@ -424,9 +401,10 @@ Expected response after running during-guard
</TabItem>
</Tabs>
</details>
## ✨ Pass additional parameters to guardrail
:::info
@ -539,10 +517,162 @@ The `get_guardrail_dynamic_request_body_params` method will return:
}
```
## Advanced: Individual Event Hooks
Pro: More flexibility
Con: You need to implement this for each LLM call type (chat completions, text completions, embeddings, image generation, moderation, audio transcription, pass through endpoint, rerank, etc. )
For more fine-grained control over when and how your guardrail runs, you can implement individual event hooks. This gives you flexibility to:
- Modify inputs before the LLM call
- Run checks in parallel with the LLM call (lower latency)
- Validate or modify outputs after the LLM call
- Process streaming responses
### Example with Individual Event Hooks
```python
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import ModelResponseStream, CallTypes
class myCustomGuardrail(CustomGuardrail):
def __init__(
self,
**kwargs,
):
# store kwargs as optional_params
self.optional_params = kwargs
super().__init__(**kwargs)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: Optional[CallTypes],
) -> Optional[Union[Exception, str, dict]]:
"""
Runs before the LLM API call
Runs on only Input
Use this if you want to MODIFY the input
"""
# In this guardrail, if a user inputs `litellm` we will mask it and then send it to the LLM
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
_content = _content.replace("litellm", "********")
message["content"] = _content
verbose_proxy_logger.debug(
"async_pre_call_hook: Message after masking %s", _messages
)
return data
async def async_moderation_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
call_type: Literal["completion", "embeddings", "image_generation", "moderation", "audio_transcription"],
):
"""
Runs in parallel to LLM API call
Runs on only Input
This can NOT modify the input, only used to reject or accept a call before going to LLM API
"""
# this works the same as async_pre_call_hook, but just runs in parallel as the LLM API Call
# In this guardrail, if a user inputs `litellm` we will mask it.
_messages = data.get("messages")
if _messages:
for message in _messages:
_content = message.get("content")
if isinstance(_content, str):
if "litellm" in _content.lower():
raise ValueError("Guardrail failed words - `litellm` detected")
async def async_post_call_success_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
response,
):
"""
Runs on response from LLM API call
It can be used to reject a response
If a response contains the word "coffee" -> we will raise an exception
"""
verbose_proxy_logger.debug("async_pre_call_hook response: %s", response)
if isinstance(response, litellm.ModelResponse):
for choice in response.choices:
if isinstance(choice, litellm.Choices):
verbose_proxy_logger.debug("async_pre_call_hook choice: %s", choice)
if (
choice.message.content
and isinstance(choice.message.content, str)
and "coffee" in choice.message.content
):
raise ValueError("Guardrail failed Coffee Detected")
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: Any,
request_data: dict,
) -> AsyncGenerator[ModelResponseStream, None]:
"""
Passes the entire stream to the guardrail
This is useful for guardrails that need to see the entire response, such as PII masking.
See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168
Triggered by mode: 'post_call'
"""
async for item in response:
yield item
```
## **CustomGuardrail methods**
| Component | Description | Optional | Checked Data | Can Modify Input | Can Modify Output | Can Fail Call |
|-----------|-------------|----------|--------------|------------------|-------------------|----------------|
| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ |
| `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ |
| `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ |
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ |
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ |
## Frequently Asked Questions
**Q. Is `apply_guardrail` relevant both in the request and in the response (pre_call, during_call and post_call hooks)?**
**A.** Yes, one function works in both - See implementation [here](https://github.com/BerriAI/litellm/blob/0292b84dc47473ddeff29bd5a86f529bc523034b/litellm/proxy/utils.py#L825)
**Q. What do I get in the inputs of `apply_guardrail`? What does each field represent (what is text, language, entities, request_data)?**
**A.** The main one you should care about is 'text' - this is what you'll want to send to your api for verification - See implementation [here](https://github.com/BerriAI/litellm/blob/0292b84dc47473ddeff29bd5a86f529bc523034b/litellm/llms/anthropic/chat/guardrail_translation/handler.py#L102)
**Q. Is this function agnostic to the LLM provider? Meaning does it pass the same values for OpenAI and Anthropic for example?
**A.** Yes
**Q. How do I know if my guardrail is running?**
**A.** If you implement `apply_guardrail`, you can query the guardrail directly via [the `/apply_guardrail` API](../../apply_guardrail).

1
docs/my-website/docs/proxy/guardrails/ibm_guardrails.md
View file

@ -95,6 +95,7 @@ curl -i http://localhost:4000/v1/chat/completions \
These go under `optional_params`:
- `detector_params` - dict - Parameters to pass to your detector
- `extra_headers` - dict - Additional headers to inject into requests to IBM Guardrails, as a key-value dict.
- `score_threshold` - float - Only count detections above this score (0.0 to 1.0)
- `block_on_detection` - bool - Block the request when violations found. Default: `true`

4
docs/my-website/docs/proxy/litellm_managed_files.md
View file

@ -21,7 +21,7 @@ Available via the `litellm[proxy]` package or any `litellm` docker image.
| Proxy | ✅ | |
| SDK | ❌ | Requires postgres DB for storing file ids. |
| Available across all providers | ✅ | |
| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning` | |
| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning`, `/responses` | |
## Usage
@ -424,4 +424,4 @@ No, as of `v1.71.2` users can only view/edit/delete files they have created.
## See Also
- [Managed Files w/ Finetuning APIs](../../docs/proxy/managed_finetuning)
- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batch)
- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batches)

2
docs/my-website/docs/proxy/logging.md
View file

@ -2439,7 +2439,7 @@ Your logs should be available on DynamoDB
"S": "{'user': 'ishaan-2'}"
},
"response": {
"S": "EmbeddingResponse(model='text-embedding-ada-002-v2', data=[{'embedding': [-0.03503197431564331, -0.020601635798811913, -0.015375726856291294,
"S": "EmbeddingResponse(model='text-embedding-ada-002', data=[{'embedding': [-0.03503197431564331, -0.020601635798811913, -0.015375726856291294,
}
}
```

13
docs/my-website/docs/proxy/managed_batches.md
View file

@ -260,4 +260,15 @@ print(f"status: {status}")
When a `target_model_names` is specified, the file is written to all deployments that match the `target_model_names`.
No additional infrastructure is required.
No additional infrastructure is required.
## Could the batch be created at the eastus-01 deployment but a subsequent get of the batch could be routed to (a different) eastus2-01 deployment ?
**A.** You can loadbalance b/w multiple models for the initial create batch. Once that's created - we return a file id, which encodes the model deployment used, so it's sticky and only sends any get/delete to that deployment.

19
docs/my-website/docs/proxy/management_cli.md
View file

@ -67,7 +67,26 @@ For an indepth guide, see [CLI Authentication](./cli_sso).
:::
### Prerequisites
:::warning[Beta Feature - Required Environment Variable]
CLI SSO Authentication is currently in beta. You must set this environment variable **when starting up your LiteLLM Proxy**:
```bash
export EXPERIMENTAL_UI_LOGIN="True"
litellm --config config.yaml
```
Or add it to your proxy startup command:
```bash
EXPERIMENTAL_UI_LOGIN="True" litellm --config config.yaml
```
:::
### Steps
1. **Set up the proxy URL**

53
docs/my-website/docs/proxy/model_hub.md
View file

@ -1,53 +0,0 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Model Hub
Tell developers what models are available on the proxy.
This feature is **available in v1.74.3-stable and above**.
## Overview
Admin can select models to expose on public model hub -> Users can go to the public url (`/ui/model_hub_table`) and see available models.
<Image img={require('../../img/final_public_model_hub_view.png')} />
## How to use
### 1. Go to the Admin UI
Navigate to the Model Hub page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=model-hub-table`)
<Image img={require('../../img/model_hub_admin_view.png')} />
### 2. Select the models you want to expose
Click on `Make Public` and select the models you want to expose.
<Image img={require('../../img/make_public_modal.png')} />
### 3. Confirm the changes
<Image img={require('../../img/make_public_modal_confirmation.png')} />
### 4. Success!
Go to the public url (`PROXY_BASE_URL/ui/model_hub_table`) and see available models.
<Image img={require('../../img/final_public_model_hub_view.png')} />
## API Endpoints
LiteLLM also exposes REST endpoints:
- `GET /public/model_hub` – returns the list of public model groups. Requires a valid user API key.
- `GET /public/model_hub/info` – returns metadata (docs title, version, useful links) for the public model hub.
- `GET /public/providers` – returns a sorted list of all providers supported by LiteLLM. No authentication required.
Example:
```bash
curl -s PROXY_BASE_URL/public/providers | jq
```

14
docs/my-website/docs/proxy/prometheus.md
View file

@ -4,15 +4,6 @@ import Image from '@theme/IdealImage';
# 📈 Prometheus metrics
:::info
✨ Prometheus metrics is on LiteLLM Enterprise
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
:::
LiteLLM Exposes a `/metrics` endpoint for Prometheus to Poll
@ -23,11 +14,12 @@ If you're using the LiteLLM CLI with `litellm --config proxy_config.yaml` then y
Add this to your proxy config.yaml
```yaml
model_list:
- model_name: gpt-4o
- model_name: gpt-4o
litellm_params:
model: gpt-4o
litellm_settings:
callbacks: ["prometheus"]
callbacks:
- prometheus
```
Start the proxy

4
docs/my-website/docs/proxy/ui.md
View file

@ -59,11 +59,13 @@ Allow others to create/delete their own keys.
The Admin UI provides comprehensive model management capabilities:
- **Add Models**: Add new models through the UI without restarting the proxy
- **Model Hub**: Make models public for developers to discover available models
- **AI Hub**: Make models and agents public for developers to discover what's available
- **Price Data Sync**: Keep model pricing data up to date by syncing from GitHub
For detailed information on model management, see [Model Management](./model_management.md).
For information on sharing models and agents, see [AI Hub](./ai_hub.md).
:::tip Sync Model Pricing Data
[Sync model pricing data from GitHub](./sync_models_github.md) to keep your model cost information current.
:::

6
docs/my-website/docs/proxy/ui_logs.md
View file

@ -76,8 +76,6 @@ Set `SPEND_LOG_CLEANUP_BATCH_SIZE` to control how many logs are deleted per batc
For detailed architecture and how it works, see [Spend Logs Deletion](../proxy/spend_logs_deletion).
## What gets logged?
[Here's a schema](https://github.com/BerriAI/litellm/blob/1cdd4065a645021aea931afb9494e7694b4ec64b/schema.prisma#L285) breakdown of what gets logged.

54
docs/my-website/docs/secret_managers/aws_secret_manager.md
View file

@ -110,3 +110,57 @@ The `primary_secret_name` allows you to read multiple keys from a single AWS Sec
This reduces the number of AWS Secrets you need to manage.
## IAM Role Assumption
Use IAM roles instead of static AWS credentials for better security.
### Basic IAM Role
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMSecretManagerRole"
aws_session_name: "litellm-session"
```
### Cross-Account Access
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::999999999999:role/CrossAccountRole"
aws_external_id: "unique-external-id"
```
### EKS with IRSA
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMServiceAccountRole"
aws_web_identity_token: "os.environ/AWS_WEB_IDENTITY_TOKEN_FILE"
```
### Configuration Parameters
| Parameter | Description |
|-----------|-------------|
| `aws_region_name` | AWS region |
| `aws_role_name` | IAM role ARN to assume |
| `aws_session_name` | Session name (optional) |
| `aws_external_id` | External ID for cross-account |
| `aws_profile_name` | AWS profile from `~/.aws/credentials` |
| `aws_web_identity_token` | OIDC token path for IRSA |
| `aws_sts_endpoint` | Custom STS endpoint for VPC |

6
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -237,11 +237,8 @@ mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
authorization_url: https://github.com/login/oauth/authorize
token_url: https://github.com/login/oauth/access_token
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
scopes: ["public_repo", "user:email"]
```
</TabItem>
@ -255,9 +252,6 @@ atlassian_mcp:
url: "https://mcp.atlassian.com/v1/sse"
transport: "sse"
auth_type: oauth2
authorization_url: https://mcp.atlassian.com/v1/authorize
token_url: https://cf.mcp.atlassian.com/v1/token
registration_url: https://cf.mcp.atlassian.com/v1/register
```
</TabItem>

120
docs/my-website/docs/vector_store_files.md Normal file
View file

@ -0,0 +1,120 @@
# /vector_stores/\{vector_store_id\}/files
Vector store files represent the individual files that live inside a vector store.
| Feature | Supported |
|---------|-----------|
| Logging | ✅ (full request/response logging) |
| Supported Providers | `openai` |
## Supported operations
| Operation | Description | OpenAI Python Client | LiteLLM Proxy |
|-----------|-------------|----------------------|---------------|
| Create vector store file | Attach a file to a vector store with optional chunking overrides | ✅ | ✅ |
| List vector store files | Paginated listing with filters | ✅ | ✅ |
| Retrieve vector store file | Fetch metadata for a single file | ✅ | ✅ |
| Delete vector store file | Remove a file from a store (file object persists) | ✅ | ✅ |
| Retrieve vector store file content | Stream processed chunks | ❌ | ✅ |
| Update vector store file attributes | Patch custom attributes | ❌ | ✅ |
:::note
Vector store support currently works **only with OpenAI vector stores and OpenAI-uploaded file IDs**.
:::
## Create vector store file
<code>POST http://localhost:4000/v1/vector_stores/&#123;vector_store_id&#125;/files</code>
```python
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000", # LiteLLM proxy or OpenAI base
api_key="sk-1234"
)
vector_store_file = client.vector_stores.files.create(
vector_store_id="vs_69172088a18c8191ab3e2621aa87d1ee",
file_id="file-NDbEDJTfqVh7S4Ugi3CGYw",
chunking_strategy={
"type": "static",
"static": {
"max_chunk_size_tokens": 800,
"chunk_overlap_tokens": 400,
},
},
)
print(vector_store_file)
```
## List vector store files
<code>GET http://localhost:4000/v1/vector_stores/&#123;vector_store_id&#125;/files</code>
Parameters:
- `vector_store_id` (path, required)
- `after` / `before` (query, optional) – pagination cursors
- `filter` (query, optional) – `in_progress`, `completed`, `failed`, `cancelled`
- `limit` (query, optional, default `20`, range `1-100`)
- `order` (query, optional, default `desc`)
```python
vector_store_files = client.vector_stores.files.list(
vector_store_id="vs_abc123"
)
print(vector_store_files)
```
## Retrieve vector store file
<code>GET http://localhost:4000/v1/vector_stores/&#123;vector_store_id&#125;/files/&#123;file_id&#125;</code>
```python
vector_store_file = client.vector_stores.files.retrieve(
vector_store_id="vs_abc123",
file_id="file-abc123"
)
print(vector_store_file)
```
## Delete vector store file
<code>DELETE http://localhost:4000/v1/vector_stores/&#123;vector_store_id&#125;/files/&#123;file_id&#125;</code>
```python
deleted_vector_store_file = client.vector_stores.files.delete(
vector_store_id="vs_abc123",
file_id="file-abc123"
)
print(deleted_vector_store_file)
```
## Proxy-only endpoints
When you need raw content chunks or attribute updates, call the LiteLLM Proxy directly.
### Retrieve file content
```bash
curl -X GET "http://localhost:4000/v1/vector_stores/\{vector_store_id\}/files/\{file_id\}/content" \
-H "Authorization: Bearer sk-1234"
```
### Update file attributes
```bash
curl -X POST "http://localhost:4000/v1/vector_stores/\{vector_store_id\}/files/\{file_id\}" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"attributes": {
"category": "support-faq",
"language": "en"
}
}'
```

22
docs/my-website/docusaurus.config.js
View file

@ -101,6 +101,21 @@ const config = {
include: ['**/*.{md,mdx}'],
},
],
[
'@docusaurus/plugin-content-blog',
{
id: 'blog',
path: './blog',
routeBasePath: 'blog',
blogTitle: 'Blog',
blogSidebarTitle: 'All Posts',
blogSidebarCount: 'ALL',
postsPerPage: 10,
showReadingTime: false,
sortPosts: 'descending',
include: ['**/index.{md,mdx}'],
},
],
() => ({
name: 'cripchat',
@ -129,6 +144,7 @@ const config = {
docs: {
sidebarPath: require.resolve('./sidebars.js'),
},
blog: false, // Disable the default blog plugin from preset-classic
theme: {
customCss: require.resolve('./src/css/custom.css'),
},
@ -177,6 +193,7 @@ const config = {
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Release Notes', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{
href: 'https://models.litellm.ai/',
label: '💸 LLM Model Cost Map',
@ -231,6 +248,11 @@ const config = {
],
copyright: `Copyright © ${new Date().getFullYear()} liteLLM`,
},
colorMode: {
defaultMode: 'light',
disableSwitch: false,
respectPrefersColorScheme: true,
},
prism: {
theme: lightCodeTheme,
darkTheme: darkCodeTheme,

BIN
docs/my-website/img/add_agent.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 616 KiB

BIN
docs/my-website/img/agent_hub_clean.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 152 KiB

BIN
docs/my-website/img/ai_hub_with_agents.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 626 KiB

BIN
docs/my-website/img/app_role2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 256 KiB

BIN
docs/my-website/img/app_role3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 174 KiB

BIN
docs/my-website/img/app_roles.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 277 KiB

BIN
docs/my-website/img/enterprise_vs_oss.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 418 KiB

BIN
docs/my-website/img/enterprise_vs_oss_2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 324 KiB

BIN
docs/my-website/img/favicon_converted.ico Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.2 KiB

BIN
docs/my-website/img/make_agents_public.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 647 KiB

BIN
docs/my-website/img/public_agent_hub.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 445 KiB

6535
docs/my-website/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

13
docs/my-website/package.json
View file

@ -18,7 +18,7 @@
"@docusaurus/plugin-google-gtag": "3.8.1",
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "^3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
@ -45,12 +45,19 @@
]
},
"engines": {
"node": ">=16.14"
"node": ">=16.14",
"npm": ">=8.3.0"
},
"resolutions": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3"
},
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0",
"js-yaml": ">=4.1.1"
"gray-matter": "4.0.3"
}
}

18
docs/my-website/release_notes/authors.yml Normal file
View file

@ -0,0 +1,18 @@
krrish:
name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
ishaan:
name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
# Alias for typo in name
ishaan-alt:
name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg

4
docs/my-website/release_notes/v1.79.3-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.79.3-stable - Built-in Guardrails on AI Gateway"
title: "v1.79.3-stable - Built-in Guardrails on AI Gateway"
slug: "v1-79-3"
date: 2025-11-08T10:00:00
authors:
@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.79.3.rc.1
ghcr.io/berriai/litellm:v1.79.3-stable
```
</TabItem>

523
docs/my-website/release_notes/v1.80.0-stable/index.md Normal file
View file

@ -0,0 +1,523 @@
---
title: "[Preview] v1.80.0-stable - Agent Hub Support"
slug: "v1-80-0"
date: 2025-11-15T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.0.rc.2
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.0
```
</TabItem>
</Tabs>
---
## Key Highlights
- **🆕 Agent Hub Support** - Register and make agents public for your organization
- **RunwayML Provider** - Complete video generation, image generation, and text-to-speech support
- **GPT-5.1 Family Support** - Day-0 support for OpenAI's latest GPT-5.1 and GPT-5.1-Codex models
- **Prometheus OSS** - Prometheus metrics now available in open-source version
- **Vector Store Files API** - Complete OpenAI-compatible Vector Store Files API with full CRUD operations
- **Embeddings Performance** - O(1) lookup optimization for router embeddings with shared sessions
---
### Agent Hub
<Image img={require('../../img/agent_hub_clean.png')} />
This release adds support for registering and making agents public for your organization. This is great for **Proxy Admins** who want a central place to make agents built in their organization, discoverable to their users.
Here's the flow:
1. Add agent to litellm.
2. Make it public.
3. Allow anyone to discover it on the public AI Hub page.
[**Get Started with Agent Hub**](../../docs/proxy/ai_hub)
### Performance – `/embeddings` 13× Lower p95 Latency
This update significantly improves `/embeddings` latency by routing it through the same optimized pipeline as `/chat/completions`, benefiting from all previously applied networking optimizations.
### Results
| Metric | Before | After | Improvement |
| --- | --- | --- | --- |
| p95 latency | 5,700 ms | **430 ms** | −92% (~13× faster)** |
| p99 latency | 7,200 ms | **780 ms** | −89% |
| Average latency | 844 ms | **262 ms** | −69% |
| Median latency | 290 ms | **230 ms** | −21% |
| RPS | 1,216.7 | **1,219.7** | **+0.25%** |
### Test Setup
| Category | Specification |
| --- | --- |
| **Load Testing** | Locust: 1,000 concurrent users, 500 ramp-up |
| **System** | 4 vCPUs, 8 GB RAM, 4 workers, 4 instances |
| **Database** | PostgreSQL (Redis unused) |
| **Configuration** | [config.yaml](https://gist.github.com/AlexsanderHamir/550791675fd752befcac6a9e44024652) |
| **Load Script** | [no_cache_hits.py](https://gist.github.com/AlexsanderHamir/99d673bf74cdd81fd39f59fa9048f2e8) |
---
### 🆕 RunwayML
Complete integration for RunwayML's Gen-4 family of models, supporting video generation, image generation, and text-to-speech.
**Supported Endpoints:**
- `/v1/videos` - Video generation (Gen-4 Turbo, Gen-4 Aleph, Gen-3A Turbo)
- `/v1/images/generations` - Image generation (Gen-4 Image, Gen-4 Image Turbo)
- `/v1/audio/speech` - Text-to-speech (ElevenLabs Multilingual v2)
**Quick Start:**
```bash showLineNumbers title="Generate Video with RunwayML"
curl --location 'http://localhost:4000/v1/videos' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "runwayml/gen4_turbo",
"prompt": "A high quality demo video of litellm ai gateway",
"input_reference": "https://example.com/image.jpg",
"seconds": 5,
"size": "1280x720"
}'
```
[Get Started with RunwayML](../../docs/providers/runwayml/videos)
---
### Prometheus Metrics - Open Source
Prometheus metrics are now available in the open-source version of LiteLLM, providing comprehensive observability for your AI Gateway without requiring an enterprise license.
**Quick Start:**
```yaml
litellm_settings:
success_callback: ["prometheus"]
failure_callback: ["prometheus"]
```
[Get Started with Prometheus](../../docs/proxy/logging#prometheus)
---
### Vector Store Files API
Complete OpenAI-compatible Vector Store Files API now stable, enabling full file lifecycle management within vector stores.
**Supported Endpoints:**
- `POST /v1/vector_stores/{vector_store_id}/files` - Create vector store file
- `GET /v1/vector_stores/{vector_store_id}/files` - List vector store files
- `GET /v1/vector_stores/{vector_store_id}/files/{file_id}` - Retrieve vector store file
- `GET /v1/vector_stores/{vector_store_id}/files/{file_id}/content` - Retrieve file content
- `DELETE /v1/vector_stores/{vector_store_id}/files/{file_id}` - Delete vector store file
- `DELETE /v1/vector_stores/{vector_store_id}` - Delete vector store
**Quick Start:**
```bash showLineNumbers title="Create Vector Store File"
curl --location 'http://localhost:4000/v1/vector_stores/vs_123/files' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"file_id": "file_abc"
}'
```
[Get Started with Vector Stores](../../docs/vector_store_files)
---
## New Providers and Endpoints
### New Providers
| Provider | Supported Endpoints | Description |
| -------- | ------------------- | ----------- |
| **[RunwayML](../../docs/providers/runwayml/videos)** | `/v1/videos`, `/v1/images/generations`, `/v1/audio/speech` | Gen-4 video generation, image generation, and text-to-speech |
### New LLM API Endpoints
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
| `/v1/vector_stores/{vector_store_id}/files` | POST | Create vector store file | [Docs](../../docs/vector_store_files) |
| `/v1/vector_stores/{vector_store_id}/files` | GET | List vector store files | [Docs](../../docs/vector_store_files) |
| `/v1/vector_stores/{vector_store_id}/files/{file_id}` | GET | Retrieve vector store file | [Docs](../../docs/vector_store_files) |
| `/v1/vector_stores/{vector_store_id}/files/{file_id}/content` | GET | Retrieve file content | [Docs](../../docs/vector_store_files) |
| `/v1/vector_stores/{vector_store_id}/files/{file_id}` | DELETE | Delete vector store file | [Docs](../../docs/vector_store_files) |
| `/v1/vector_stores/{vector_store_id}` | DELETE | Delete vector store | [Docs](../../docs/vector_store_files) |
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| OpenAI | `gpt-5.1` | 272K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API |
| OpenAI | `gpt-5.1-2025-11-13` | 272K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API |
| OpenAI | `gpt-5.1-chat-latest` | 128K | $1.25 | $10.00 | Reasoning, vision, PDF input |
| OpenAI | `gpt-5.1-codex` | 272K | $1.25 | $10.00 | Responses API, reasoning, vision |
| OpenAI | `gpt-5.1-codex-mini` | 272K | $0.25 | $2.00 | Responses API, reasoning, vision |
| Moonshot | `moonshot/kimi-k2-thinking` | 262K | $0.60 | $2.50 | Function calling, web search, reasoning |
| Mistral | `mistral/magistral-medium-2509` | 40K | $2.00 | $5.00 | Reasoning, function calling |
| Vertex AI | `vertex_ai/moonshotai/kimi-k2-thinking-maas` | 256K | $0.60 | $2.50 | Function calling, web search |
| OpenRouter | `openrouter/deepseek/deepseek-v3.2-exp` | 164K | $0.20 | $0.40 | Function calling, prompt caching |
| OpenRouter | `openrouter/minimax/minimax-m2` | 205K | $0.26 | $1.02 | Function calling, reasoning |
| OpenRouter | `openrouter/z-ai/glm-4.6` | 203K | $0.40 | $1.75 | Function calling, reasoning |
| OpenRouter | `openrouter/z-ai/glm-4.6:exacto` | 203K | $0.45 | $1.90 | Function calling, reasoning |
| Voyage | `voyage/voyage-3.5` | 32K | $0.06 | - | Embeddings |
| Voyage | `voyage/voyage-3.5-lite` | 32K | $0.02 | - | Embeddings |
#### Video Generation Models
| Provider | Model | Cost Per Second | Resolutions | Features |
| -------- | ----- | --------------- | ----------- | -------- |
| RunwayML | `runwayml/gen4_turbo` | $0.05 | 1280x720, 720x1280 | Text + image to video |
| RunwayML | `runwayml/gen4_aleph` | $0.15 | 1280x720, 720x1280 | Text + image to video |
| RunwayML | `runwayml/gen3a_turbo` | $0.05 | 1280x720, 720x1280 | Text + image to video |
#### Image Generation Models
| Provider | Model | Cost Per Image | Resolutions | Features |
| -------- | ----- | -------------- | ----------- | -------- |
| RunwayML | `runwayml/gen4_image` | $0.05 | 1280x720, 1920x1080 | Text + image to image |
| RunwayML | `runwayml/gen4_image_turbo` | $0.02 | 1280x720, 1920x1080 | Text + image to image |
| Fal.ai | `fal_ai/fal-ai/flux-pro/v1.1` | $0.04/image | - | Image generation |
| Fal.ai | `fal_ai/fal-ai/flux/schnell` | $0.003/image | - | Fast image generation |
| Fal.ai | `fal_ai/fal-ai/bytedance/seedream/v3/text-to-image` | $0.03/image | - | Image generation |
| Fal.ai | `fal_ai/fal-ai/bytedance/dreamina/v3.1/text-to-image` | $0.03/image | - | Image generation |
| Fal.ai | `fal_ai/fal-ai/ideogram/v3` | $0.06/image | - | Image generation |
| Fal.ai | `fal_ai/fal-ai/imagen4/preview/fast` | $0.02/image | - | Fast image generation |
| Fal.ai | `fal_ai/fal-ai/imagen4/preview/ultra` | $0.06/image | - | High-quality image generation |
#### Audio Models
| Provider | Model | Cost | Features |
| -------- | ----- | ---- | -------- |
| RunwayML | `runwayml/eleven_multilingual_v2` | $0.0003/char | Text-to-speech |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Add GPT-5.1 family support with reasoning capabilities - [PR #16598](https://github.com/BerriAI/litellm/pull/16598)
- Add support for `reasoning_effort='none'` for GPT-5.1 - [PR #16658](https://github.com/BerriAI/litellm/pull/16658)
- Add `verbosity` parameter support for GPT-5 family models - [PR #16660](https://github.com/BerriAI/litellm/pull/16660)
- Fix forward OpenAI organization for image generation - [PR #16607](https://github.com/BerriAI/litellm/pull/16607)
- **[Gemini (Google AI Studio + Vertex AI)](../../docs/providers/gemini)**
- Add support for `reasoning_effort='none'` for Gemini models - [PR #16548](https://github.com/BerriAI/litellm/pull/16548)
- Add all Gemini image models support in image generation - [PR #16526](https://github.com/BerriAI/litellm/pull/16526)
- Add Gemini image edit support - [PR #16430](https://github.com/BerriAI/litellm/pull/16430)
- Fix preserve non-ASCII characters in function call arguments - [PR #16550](https://github.com/BerriAI/litellm/pull/16550)
- Fix Gemini conversation format issue with MCP auto-execution - [PR #16592](https://github.com/BerriAI/litellm/pull/16592)
- **[Bedrock](../../docs/providers/bedrock)**
- Add support for filtering knowledge base queries - [PR #16543](https://github.com/BerriAI/litellm/pull/16543)
- Ensure correct `aws_region` is used when provided dynamically for embeddings - [PR #16547](https://github.com/BerriAI/litellm/pull/16547)
- Add support for custom KMS encryption keys in Bedrock Batch operations - [PR #16662](https://github.com/BerriAI/litellm/pull/16662)
- Add bearer token authentication support for AgentCore - [PR #16556](https://github.com/BerriAI/litellm/pull/16556)
- Fix AgentCore SSE stream iterator to async for proper streaming support - [PR #16293](https://github.com/BerriAI/litellm/pull/16293)
- **[Anthropic](../../docs/providers/anthropic)**
- Add context management param support - [PR #16528](https://github.com/BerriAI/litellm/pull/16528)
- Fix preserve `$defs` for Anthropic tools input schema - [PR #16648](https://github.com/BerriAI/litellm/pull/16648)
- Fix support Anthropic tool_use and tool_result in token counter - [PR #16351](https://github.com/BerriAI/litellm/pull/16351)
- **[Vertex AI](../../docs/providers/vertex_ai)**
- Add Vertex Kimi-K2-Thinking support - [PR #16671](https://github.com/BerriAI/litellm/pull/16671)
- Add `vertex_credentials` support to `litellm.rerank()` - [PR #16479](https://github.com/BerriAI/litellm/pull/16479)
- **[Mistral](../../docs/providers/mistral)**
- Fix Magistral streaming to emit reasoning chunks - [PR #16434](https://github.com/BerriAI/litellm/pull/16434)
- **[Moonshot (Kimi)](../../docs/providers/moonshot)**
- Add Kimi K2 thinking model support - [PR #16445](https://github.com/BerriAI/litellm/pull/16445)
- **[SambaNova](../../docs/providers/sambanova)**
- Fix SambaNova API rejecting requests when message content is passed as a list format - [PR #16612](https://github.com/BerriAI/litellm/pull/16612)
- **[VLLM](../../docs/providers/vllm)**
- Fix use vllm passthrough config for hosted vllm provider instead of raising error - [PR #16537](https://github.com/BerriAI/litellm/pull/16537)
- Add headers to VLLM Passthrough requests with success event logging - [PR #16532](https://github.com/BerriAI/litellm/pull/16532)
- **[Azure](../../docs/providers/azure)**
- Fix improve Azure auth parameter handling for None values - [PR #14436](https://github.com/BerriAI/litellm/pull/14436)
- **[Groq](../../docs/providers/groq)**
- Fix parse failed chunks for Groq - [PR #16595](https://github.com/BerriAI/litellm/pull/16595)
- **[Voyage](../../docs/providers/voyage)**
- Add Voyage 3.5 and 3.5-lite embeddings pricing and doc update - [PR #16641](https://github.com/BerriAI/litellm/pull/16641)
- **[Fal.ai](../../docs/image_generation)**
- Add fal-ai/flux/schnell support - [PR #16580](https://github.com/BerriAI/litellm/pull/16580)
- Add all Imagen4 variants of fal ai in model map - [PR #16579](https://github.com/BerriAI/litellm/pull/16579)
### Bug Fixes
- **General**
- Fix sanitize null token usage in OpenAI-compatible responses - [PR #16493](https://github.com/BerriAI/litellm/pull/16493)
- Fix apply provided timeout value to ClientTimeout.total - [PR #16395](https://github.com/BerriAI/litellm/pull/16395)
- Fix raising wrong 429 error on wrong exception - [PR #16482](https://github.com/BerriAI/litellm/pull/16482)
- Add new models, delete repeat models, update pricing - [PR #16491](https://github.com/BerriAI/litellm/pull/16491)
- Update model logging format for custom LLM provider - [PR #16485](https://github.com/BerriAI/litellm/pull/16485)
---
## LLM API Endpoints
#### New Endpoints
- **[GET /providers](../../docs/proxy/management_endpoints)**
- Add GET list of providers endpoint - [PR #16432](https://github.com/BerriAI/litellm/pull/16432)
#### Features
- **[Video Generation API](../../docs/video_generation)**
- Allow internal users to access video generation routes - [PR #16472](https://github.com/BerriAI/litellm/pull/16472)
- **[Vector Stores API](../../docs/vector_stores)**
- Vector store files stable release with complete CRUD operations - [PR #16643](https://github.com/BerriAI/litellm/pull/16643)
- `POST /v1/vector_stores/{vector_store_id}/files` - Create vector store file
- `GET /v1/vector_stores/{vector_store_id}/files` - List vector store files
- `GET /v1/vector_stores/{vector_store_id}/files/{file_id}` - Retrieve vector store file
- `GET /v1/vector_stores/{vector_store_id}/files/{file_id}/content` - Retrieve file content
- `DELETE /v1/vector_stores/{vector_store_id}/files/{file_id}` - Delete vector store file
- `DELETE /v1/vector_stores/{vector_store_id}` - Delete vector store
- Ensure users can access `search_results` for both stream + non-stream response - [PR #16459](https://github.com/BerriAI/litellm/pull/16459)
#### Bugs
- **[Video Generation API](../../docs/video_generation)**
- Fix use GET for `/v1/videos/{video_id}/content` - [PR #16672](https://github.com/BerriAI/litellm/pull/16672)
- **General**
- Fix remove generic exception handling - [PR #16599](https://github.com/BerriAI/litellm/pull/16599)
---
## Management Endpoints / UI
#### Features
- **Proxy CLI Auth**
- Fix remove strict master_key check in add_deployment - [PR #16453](https://github.com/BerriAI/litellm/pull/16453)
- **Virtual Keys**
- UI - Add Tags To Edit Key Flow - [PR #16500](https://github.com/BerriAI/litellm/pull/16500)
- UI - Test Key Page show models based on selected endpoint - [PR #16452](https://github.com/BerriAI/litellm/pull/16452)
- UI - Expose user_alias in view and update path - [PR #16669](https://github.com/BerriAI/litellm/pull/16669)
- **Models + Endpoints**
- UI - Add LiteLLM Params to Edit Model - [PR #16496](https://github.com/BerriAI/litellm/pull/16496)
- UI - Add Model use backend data - [PR #16664](https://github.com/BerriAI/litellm/pull/16664)
- UI - Remove Description Field from LLM Credentials - [PR #16608](https://github.com/BerriAI/litellm/pull/16608)
- UI - Add RunwayML on Admin UI supported models/providers - [PR #16606](https://github.com/BerriAI/litellm/pull/16606)
- Infra - Migrate Add Model Fields to Backend - [PR #16620](https://github.com/BerriAI/litellm/pull/16620)
- Add API Endpoint for creating model access group - [PR #16663](https://github.com/BerriAI/litellm/pull/16663)
- **Teams**
- UI - Invite User Searchable Team Select - [PR #16454](https://github.com/BerriAI/litellm/pull/16454)
- Fix use user budget instead of key budget when creating new team - [PR #16074](https://github.com/BerriAI/litellm/pull/16074)
- **Budgets**
- UI - Move Budgets out of Experimental - [PR #16544](https://github.com/BerriAI/litellm/pull/16544)
- **Guardrails**
- UI - Config Guardrails should not be deletable from table - [PR #16540](https://github.com/BerriAI/litellm/pull/16540)
- Fix remove enterprise restriction from guardrails list endpoint - [PR #15333](https://github.com/BerriAI/litellm/pull/15333)
- **Callbacks**
- UI - New Callbacks table - [PR #16512](https://github.com/BerriAI/litellm/pull/16512)
- Fix delete callbacks failing - [PR #16473](https://github.com/BerriAI/litellm/pull/16473)
- **Usage & Analytics**
- UI - Improve Usage Indicator - [PR #16504](https://github.com/BerriAI/litellm/pull/16504)
- UI - Model Info Page Health Check - [PR #16416](https://github.com/BerriAI/litellm/pull/16416)
- Infra - Show Deprecation Warning for Model Analytics Tab - [PR #16417](https://github.com/BerriAI/litellm/pull/16417)
- Fix Litellm tags usage add request_id - [PR #16111](https://github.com/BerriAI/litellm/pull/16111)
- **Health Check**
- Add Langfuse OTEL and SQS to Health Check - [PR #16514](https://github.com/BerriAI/litellm/pull/16514)
- **General UI**
- UI - Normalize table action columns appearance - [PR #16657](https://github.com/BerriAI/litellm/pull/16657)
- UI - Button Styles and Sizing in Settings Pages - [PR #16600](https://github.com/BerriAI/litellm/pull/16600)
- UI - SSO Modal Cosmetic Changes - [PR #16554](https://github.com/BerriAI/litellm/pull/16554)
- Fix UI logos loading with SERVER_ROOT_PATH - [PR #16618](https://github.com/BerriAI/litellm/pull/16618)
- Fix remove misleading 'Custom' option mention from OpenAI endpoint tooltips - [PR #16622](https://github.com/BerriAI/litellm/pull/16622)
#### Bugs
- **Management Endpoints**
- Fix inconsistent error responses in customer management endpoints - [PR #16450](https://github.com/BerriAI/litellm/pull/16450)
- Fix correct date range filtering in /spend/logs endpoint - [PR #16443](https://github.com/BerriAI/litellm/pull/16443)
- Fix /spend/logs/ui Access Control - [PR #16446](https://github.com/BerriAI/litellm/pull/16446)
- Add pagination for /spend/logs/session/ui endpoint - [PR #16603](https://github.com/BerriAI/litellm/pull/16603)
- Fix LiteLLM Usage shows key_hash - [PR #16471](https://github.com/BerriAI/litellm/pull/16471)
- Fix app_roles missing from jwt payload - [PR #16448](https://github.com/BerriAI/litellm/pull/16448)
---
## Logging / Guardrail / Prompt Management Integrations
#### New Integration
- **🆕 [Zscaler AI Guard](../../docs/proxy/guardrails/zscaler_ai_guard)**
- Add Zscaler AI Guard hook for security policy enforcement - [PR #15691](https://github.com/BerriAI/litellm/pull/15691)
#### Logging
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Fix handle null usage values to prevent validation errors - [PR #16396](https://github.com/BerriAI/litellm/pull/16396)
- **[CloudZero](../../docs/proxy/logging)**
- Fix updated spend would not be sent to CloudZero - [PR #16201](https://github.com/BerriAI/litellm/pull/16201)
#### Guardrails
- **[IBM Detector](../../docs/proxy/guardrails)**
- Ensure detector-id is passed as header to IBM detector server - [PR #16649](https://github.com/BerriAI/litellm/pull/16649)
#### Prompt Management
- **[Custom Prompt Management](../../docs/proxy/prompt_management)**
- Add SDK focused examples for custom prompt management - [PR #16441](https://github.com/BerriAI/litellm/pull/16441)
---
## Spend Tracking, Budgets and Rate Limiting
- **End User Budgets**
- Allow pointing max_end_user budget to an id, so the default ID applies to all end users - [PR #16456](https://github.com/BerriAI/litellm/pull/16456)
---
## MCP Gateway
- **Configuration**
- Add dynamic OAuth2 metadata discovery for MCP servers - [PR #16676](https://github.com/BerriAI/litellm/pull/16676)
- Fix allow tool call even when server name prefix is missing - [PR #16425](https://github.com/BerriAI/litellm/pull/16425)
- Fix exclude unauthorized MCP servers from allowed server list - [PR #16551](https://github.com/BerriAI/litellm/pull/16551)
- Fix unable to delete MCP server from permission settings - [PR #16407](https://github.com/BerriAI/litellm/pull/16407)
- Fix avoid crashing when MCP server record lacks credentials - [PR #16601](https://github.com/BerriAI/litellm/pull/16601)
---
## Agents
- **[Agent Registration (A2A Spec)](../../docs/agents)**
- Support agent registration + discovery following Agent-to-Agent specification - [PR #16615](https://github.com/BerriAI/litellm/pull/16615)
---
## Performance / Loadbalancing / Reliability improvements
- **Embeddings Performance**
- Use router's O(1) lookup and shared sessions for embeddings - [PR #16344](https://github.com/BerriAI/litellm/pull/16344)
- **Router Reliability**
- Support default fallbacks for unknown models - [PR #16419](https://github.com/BerriAI/litellm/pull/16419)
- **Callback Management**
- Add atexit handlers to flush callbacks for async completions - [PR #16487](https://github.com/BerriAI/litellm/pull/16487)
---
## General Proxy Improvements
- **Configuration Management**
- Fix update model_cost_map_url to use environment variable - [PR #16429](https://github.com/BerriAI/litellm/pull/16429)
---
## Documentation Updates
- **Provider Documentation**
- Fix streaming example in README - [PR #16461](https://github.com/BerriAI/litellm/pull/16461)
- Update broken Slack invite links to support page - [PR #16546](https://github.com/BerriAI/litellm/pull/16546)
- Fix code block indentation for fallbacks page - [PR #16542](https://github.com/BerriAI/litellm/pull/16542)
- Documentation code example corrections - [PR #16502](https://github.com/BerriAI/litellm/pull/16502)
- Document `reasoning_effort` summary field options - [PR #16549](https://github.com/BerriAI/litellm/pull/16549)
- **API Documentation**
- Add docs on APIs for model access management - [PR #16673](https://github.com/BerriAI/litellm/pull/16673)
- Add docs for showing how to auto reload new pricing data - [PR #16675](https://github.com/BerriAI/litellm/pull/16675)
- LiteLLM Quick start - show how model resolution works - [PR #16602](https://github.com/BerriAI/litellm/pull/16602)
- Add docs for tracking callback failure - [PR #16474](https://github.com/BerriAI/litellm/pull/16474)
- **General Documentation**
- Fix container api link in release page - [PR #16440](https://github.com/BerriAI/litellm/pull/16440)
- Add softgen to projects that are using litellm - [PR #16423](https://github.com/BerriAI/litellm/pull/16423)
---
## New Contributors
* @artplan1 made their first contribution in [PR #16423](https://github.com/BerriAI/litellm/pull/16423)
* @JehandadK made their first contribution in [PR #16472](https://github.com/BerriAI/litellm/pull/16472)
* @vmiscenko made their first contribution in [PR #16453](https://github.com/BerriAI/litellm/pull/16453)
* @mcowger made their first contribution in [PR #16429](https://github.com/BerriAI/litellm/pull/16429)
* @yellowsubmarine372 made their first contribution in [PR #16395](https://github.com/BerriAI/litellm/pull/16395)
* @Hebruwu made their first contribution in [PR #16201](https://github.com/BerriAI/litellm/pull/16201)
* @jwang-gif made their first contribution in [PR #15691](https://github.com/BerriAI/litellm/pull/15691)
* @AnthonyMonaco made their first contribution in [PR #16502](https://github.com/BerriAI/litellm/pull/16502)
* @andrewm4894 made their first contribution in [PR #16487](https://github.com/BerriAI/litellm/pull/16487)
* @f14-bertolotti made their first contribution in [PR #16485](https://github.com/BerriAI/litellm/pull/16485)
* @busla made their first contribution in [PR #16293](https://github.com/BerriAI/litellm/pull/16293)
* @MightyGoldenOctopus made their first contribution in [PR #16537](https://github.com/BerriAI/litellm/pull/16537)
* @ultmaster made their first contribution in [PR #14436](https://github.com/BerriAI/litellm/pull/14436)
* @bchrobot made their first contribution in [PR #16542](https://github.com/BerriAI/litellm/pull/16542)
* @sep-grindr made their first contribution in [PR #16622](https://github.com/BerriAI/litellm/pull/16622)
* @pnookala-godaddy made their first contribution in [PR #16607](https://github.com/BerriAI/litellm/pull/16607)
* @dtunikov made their first contribution in [PR #16592](https://github.com/BerriAI/litellm/pull/16592)
* @lukapecnik made their first contribution in [PR #16648](https://github.com/BerriAI/litellm/pull/16648)
* @jyeros made their first contribution in [PR #16618](https://github.com/BerriAI/litellm/pull/16618)
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.79.3.rc.1...v1.80.0.rc.1)**
---

5
docs/my-website/sidebars.js
View file

@ -146,13 +146,13 @@ const sidebars = {
type: "category",
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/custom_sso",
"proxy/model_hub",
"proxy/ai_hub",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui",
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
@ -368,6 +368,7 @@ const sidebars = {
]
},
"videos",
"vector_store_files",
{
type: "category",
label: "/mcp - Model Context Protocol",

BIN
docs/my-website/static/img/favicon.ico
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 6.2 KiB

BIN
enterprise/dist/litellm_enterprise-0.1.21-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.21.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.22.tar.gz vendored Normal file
View file

Binary file not shown.

11
enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
View file

@ -40,7 +40,7 @@ class EnterpriseCallbackControls:
#########################################################
# premium user check
#########################################################
if not EnterpriseCallbackControls._premium_user_check():
if not EnterpriseCallbackControls._should_allow_dynamic_callback_disabling():
return False
#########################################################
if isinstance(callback, str):
@ -84,8 +84,15 @@ class EnterpriseCallbackControls:
return None
@staticmethod
def _premium_user_check():
def _should_allow_dynamic_callback_disabling():
import litellm
from litellm.proxy.proxy_server import premium_user
# Check if admin has disabled this feature
if litellm.allow_dynamic_callback_disabling is not True:
verbose_logger.debug("Dynamic callback disabling is disabled by admin via litellm.allow_dynamic_callback_disabling")
return False
if premium_user:
return True
verbose_logger.warning(f"Disabling callbacks using request headers is an enterprise feature. {CommonProxyErrors.not_premium_user.value}")

63
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -296,6 +296,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
# Handle managed files in responses API input
input_data = data.get("input")
if input_data:
file_ids = self.get_file_ids_from_responses_input(input_data)
if file_ids:
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.afile_content.value:
retrieve_file_id = cast(Optional[str], data.get("file_id"))
@ -453,6 +463,47 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_ids.append(file_id)
return file_ids
def get_file_ids_from_responses_input(
self, input: Union[str, List[Dict[str, Any]]]
) -> List[str]:
"""
Gets file ids from responses API input.
The input can be:
- A string (no files)
- A list of input items, where each item can have:
- type: "input_file" with file_id
- content: a list that can contain items with type: "input_file" and file_id
"""
file_ids: List[str] = []
if isinstance(input, str):
return file_ids
if not isinstance(input, list):
return file_ids
for item in input:
if not isinstance(item, dict):
continue
# Check for direct input_file type
if item.get("type") == "input_file":
file_id = item.get("file_id")
if file_id:
file_ids.append(file_id)
# Check for input_file in content array
content = item.get("content")
if isinstance(content, list):
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
file_id = content_item.get("file_id")
if file_id:
file_ids.append(file_id)
return file_ids
async def get_model_file_id_mapping(
self, file_ids: List[str], litellm_parent_otel_span: Span
) -> dict:
@ -478,7 +529,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in file_ids:
## CHECK IF FILE ID IS MANAGED BY LITELM
is_base64_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
litellm_managed_file_ids.append(file_id)
@ -489,6 +539,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
unified_file_object = await self.get_unified_file_id(
file_id, litellm_parent_otel_span
)
if unified_file_object:
file_id_mapping[file_id] = unified_file_object.model_mappings
@ -764,18 +815,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Router,
**data: Dict,
) -> OpenAIFileObject:
file_id = convert_b64_uid_to_unified_uid(file_id)
# file_id = convert_b64_uid_to_unified_uid(file_id)
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span
)
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
for model_id, file_id in specific_model_file_id_mapping.items():
await llm_router.afile_delete(model=model_id, file_id=file_id, **data) # type: ignore
for model_id, model_file_id in specific_model_file_id_mapping.items():
await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore
stored_file_object = await self.delete_unified_file_id(
file_id, litellm_parent_otel_span
)
if stored_file_object:
return stored_file_object
else:
@ -796,6 +850,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping
or await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
)
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.20"
version = "0.1.22"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.20"
version = "0.1.22"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.5-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.5.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz vendored Normal file
View file

Binary file not shown.

17
litellm-proxy-extras/litellm_proxy_extras/migrations/20251114182247_agents_table/migration.sql Normal file
View file

@ -0,0 +1,17 @@
-- CreateTable
CREATE TABLE "LiteLLM_AgentsTable" (
"agent_id" TEXT NOT NULL,
"agent_name" TEXT NOT NULL,
"litellm_params" JSONB,
"agent_card_params" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT NOT NULL,
CONSTRAINT "LiteLLM_AgentsTable_pkey" PRIMARY KEY ("agent_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_AgentsTable_agent_name_key" ON "LiteLLM_AgentsTable"("agent_name");

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120021_baseline_diff/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- This is an empty migration.

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20251115120539_baseline_diff/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- This is an empty migration.

12
litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql Normal file
View file

@ -0,0 +1,12 @@
-- DropIndex
DROP INDEX "LiteLLM_PromptTable_prompt_id_key";
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
-- CreateIndex
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version");

21
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -54,6 +54,19 @@ model LiteLLM_ProxyModelTable {
updated_by String
}
// Agents on proxy
model LiteLLM_AgentsTable {
agent_id String @id @default(uuid())
agent_name String @unique
litellm_params Json?
agent_card_params Json
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
updated_by String
}
model LiteLLM_OrganizationTable {
organization_id String @id @default(uuid())
organization_alias String
@ -548,11 +561,15 @@ model LiteLLM_GuardrailsTable {
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String @unique
prompt_id String
version Int @default(1)
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@index([prompt_id])
}
model LiteLLM_HealthCheckTable {
@ -610,4 +627,4 @@ model LiteLLM_CacheConfig {
cache_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.4"
version = "0.4.6"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.4"
version = "0.4.6"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

141
litellm/__init__.py
View file

@ -181,22 +181,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[
bool
] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[
bool
] = False # if you want to use v1 generic api logged payload
gcs_pub_sub_use_v1: Optional[bool] = (
False # if you want to use v1 gcs pubsub logged payload
)
generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_success_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[
Union[str, Callable, CustomLogger]
] = [] # internal variable - async custom callbacks are routed here.
_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
@ -204,18 +204,18 @@ log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
### end of callbacks #############
email: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
token: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
email: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
token: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@ -271,9 +271,9 @@ use_client: bool = False
ssl_verify: Union[str, bool] = True
ssl_security_level: Optional[str] = None
ssl_certificate: Optional[str] = None
ssl_ecdh_curve: Optional[
str
] = None # Set to 'X25519' to disable PQC and improve performance
ssl_ecdh_curve: Optional[str] = (
None # Set to 'X25519' to disable PQC and improve performance
)
disable_streaming_logging: bool = False
disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
@ -319,20 +319,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
cache: Optional[
Cache
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional[Cache] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[
str
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
budget_duration: Optional[str] = (
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
)
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@ -341,7 +345,9 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
add_function_to_prompt: bool = (
False # if function calling not supported by api, append function call details to system prompt
)
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
@ -379,8 +385,12 @@ prometheus_metrics_config: Optional[List] = None
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: bool = (
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
)
public_mcp_servers: Optional[List[str]] = None
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, PriorityReservationDict]]] = None
@ -390,13 +400,17 @@ priority_reservation_settings: "PriorityReservationSettings" = (
######## Networking Settings ########
use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
use_aiohttp_transport: bool = (
True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
)
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = (
False # When False, aiohttp will respect HTTP(S)_PROXY env vars
)
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
module_level_aclient = AsyncHTTPHandler(
timeout=request_timeout, client_alias="module level aclient"
)
@ -410,13 +424,14 @@ fallbacks: Optional[List] = None
context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
num_retries_per_request: Optional[
int
] = None # for the request overall (incl. fallbacks + model retries)
allow_dynamic_callback_disabling: bool = True
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[
Any
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional[KeyManagementSystem] = None
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
@ -426,9 +441,9 @@ output_parse_pii: bool = False
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[
str, float
] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_discount_config: Dict[str, float] = (
{}
) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
custom_prompt_dict: Dict[str, dict] = {}
check_provider_endpoint = False
@ -1328,6 +1343,9 @@ from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig
from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig
from .llms.github_copilot.responses.transformation import (
GithubCopilotResponsesAPIConfig,
)
from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
@ -1342,6 +1360,7 @@ from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig
from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig
from .main import * # type: ignore
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
@ -1386,6 +1405,20 @@ from .search.main import *
from .realtime_api.main import _arealtime
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
acreate as avector_store_file_create,
adelete as avector_store_file_delete,
alist as avector_store_file_list,
aretrieve as avector_store_file_retrieve,
aretrieve_content as avector_store_file_content,
aupdate as avector_store_file_update,
create as vector_store_file_create,
delete as vector_store_file_delete,
list as vector_store_file_list,
retrieve as vector_store_file_retrieve,
retrieve_content as vector_store_file_content,
update as vector_store_file_update,
)
from .scheduler import *
from .cost_calculator import response_cost_calculator, cost_per_token
@ -1409,12 +1442,12 @@ from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[
str
] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[
bool
] = None # disable huggingface tokenizer download. Defaults to openai clk100
_custom_providers: List[str] = (
[]
) # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
None # disable huggingface tokenizer download. Defaults to openai clk100
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###

76
litellm/batches/main.py
View file

@ -17,6 +17,8 @@ from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Metadata as BatchMetadata
import litellm
from litellm._logging import verbose_logger
@ -223,10 +225,12 @@ def create_batch(
api_key=optional_params.api_key,
logging_obj=litellm_logging_obj,
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
model=model,
)
@ -609,10 +613,12 @@ def retrieve_batch(
function_id="batch_retrieve",
),
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
model=model,
)
@ -799,6 +805,7 @@ def list_batches(
async def acancel_batch(
batch_id: str,
model: Optional[str] = None,
custom_llm_provider: Literal["openai", "azure"] = "openai",
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
@ -813,11 +820,13 @@ async def acancel_batch(
try:
loop = asyncio.get_event_loop()
kwargs["acancel_batch"] = True
model = kwargs.pop("model", None)
# Use a partial function to pass your keyword arguments
func = partial(
cancel_batch,
batch_id,
model,
custom_llm_provider,
metadata,
extra_headers,
@ -840,7 +849,8 @@ async def acancel_batch(
def cancel_batch(
batch_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -852,6 +862,17 @@ def cancel_batch(
LiteLLM Equivalent of POST https://api.openai.com/v1/batches/{batch_id}/cancel
"""
try:
try:
if model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
)
except Exception as e:
verbose_logger.exception(
f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {str(e)}"
)
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params = get_litellm_params(
custom_llm_provider=custom_llm_provider,
@ -1005,21 +1026,28 @@ def _handle_async_invoke_status(
created_at=status_response["submitTime"],
in_progress_at=status_response["lastModifiedTime"],
completed_at=status_response.get("endTime"),
failed_at=status_response.get("endTime")
if status_response["status"] == "failed"
else None,
request_counts={
"total": 1,
"completed": 1 if status_response["status"] == "completed" else 0,
"failed": 1 if status_response["status"] == "failed" else 0,
},
metadata={
"output_file_id": status_response["outputDataConfig"][
"s3OutputDataConfig"
]["s3Uri"],
"failure_message": status_response.get("failureMessage"),
"model_arn": status_response["modelArn"],
},
failed_at=(
status_response.get("endTime")
if status_response["status"] == "failed"
else None
),
request_counts=BatchRequestCounts(
total=1,
completed=1 if status_response["status"] == "completed" else 0,
failed=1 if status_response["status"] == "failed" else 0,
),
metadata=dict(
**{
"output_file_id": status_response["outputDataConfig"][
"s3OutputDataConfig"
]["s3Uri"],
"failure_message": status_response.get("failureMessage") or "",
"model_arn": status_response["modelArn"],
}
),
completion_window="24h",
endpoint="/v1/embeddings",
input_file_id="",
)
return result

50
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -26,7 +26,12 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning
from litellm.types.llms.openai import (
ChatCompletionToolParamFunctionChunk,
Reasoning,
ResponsesAPIOptionalRequestParams,
ResponsesAPIStreamEvents,
)
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam
@ -165,13 +170,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
litellm_logging_obj: "LiteLLMLoggingObj",
client: Optional[Any] = None,
) -> dict:
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
(
input_items,
instructions,
) = self.convert_chat_completion_messages_to_responses_api(messages)
optional_params = self._extract_extra_body_params(optional_params)
# Build responses API request using the reverse transformation logic
responses_api_request = ResponsesAPIOptionalRequestParams()
@ -194,9 +199,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key in ("metadata"):
elif key == "metadata":
responses_api_request["metadata"] = value
elif key in ("previous_response_id"):
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self._map_reasoning_effort(value)
@ -538,6 +543,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
def _extract_extra_body_params(self, optional_params: dict):
"""
Extract extra_body from optional_params and separate supported Responses API params
from unsupported ones. Supported params are moved to top-level optional_params,
unsupported params remain in extra_body.
"""
# Extract extra_body and separate supported params from unsupported ones
extra_body = optional_params.pop("extra_body", None) or {}
if not extra_body:
return optional_params
supported_responses_api_params = set(
ResponsesAPIOptionalRequestParams.__annotations__.keys()
)
# Also include params we handle specially
supported_responses_api_params.update({
"previous_response_id",
"reasoning_effort", # We map this to "reasoning"
})
# Extract supported params from extra_body and merge into optional_params
extra_body_copy = extra_body.copy()
for key, value in extra_body_copy.items():
if key in supported_responses_api_params:
# Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
optional_params[key] = extra_body.pop(key)
return optional_params
def _map_reasoning_effort(self, reasoning_effort: Union[str, Dict[str, Any]]) -> Optional[Reasoning]:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
@ -619,6 +653,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
# Handle different event types from responses API
event_type = parsed_chunk.get("type")
if isinstance(event_type, ResponsesAPIStreamEvents):
event_type = event_type.value
verbose_logger.debug(f"Chat provider: Processing event type: {event_type}")
if event_type == "response.created":
@ -638,7 +674,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
index=0,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=parsed_chunk.get("name", None),
name=output_item.get("name", None),
arguments=parsed_chunk.get("arguments", ""),
),
),
@ -684,7 +720,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
index=0,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=parsed_chunk.get("name", None),
name=output_item.get("name", None),
arguments="", # responses API sends everything again, we don't
),
),

80
litellm/constants.py
View file

@ -1,7 +1,9 @@
import os
from typing import List, Literal
DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
DEFAULT_HEALTH_CHECK_PROMPT = str(
os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")
)
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
@ -18,7 +20,9 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
)
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1))
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(
os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)
)
DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
SQS_SEND_MESSAGE_ACTION = "SendMessage"
SQS_API_VERSION = "2012-11-05"
@ -85,8 +89,12 @@ MAX_TOKEN_TRIMMING_ATTEMPTS = int(
os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10)
) # Maximum number of attempts to trim the message
RUNWAYML_DEFAULT_API_VERSION = str(os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06"))
RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 minutes default for image generation
RUNWAYML_DEFAULT_API_VERSION = str(
os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06")
)
RUNWAYML_POLLING_TIMEOUT = int(
os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)
) # 10 minutes default for image generation
########## Networking constants ##############################################################
_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour
@ -110,22 +118,21 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES = (
DEFAULT_SSL_CIPHERS = os.getenv(
"LITELLM_SSL_CIPHERS",
# Priority 1: TLS 1.3 ciphers (fastest, ~50ms handshake)
"TLS_AES_256_GCM_SHA384:" # Fastest observed in testing
"TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit
"TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile
"TLS_AES_256_GCM_SHA384:" # Fastest observed in testing
"TLS_AES_128_GCM_SHA256:" # Slightly faster than 256-bit
"TLS_CHACHA20_POLY1305_SHA256:" # Fast on ARM/mobile
# Priority 2: TLS 1.2 ECDHE+GCM (fast, ~100ms handshake, widely supported)
"ECDHE-RSA-AES256-GCM-SHA384:"
"ECDHE-RSA-AES128-GCM-SHA256:"
"ECDHE-ECDSA-AES256-GCM-SHA384:"
"ECDHE-ECDSA-AES128-GCM-SHA256:"
# Priority 3: Additional modern ciphers (good balance)
"ECDHE-RSA-CHACHA20-POLY1305:"
"ECDHE-ECDSA-CHACHA20-POLY1305:"
"ECDHE-RSA-CHACHA20-POLY1305:" "ECDHE-ECDSA-CHACHA20-POLY1305:"
# Priority 4: Widely compatible fallbacks (slower but universally supported)
"ECDHE-RSA-AES256-SHA384:" # Common fallback
"ECDHE-RSA-AES128-SHA256:" # Very widely supported
"AES256-GCM-SHA384:" # Non-PFS fallback (compatibility)
"AES128-GCM-SHA256", # Last resort (maximum compatibility)
"ECDHE-RSA-AES256-SHA384:" # Common fallback
"ECDHE-RSA-AES128-SHA256:" # Very widely supported
"AES256-GCM-SHA384:" # Non-PFS fallback (compatibility)
"AES128-GCM-SHA256", # Last resort (maximum compatibility)
)
########### v2 Architecture constants for managing writing updates to the database ###########
@ -282,7 +289,9 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
DEFAULT_IMAGE_ENDPOINT_MODEL = "dall-e-2"
DEFAULT_VIDEO_ENDPOINT_MODEL = "sora-2"
DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS = int(os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8))
DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS = int(
os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)
)
### DATAFORSEO CONSTANTS ###
DEFAULT_DATAFORSEO_LOCATION_CODE = int(
@ -371,7 +380,7 @@ LITELLM_CHAT_PROVIDERS = [
"vercel_ai_gateway",
"wandb",
"ovhcloud",
"lemonade"
"lemonade",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@ -475,6 +484,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
"additional_drop_params": None,
"messages": None,
"reasoning_effort": None,
"verbosity": None,
"thinking": None,
"web_search_options": None,
"service_tier": None,
@ -630,7 +640,7 @@ clarifai_models: set = set(
"clarifai/qwen.qwenLM.Qwen3-14B",
"clarifai/qwen.qwenLM.QwQ-32B-AWQ",
"clarifai/anthropic.completion.claude-3_5-haiku",
"clarifai/anthropic.completion.claude-3_7-sonnet",
"clarifai/anthropic.completion.claude-3_7-sonnet",
]
)
@ -796,28 +806,22 @@ WANDB_MODELS: set = set(
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# zai-org models
"zai-org/GLM-4.5",
# Qwen models
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Thinking-2507",
# moonshotai
"moonshotai/Kimi-K2-Instruct",
# meta models
"meta-llama/Llama-3.1-8B-Instruct",
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
# deepseek-ai
"deepseek-ai/DeepSeek-V3.1",
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
# microsoft
"microsoft/Phi-4-mini-instruct",
]
@ -1031,13 +1035,17 @@ LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
# Key Rotation Constants
LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400)) # 24 hours default
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400)
) # 24 hours default
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
@ -1059,14 +1067,28 @@ PROXY_BATCH_POLLING_INTERVAL = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 360
PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(
os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
)
PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10
PROXY_BATCH_WRITE_AT = int(
os.getenv("PROXY_BATCH_WRITE_AT", 10)
) # in seconds, increased from 10
# APScheduler Configuration - MEMORY LEAK FIX
# These settings prevent memory leaks in APScheduler's normalize() and _apply_jitter() functions
APSCHEDULER_COALESCE = os.getenv("APSCHEDULER_COALESCE", "True").lower() in ["true", "1"] # collapse many missed runs into one
APSCHEDULER_MISFIRE_GRACE_TIME = int(os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600)) # ignore runs older than 1 hour (was 120)
APSCHEDULER_MAX_INSTANCES = int(os.getenv("APSCHEDULER_MAX_INSTANCES", 1)) # prevent concurrent job instances
APSCHEDULER_REPLACE_EXISTING = os.getenv("APSCHEDULER_REPLACE_EXISTING", "True").lower() in ["true", "1"] # always replace existing jobs
APSCHEDULER_COALESCE = os.getenv("APSCHEDULER_COALESCE", "True").lower() in [
"true",
"1",
] # collapse many missed runs into one
APSCHEDULER_MISFIRE_GRACE_TIME = int(
os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600)
) # ignore runs older than 1 hour (was 120)
APSCHEDULER_MAX_INSTANCES = int(
os.getenv("APSCHEDULER_MAX_INSTANCES", 1)
) # prevent concurrent job instances
APSCHEDULER_REPLACE_EXISTING = os.getenv(
"APSCHEDULER_REPLACE_EXISTING", "True"
).lower() in [
"true",
"1",
] # always replace existing jobs
DEFAULT_HEALTH_CHECK_INTERVAL = int(
os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)
@ -1096,6 +1118,8 @@ SECRET_MANAGER_REFRESH_INTERVAL = int(
)
LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"default_internal_user_params",
"public_mcp_servers",
"public_agent_groups",
"public_model_groups",
"public_model_groups_links",
]

36
litellm/cost_calculator.py
View file

@ -133,6 +133,25 @@ def _cost_per_token_custom_pricing_helper(
return None
def _transcription_usage_has_token_details(
usage_block: Optional[Usage],
) -> bool:
if usage_block is None:
return False
prompt_tokens_val = getattr(usage_block, "prompt_tokens", 0) or 0
completion_tokens_val = getattr(usage_block, "completion_tokens", 0) or 0
prompt_details = getattr(usage_block, "prompt_tokens_details", None)
if prompt_details is not None:
audio_token_count = getattr(prompt_details, "audio_tokens", 0) or 0
text_token_count = getattr(prompt_details, "text_tokens", 0) or 0
if audio_token_count > 0 or text_token_count > 0:
return True
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def cost_per_token( # noqa: PLR0915
model: str = "",
prompt_tokens: int = 0,
@ -324,19 +343,18 @@ def cost_per_token( # noqa: PLR0915
usage=usage_block, model=model, custom_llm_provider=custom_llm_provider
)
elif call_type == "atranscription" or call_type == "transcription":
if model == "gpt-4o-mini-transcribe":
if _transcription_usage_has_token_details(usage_block):
return openai_cost_per_token(
model=model,
model=model_without_prefix,
usage=usage_block,
service_tier=service_tier,
)
else:
return openai_cost_per_second(
model=model,
custom_llm_provider=custom_llm_provider,
duration=audio_transcription_file_duration,
)
return openai_cost_per_second(
model=model_without_prefix,
custom_llm_provider=custom_llm_provider,
duration=audio_transcription_file_duration,
)
elif call_type == "search" or call_type == "asearch":
# Search providers use per-query pricing
from litellm.search import search_provider_cost_per_query

447
litellm/experimental_mcp_client/client.py
View file

@ -5,17 +5,24 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
import asyncio
import base64
from datetime import timedelta
from typing import Callable, Dict, List, Optional, Union
from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union
import httpx
from mcp import ClientSession, StdioServerParameters
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
from mcp.client.streamable_http import streamablehttp_client
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import (
CallToolRequestParams as MCPCallToolRequestParams,
GetPromptRequestParams,
GetPromptResult,
Prompt,
ResourceTemplate,
)
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import TextContent
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
@ -34,6 +41,9 @@ def to_basic_auth(auth_value: str) -> str:
return base64.b64encode(auth_value.encode("utf-8")).decode()
TSessionResult = TypeVar("TSessionResult")
class MCPClient:
"""
MCP Client supporting:
@ -58,12 +68,6 @@ class MCPClient:
self.auth_type: MCPAuthType = auth_type
self.timeout: float = timeout
self._mcp_auth_value: Optional[Union[str, Dict[str, str]]] = None
self._session: Optional[ClientSession] = None
self._context = None
self._transport_ctx = None
self._transport = None
self._session_ctx = None
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.extra_headers: Optional[Dict[str, str]] = extra_headers
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
@ -71,33 +75,14 @@ class MCPClient:
if auth_value:
self.update_auth_value(auth_value)
async def __aenter__(self):
"""
Enable async context manager support.
Initializes the transport and session.
"""
try:
await self.connect()
return self
except Exception:
await self.disconnect()
raise
async def connect(self):
"""Initialize the transport and session."""
if self._session:
verbose_logger.debug(
f"MCP client already connected to {self.server_url or 'stdio'}"
)
return # Already connected
verbose_logger.info(
f"MCP client connecting to {self.server_url or 'stdio'} via {self.transport_type}"
)
async def run_with_session(
self, operation: Callable[[ClientSession], Awaitable[TSessionResult]]
) -> TSessionResult:
"""Open a session, run the provided coroutine, and clean up."""
transport_ctx = None
try:
if self.transport_type == MCPTransport.stdio:
# For stdio transport, use stdio_client with command-line parameters
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
@ -106,117 +91,43 @@ class MCPClient:
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {}),
)
self._transport_ctx = stdio_client(server_params)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via stdio: {self.stdio_config.get('command', '')}"
)
transport_ctx = stdio_client(server_params)
elif self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
self._transport_ctx = sse_client(
transport_ctx = sse_client(
url=self.server_url,
timeout=self.timeout,
headers=headers,
httpx_client_factory=httpx_client_factory,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via SSE to {self.server_url}"
)
else: # http
else:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug(
"litellm headers for streamablehttp_client: %s", headers
)
self._transport_ctx = streamablehttp_client(
transport_ctx = streamablehttp_client(
url=self.server_url,
timeout=timedelta(seconds=self.timeout),
headers=headers,
httpx_client_factory=httpx_client_factory,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
verbose_logger.info(
f"MCP client successfully connected via HTTP to {self.server_url}"
)
except ValueError as e:
# Re-raise ValueError exceptions (like missing stdio_config)
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
await self.disconnect()
if transport_ctx is None:
raise RuntimeError("Failed to create transport context")
async with transport_ctx as transport:
read_stream, write_stream = transport[0], transport[1]
session_ctx = ClientSession(read_stream, write_stream)
async with session_ctx as session:
await session.initialize()
return await operation(session)
except Exception:
verbose_logger.warning(
"MCP client run_with_session failed for %s", self.server_url or "stdio"
)
raise
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
await self.disconnect()
# Don't raise other exceptions, let the calling code handle it gracefully
# This allows the server manager to continue with other servers
# Instead of raising, we'll let the calling code handle the failure
pass
async def __aexit__(self, exc_type, exc_val, exc_tb):
"""Cleanup when exiting context manager."""
await self.disconnect()
async def disconnect(self):
"""Clean up session and connections."""
verbose_logger.info(
f"MCP client disconnecting from {self.server_url or 'stdio'}"
)
if self._task and not self._task.done():
verbose_logger.debug("MCP client cancelling background task")
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
if self._session:
try:
verbose_logger.debug("MCP client closing session")
await self._session_ctx.__aexit__(None, None, None) # type: ignore
except Exception as e:
verbose_logger.debug(
f"Error closing MCP session: {type(e).__name__}: {str(e)}"
)
pass
self._session = None
self._session_ctx = None
if self._transport_ctx:
try:
verbose_logger.debug("MCP client closing transport")
await self._transport_ctx.__aexit__(None, None, None)
except Exception as e:
verbose_logger.debug(
f"Error closing MCP transport: {type(e).__name__}: {str(e)}"
)
pass
self._transport_ctx = None
self._transport = None
if self._context:
try:
await self._context.__aexit__(None, None, None) # type: ignore
except Exception:
pass
self._context = None
def update_auth_value(self, mcp_auth_value: Union[str, Dict[str, str]]):
"""
@ -294,24 +205,11 @@ class MCPClient:
f"MCP client listing tools from {self.server_url or 'stdio'}"
)
if not self._session:
verbose_logger.debug("MCP client session not found, attempting to connect")
try:
await self.connect()
except Exception as e:
verbose_logger.error(
f"MCP client connection failed during list_tools: {type(e).__name__}: {str(e)}"
)
return []
if self._session is None:
verbose_logger.error(
"MCP client session is not initialized after connection attempt"
)
return []
async def _list_tools_operation(session: ClientSession):
return await session.list_tools()
try:
result = await self._session.list_tools()
result = await self.run_with_session(_list_tools_operation)
tool_count = len(result.tools)
tool_names = [tool.name for tool in result.tools]
verbose_logger.info(
@ -320,11 +218,10 @@ class MCPClient:
return result.tools
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_tools was cancelled")
await self.disconnect()
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
verbose_logger.exception(
f"MCP client list_tools failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
@ -339,7 +236,6 @@ class MCPClient:
"the MCP server may have crashed, disconnected, or timed out"
)
await self.disconnect()
# Return empty list instead of raising to allow graceful degradation
return []
@ -353,55 +249,21 @@ class MCPClient:
f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}"
)
if not self._session:
verbose_logger.warning(
"MCP client session not found, attempting to connect"
)
try:
await self.connect()
except Exception as e:
verbose_logger.error(
f"MCP client connection failed before tool call: {type(e).__name__}: {str(e)}"
)
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")], isError=True
)
if self._session is None:
verbose_logger.error(
"MCP client session is not initialized after connection attempt"
)
return MCPCallToolResult(
content=[
TextContent(
type="text", text="MCP client session is not initialized"
)
],
isError=True,
)
# Check session and transport state before calling tool
verbose_logger.debug(
f"MCP client state before tool call - "
f"session: {'active' if self._session else 'none'}, "
f"transport: {'active' if self._transport else 'none'}, "
f"session_ctx: {'active' if self._session_ctx else 'none'}, "
f"transport_ctx: {'active' if self._transport_ctx else 'none'}"
)
try:
async def _call_tool_operation(session: ClientSession):
verbose_logger.debug("MCP client sending tool call to session")
tool_result = await self._session.call_tool(
return await session.call_tool(
name=call_tool_request_params.name,
arguments=call_tool_request_params.arguments,
)
try:
tool_result = await self.run_with_session(_call_tool_operation)
verbose_logger.info(
f"MCP client tool call '{call_tool_request_params.name}' completed successfully"
)
return tool_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client tool call was cancelled")
await self.disconnect()
raise
except Exception as e:
import traceback
@ -424,11 +286,9 @@ class MCPClient:
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out. "
"Session and transport will be disconnected."
"the MCP server may have crashed, disconnected, or timed out."
)
await self.disconnect()
# Return a default error result instead of raising
return MCPCallToolResult(
content=[
@ -436,3 +296,218 @@ class MCPClient:
], # Empty content for error case
isError=True,
)
async def list_prompts(self) -> List[Prompt]:
"""List available prompts from the server."""
verbose_logger.debug(
f"MCP client listing tools from {self.server_url or 'stdio'}"
)
async def _list_prompts_operation(session: ClientSession):
return await session.list_prompts()
try:
result = await self.run_with_session(_list_prompts_operation)
prompt_count = len(result.prompts)
prompt_names = [prompt.name for prompt in result.prompts]
verbose_logger.info(
f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}"
)
return result.prompts
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_prompts was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_prompts failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_tools - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def get_prompt(
self, get_prompt_request_params: GetPromptRequestParams
) -> GetPromptResult:
"""Fetch a prompt definition from the MCP server."""
verbose_logger.info(
f"MCP client fetching prompt '{get_prompt_request_params.name}' with arguments: {get_prompt_request_params.arguments}"
)
async def _get_prompt_operation(session: ClientSession):
verbose_logger.debug("MCP client sending get_prompt request to session")
return await session.get_prompt(
name=get_prompt_request_params.name,
arguments=get_prompt_request_params.arguments,
)
try:
get_prompt_result = await self.run_with_session(_get_prompt_operation)
verbose_logger.info(
f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully"
)
return get_prompt_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client get_prompt was cancelled")
raise
except Exception as e:
import traceback
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
f"MCP client get_prompt failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Prompt: {get_prompt_request_params.name}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during get_prompt - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise
async def list_resources(self) -> list[Resource]:
"""List available resources from the server."""
verbose_logger.debug(
f"MCP client listing resources from {self.server_url or 'stdio'}"
)
async def _list_resources_operation(session: ClientSession):
return await session.list_resources()
try:
result = await self.run_with_session(_list_resources_operation)
resource_count = len(result.resources)
resource_names = [resource.name for resource in result.resources]
verbose_logger.info(
f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}"
)
return result.resources
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resources was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_resources failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resources - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def list_resource_templates(self) -> list[ResourceTemplate]:
"""List available resource templates from the server."""
verbose_logger.debug(
f"MCP client listing resource templates from {self.server_url or 'stdio'}"
)
async def _list_resource_templates_operation(session: ClientSession):
return await session.list_resource_templates()
try:
result = await self.run_with_session(_list_resource_templates_operation)
resource_template_count = len(result.resourceTemplates)
resource_template_names = [
resourceTemplate.name for resourceTemplate in result.resourceTemplates
]
verbose_logger.info(
f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}"
)
return result.resourceTemplates
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resource_templates was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_resource_templates failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resource_templates - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def read_resource(self, url: AnyUrl) -> ReadResourceResult:
"""Fetch resource contents from the MCP server."""
verbose_logger.info(f"MCP client fetching resource '{url}'")
async def _read_resource_operation(session: ClientSession):
verbose_logger.debug("MCP client sending read_resource request to session")
return await session.read_resource(url)
try:
read_resource_result = await self.run_with_session(_read_resource_operation)
verbose_logger.info(
f"MCP client read_resource '{url}' completed successfully"
)
return read_resource_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client read_resource was cancelled")
raise
except Exception as e:
import traceback
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
f"MCP client read_resource failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Url: {url}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during read_resource - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise

28
litellm/files/main.py
View file

@ -95,7 +95,9 @@ async def acreate_file(
def create_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock"]] = None,
custom_llm_provider: Optional[
Literal["openai", "azure", "vertex_ai", "bedrock"]
] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -155,10 +157,12 @@ def create_file(
api_key=optional_params.api_key,
logging_obj=logging_obj,
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
elif custom_llm_provider == "openai":
@ -441,12 +445,14 @@ async def afile_delete(
"""
try:
loop = asyncio.get_event_loop()
model = kwargs.pop("model", None)
kwargs["is_async"] = True
# Use a partial function to pass your keyword arguments
func = partial(
file_delete,
file_id,
model,
custom_llm_provider,
extra_headers,
extra_body,
@ -470,7 +476,8 @@ async def afile_delete(
@client
def file_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -481,6 +488,13 @@ def file_delete(
LiteLLM Equivalent of DELETE https://api.openai.com/v1/files
"""
try:
try:
if model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model, custom_llm_provider
)
except Exception:
pass
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params_dict = get_litellm_params(**kwargs)
### TIMEOUT LOGIC ###
@ -566,7 +580,7 @@ def file_delete(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format(
custom_llm_provider
),
model="n/a",

3
litellm/integrations/arize/_utils.py
View file

@ -208,7 +208,10 @@ def set_attributes(
)
try:
# Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers)
optional_params = kwargs.get("optional_params", {})
if isinstance(optional_params, dict):
optional_params.pop("secret_fields", None)
litellm_params = kwargs.get("litellm_params", {})
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"

404
litellm/integrations/callback_configs.json Normal file
View file

@ -0,0 +1,404 @@
[
{
"id": "arize",
"displayName": "Arize",
"logo": "arize.png",
"supports_key_team_logging": true,
"dynamic_params": {
"arize_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Arize API key for authentication",
"required": true
},
"arize_space_key": {
"type": "password",
"ui_name": "Space Key",
"description": "Arize Space key to identify your workspace",
"required": true
}
},
"description": "Arize Logging Integration"
},
{
"id": "braintrust",
"displayName": "Braintrust",
"logo": "braintrust.png",
"supports_key_team_logging": false,
"dynamic_params": {
"braintrust_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Braintrust API key for authentication",
"required": true
},
"braintrust_project_name": {
"type": "text",
"ui_name": "Project Name",
"description": "Name of the Braintrust project to log to",
"required": true
}
},
"description": "Braintrust Logging Integration"
},
{
"id": "custom_callback_api",
"displayName": "Custom Callback API",
"logo": "custom.svg",
"supports_key_team_logging": true,
"dynamic_params": {
"custom_callback_api_url": {
"type": "text",
"ui_name": "Callback URL",
"description": "Your custom webhook/API endpoint URL to receive logs",
"required": true
},
"custom_callback_api_headers": {
"type": "text",
"ui_name": "Headers (JSON)",
"description": "Custom HTTP headers as JSON string (e.g., {\"Authorization\": \"Bearer token\"})",
"required": false
}
},
"description": "Custom Callback API Logging Integration"
},
{
"id": "datadog",
"displayName": "Datadog",
"logo": "datadog.png",
"supports_key_team_logging": false,
"dynamic_params": {
"dd_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Datadog API key for authentication",
"required": true
},
"dd_site": {
"type": "text",
"ui_name": "Site",
"description": "Datadog site URL (e.g., us5.datadoghq.com)",
"required": true
}
},
"description": "Datadog Logging Integration"
},
{
"id": "lago",
"displayName": "Lago",
"logo": "lago.svg",
"supports_key_team_logging": false,
"dynamic_params": {
"lago_api_url": {
"type": "text",
"ui_name": "API URL",
"description": "Lago API base URL",
"required": true
},
"lago_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "Lago API key for authentication",
"required": true
}
},
"description": "Lago Billing Logging Integration"
},
{
"id": "langfuse",
"displayName": "Langfuse",
"logo": "langfuse.png",
"supports_key_team_logging": true,
"dynamic_params": {
"langfuse_public_key": {
"type": "text",
"ui_name": "Public Key",
"description": "Langfuse public key",
"required": true
},
"langfuse_secret_key": {
"type": "password",
"ui_name": "Secret Key",
"description": "Langfuse secret key for authentication",
"required": true
},
"langfuse_host": {
"type": "text",
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
}
},
"description": "Langfuse v2 Logging Integration"
},
{
"id": "langfuse_otel",
"displayName": "Langfuse OTEL",
"logo": "langfuse.png",
"supports_key_team_logging": true,
"dynamic_params": {
"langfuse_public_key": {
"type": "text",
"ui_name": "Public Key",
"description": "Langfuse public key",
"required": true
},
"langfuse_secret_key": {
"type": "password",
"ui_name": "Secret Key",
"description": "Langfuse secret key for authentication",
"required": true
},
"langfuse_host": {
"type": "text",
"ui_name": "Host URL",
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
"required": false
}
},
"description": "Langfuse v3 OTEL Logging Integration"
},
{
"id": "langsmith",
"displayName": "LangSmith",
"logo": "langsmith.png",
"supports_key_team_logging": true,
"dynamic_params": {
"langsmith_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "LangSmith API key for authentication",
"required": true
},
"langsmith_project": {
"type": "text",
"ui_name": "Project Name",
"description": "LangSmith project name (default: litellm-completion)",
"required": false
},
"langsmith_base_url": {
"type": "text",
"ui_name": "Base URL",
"description": "LangSmith base URL (default: https://api.smith.langchain.com)",
"required": false
},
"langsmith_sampling_rate": {
"type": "number",
"ui_name": "Sampling Rate",
"description": "Sampling rate for logging (0.0 to 1.0, default: 1.0)",
"required": false
}
},
"description": "Langsmith Logging Integration"
},
{
"id": "openmeter",
"displayName": "OpenMeter",
"logo": "openmeter.png",
"supports_key_team_logging": false,
"dynamic_params": {
"openmeter_api_key": {
"type": "password",
"ui_name": "API Key",
"description": "OpenMeter API key for authentication",
"required": true
},
"openmeter_base_url": {
"type": "text",
"ui_name": "Base URL",
"description": "OpenMeter base URL (default: https://openmeter.cloud)",
"required": false
}
},
"description": "OpenMeter Logging Integration"
},
{
"id": "otel",
"displayName": "Open Telemetry",
"logo": "otel.png",
"supports_key_team_logging": false,
"dynamic_params": {
"otel_endpoint": {
"type": "text",
"ui_name": "Endpoint URL",
"description": "OpenTelemetry collector endpoint URL",
"required": true
},
"otel_headers": {
"type": "text",
"ui_name": "Headers",
"description": "Headers for OTEL exporter (e.g., x-honeycomb-team=YOUR_API_KEY)",
"required": false
}
},
"description": "OpenTelemetry Logging Integration"
},
{
"id": "s3",
"displayName": "S3",
"logo": "aws.svg",
"supports_key_team_logging": false,
"dynamic_params": {
"s3_bucket_name": {
"type": "text",
"ui_name": "Bucket Name",
"description": "AWS S3 bucket name to store logs",
"required": true
},
"s3_region_name": {
"type": "text",
"ui_name": "AWS Region",
"description": "AWS region name (e.g., us-east-1)",
"required": false
},
"s3_aws_access_key_id": {
"type": "password",
"ui_name": "AWS Access Key ID",
"description": "AWS access key ID for authentication",
"required": false
},
"s3_aws_secret_access_key": {
"type": "password",
"ui_name": "AWS Secret Access Key",
"description": "AWS secret access key for authentication",
"required": false
},
"s3_aws_session_token": {
"type": "password",
"ui_name": "AWS Session Token",
"description": "AWS session token for temporary credentials",
"required": false
},
"s3_endpoint_url": {
"type": "text",
"ui_name": "S3 Endpoint URL",
"description": "Custom S3 endpoint URL (for MinIO or custom S3-compatible services)",
"required": false
},
"s3_path": {
"type": "text",
"ui_name": "S3 Path Prefix",
"description": "Path prefix within the bucket for organizing logs",
"required": false
}
},
"description": "S3 Bucket (AWS) Logging Integration"
},
{
"id": "sqs",
"displayName": "SQS",
"logo": "aws.svg",
"supports_key_team_logging": false,
"dynamic_params": {
"sqs_queue_url": {
"type": "text",
"ui_name": "Queue URL",
"description": "AWS SQS Queue URL",
"required": true
},
"sqs_region_name": {
"type": "text",
"ui_name": "AWS Region",
"description": "AWS region name (e.g., us-east-1)",
"required": false
},
"sqs_aws_access_key_id": {
"type": "password",
"ui_name": "AWS Access Key ID",
"description": "AWS access key ID for authentication",
"required": false
},
"sqs_aws_secret_access_key": {
"type": "password",
"ui_name": "AWS Secret Access Key",
"description": "AWS secret access key for authentication",
"required": false
},
"sqs_aws_session_token": {
"type": "password",
"ui_name": "AWS Session Token",
"description": "AWS session token for temporary credentials",
"required": false
},
"sqs_aws_session_name": {
"type": "text",
"ui_name": "AWS Session Name",
"description": "Name for AWS session",
"required": false
},
"sqs_aws_profile_name": {
"type": "text",
"ui_name": "AWS Profile Name",
"description": "AWS profile name from credentials file",
"required": false
},
"sqs_aws_role_name": {
"type": "text",
"ui_name": "AWS Role Name",
"description": "AWS IAM role name to assume",
"required": false
},
"sqs_aws_web_identity_token": {
"type": "password",
"ui_name": "AWS Web Identity Token",
"description": "AWS web identity token for authentication",
"required": false
},
"sqs_aws_sts_endpoint": {
"type": "text",
"ui_name": "AWS STS Endpoint",
"description": "AWS STS endpoint URL",
"required": false
},
"sqs_endpoint_url": {
"type": "text",
"ui_name": "SQS Endpoint URL",
"description": "Custom SQS endpoint URL (for LocalStack or custom endpoints)",
"required": false
},
"sqs_api_version": {
"type": "text",
"ui_name": "API Version",
"description": "SQS API version",
"required": false
},
"sqs_use_ssl": {
"type": "boolean",
"ui_name": "Use SSL",
"description": "Whether to use SSL for SQS connections",
"required": false
},
"sqs_verify": {
"type": "boolean",
"ui_name": "Verify SSL",
"description": "Whether to verify SSL certificates",
"required": false
},
"sqs_strip_base64_files": {
"type": "boolean",
"ui_name": "Strip Base64 Files",
"description": "Remove base64-encoded files from logs to reduce payload size",
"required": false
},
"sqs_aws_use_application_level_encryption": {
"type": "boolean",
"ui_name": "Use Application-Level Encryption",
"description": "Enable application-level encryption for SQS messages",
"required": false
},
"sqs_app_encryption_key_b64": {
"type": "password",
"ui_name": "Encryption Key (Base64)",
"description": "Base64-encoded encryption key for application-level encryption",
"required": false
},
"sqs_app_encryption_aad": {
"type": "text",
"ui_name": "Encryption AAD",
"description": "Additional authenticated data for encryption",
"required": false
}
},
"description": "SQS Queue (AWS) Logging Integration"
}
]

21
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -108,7 +108,7 @@ class DotpromptManager(CustomPromptManagement):
Compile a .prompt file into a PromptManagementClient structure.
This method:
1. Loads the prompt template from the .prompt file
1. Loads the prompt template from the .prompt file (with optional version)
2. Renders it with the provided variables
3. Converts the rendered text into chat messages
4. Extracts model and optional parameters from metadata
@ -116,13 +116,22 @@ class DotpromptManager(CustomPromptManagement):
try:
# Get the prompt template
template = self.prompt_manager.get_prompt(prompt_id)
# Get the prompt template (versioned or base)
template = self.prompt_manager.get_prompt(
prompt_id=prompt_id, version=prompt_version
)
if template is None:
raise ValueError(f"Prompt '{prompt_id}' not found in prompt directory")
version_str = f" (version {prompt_version})" if prompt_version else ""
raise ValueError(
f"Prompt '{prompt_id}'{version_str} not found in prompt directory"
)
# Render the template with variables
rendered_content = self.prompt_manager.render(prompt_id, prompt_variables)
# Render the template with variables (pass version for proper lookup)
rendered_content = self.prompt_manager.render(
prompt_id=prompt_id,
prompt_variables=prompt_variables,
version=prompt_version,
)
# Convert rendered content to chat messages
messages = self._convert_to_messages(rendered_content)

37
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -183,7 +183,10 @@ class PromptManager:
return frontmatter, template_content
def render(
self, prompt_id: str, prompt_variables: Optional[Dict[str, Any]] = None
self,
prompt_id: str,
prompt_variables: Optional[Dict[str, Any]] = None,
version: Optional[int] = None,
) -> str:
"""
Render a prompt template with the given variables.
@ -191,6 +194,7 @@ class PromptManager:
Args:
prompt_id: The ID of the prompt template to render
prompt_variables: Variables to substitute in the template
version: Optional version number. If provided, looks for {prompt_id}.v{version}
Returns:
The rendered prompt string
@ -199,13 +203,16 @@ class PromptManager:
KeyError: If prompt_id is not found
ValueError: If template rendering fails
"""
if prompt_id not in self.prompts:
# Get the template (versioned or base)
template = self.get_prompt(prompt_id=prompt_id, version=version)
if template is None:
available_prompts = list(self.prompts.keys())
version_str = f" (version {version})" if version else ""
raise KeyError(
f"Prompt '{prompt_id}' not found. Available prompts: {available_prompts}"
f"Prompt '{prompt_id}'{version_str} not found. Available prompts: {available_prompts}"
)
template = self.prompts[prompt_id]
variables = prompt_variables or {}
# Validate input variables against schema if defined
@ -254,8 +261,26 @@ class PromptManager:
return type_mapping.get(schema_type.lower(), str) # type: ignore
def get_prompt(self, prompt_id: str) -> Optional[PromptTemplate]:
"""Get a prompt template by ID."""
def get_prompt(
self, prompt_id: str, version: Optional[int] = None
) -> Optional[PromptTemplate]:
"""
Get a prompt template by ID and optional version.
Args:
prompt_id: The base prompt ID
version: Optional version number. If provided, looks for {prompt_id}.v{version}
Returns:
The prompt template if found, None otherwise
"""
if version is not None:
# Try versioned prompt first: prompt_id.v{version}
versioned_id = f"{prompt_id}.v{version}"
if versioned_id in self.prompts:
return self.prompts[versioned_id]
# Fall back to base prompt_id
return self.prompts.get(prompt_id)
def list_prompts(self) -> List[str]:

2
litellm/integrations/langfuse/langfuse.py
View file

@ -228,6 +228,8 @@ class LangFuseLogger:
functions = optional_params.pop("functions", None)
tools = optional_params.pop("tools", None)
# Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers)
optional_params.pop("secret_fields", None)
if functions is not None:
prompt["functions"] = functions
if tools is not None:

24
enterprise/litellm_enterprise/integrations/prometheus.py → litellm/integrations/prometheus.py
View file

@ -41,21 +41,9 @@ class PrometheusLogger(CustomLogger):
try:
from prometheus_client import Counter, Gauge, Histogram
from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
if premium_user is not True:
verbose_logger.warning(
f"🚨🚨🚨 Prometheus Metrics is on LiteLLM Enterprise\n🚨 {CommonProxyErrors.not_premium_user.value}"
)
self.litellm_not_a_premium_user_metric = Counter(
name="litellm_not_a_premium_user_metric",
documentation=f"🚨🚨🚨 Prometheus Metrics is on LiteLLM Enterprise. 🚨 {CommonProxyErrors.not_premium_user.value}",
)
return
# Create metric factory functions
self._counter_factory = self._create_metric_factory(Counter)
self._gauge_factory = self._create_metric_factory(Gauge)
@ -2184,9 +2172,6 @@ class PrometheusLogger(CustomLogger):
It emits the current remaining budget metrics for all Keys and Teams.
"""
from enterprise.litellm_enterprise.integrations.prometheus import (
PrometheusLogger,
)
from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
@ -2213,26 +2198,19 @@ class PrometheusLogger(CustomLogger):
)
@staticmethod
def _mount_metrics_endpoint(premium_user: bool):
def _mount_metrics_endpoint():
"""
Mount the Prometheus metrics endpoint with optional authentication.
Args:
premium_user (bool): Whether the user is a premium user
require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
Defaults to False.
"""
from prometheus_client import make_asgi_app
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.proxy_server import app
if premium_user is not True:
verbose_proxy_logger.warning(
f"Prometheus metrics are only available for premium users. {CommonProxyErrors.not_premium_user.value}"
)
# Create metrics ASGI app
if "PROMETHEUS_MULTIPROC_DIR" in os.environ:
from prometheus_client import CollectorRegistry, multiprocess

12
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -16,14 +16,16 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
from litellm.integrations.argilla import ArgillaLogger
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.gitlab import GitLabPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.datadog.datadog import DataDogLogger
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from litellm.integrations.deepeval import DeepEvalLogger
from litellm.integrations.dotprompt import DotpromptManager
from litellm.integrations.galileo import GalileoObserve
from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger
from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger
from litellm.integrations.gitlab import GitLabPromptManager
from litellm.integrations.humanloop import HumanloopLogger
from litellm.integrations.lago import LagoLogger
from litellm.integrations.langfuse.langfuse_prompt_management import (
@ -36,13 +38,7 @@ from litellm.integrations.openmeter import OpenMeterLogger
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.integrations.opik.opik import OpikLogger
from litellm.integrations.posthog import PostHogLogger
try:
from litellm_enterprise.integrations.prometheus import PrometheusLogger
except Exception:
PrometheusLogger = None
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.dotprompt import DotpromptManager
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.s3_v2 import S3Logger
from litellm.integrations.sqs import SQSLogger
from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (

12
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -693,12 +693,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) # type: ignore
dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")
elif custom_llm_provider == "snowflake":
api_base = (
api_base
or get_secret_str("SNOWFLAKE_API_BASE")
or f"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT")
(
api_base,
dynamic_api_key,
) = litellm.SnowflakeConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "gradient_ai":
(
api_base,

199
litellm/litellm_core_utils/litellm_logging.py
View file

@ -58,6 +58,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
@ -176,7 +177,6 @@ try:
from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import (
SMTPEmailLogger,
)
from litellm_enterprise.integrations.prometheus import PrometheusLogger
from litellm_enterprise.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup,
)
@ -194,7 +194,6 @@ except Exception as e:
PagerDutyAlerting = CustomLogger # type: ignore
EnterpriseCallbackControls = None # type: ignore
EnterpriseStandardLoggingPayloadSetupVAR = None
PrometheusLogger = None
_in_memory_loggers: List[Any] = []
### GLOBAL VARIABLES ###
@ -586,7 +585,10 @@ class Logging(LiteLLMLoggingBaseClass):
custom_logger = (
prompt_management_logger
or self.get_custom_logger_for_prompt_management(
model=model, non_default_params=non_default_params
model=model,
non_default_params=non_default_params,
prompt_id=prompt_id,
dynamic_callback_params=self.standard_callback_dynamic_params,
)
)
@ -623,7 +625,11 @@ class Logging(LiteLLMLoggingBaseClass):
custom_logger = (
prompt_management_logger
or self.get_custom_logger_for_prompt_management(
model=model, tools=tools, non_default_params=non_default_params
model=model,
tools=tools,
non_default_params=non_default_params,
prompt_id=prompt_id,
dynamic_callback_params=self.standard_callback_dynamic_params,
)
)
@ -647,19 +653,69 @@ class Logging(LiteLLMLoggingBaseClass):
self.messages = messages
return model, messages, non_default_params
def _auto_detect_prompt_management_logger(
self,
prompt_id: str,
dynamic_callback_params: StandardCallbackDynamicParams,
) -> Optional[CustomLogger]:
"""
Auto-detect which prompt management system owns the given prompt_id.
This allows a user to just pass prompt_id in the completion call and it will be auto-detected which system owns this prompt.
Args:
prompt_id: The prompt ID to check
dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks
Returns:
A CustomLogger instance if a matching prompt management system is found, None otherwise
"""
prompt_management_loggers = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CustomPromptManagement
)
)
for logger in prompt_management_loggers:
if isinstance(logger, CustomPromptManagement):
try:
if logger.should_run_prompt_management(
prompt_id=prompt_id,
dynamic_callback_params=dynamic_callback_params,
):
self.model_call_details["prompt_integration"] = (
logger.__class__.__name__
)
return logger
except Exception:
# If check fails, continue to next logger
continue
return None
def get_custom_logger_for_prompt_management(
self, model: str, non_default_params: Dict, tools: Optional[List[Dict]] = None
self,
model: str,
non_default_params: Dict,
tools: Optional[List[Dict]] = None,
prompt_id: Optional[str] = None,
dynamic_callback_params: Optional[StandardCallbackDynamicParams] = None,
) -> Optional[CustomLogger]:
"""
Get a custom logger for prompt management based on model name or available callbacks.
Args:
model: The model name to check for prompt management integration
non_default_params: Non-default parameters passed to the completion call
tools: Optional tools passed to the completion call
prompt_id: Optional prompt ID to auto-detect which system owns this prompt
dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks
Returns:
A CustomLogger instance if one is found, None otherwise
"""
# First check if model starts with a known custom logger compatible callback
# This takes precedence for backward compatibility
for callback_name in litellm._known_custom_logger_compatible_callbacks:
if model.startswith(callback_name):
custom_logger = _init_custom_logger_compatible_class(
@ -671,7 +727,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["prompt_integration"] = model.split("/")[0]
return custom_logger
# Then check for any registered CustomPromptManagement loggers
# If prompt_id is provided, try to auto-detect which system has this prompt
if prompt_id and dynamic_callback_params is not None:
auto_detected_logger = self._auto_detect_prompt_management_logger(
prompt_id=prompt_id,
dynamic_callback_params=dynamic_callback_params,
)
if auto_detected_logger is not None:
return auto_detected_logger
# Then check for any registered CustomPromptManagement loggers (fallback)
prompt_management_loggers = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CustomPromptManagement
@ -1475,33 +1540,58 @@ class Logging(LiteLLMLoggingBaseClass):
if self.model_call_details["litellm_params"]["metadata"] is None:
self.model_call_details["litellm_params"]["metadata"] = {}
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr(logging_result, "_hidden_params", {}) # type: ignore
if "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
else:
self.model_call_details["response_cost"] = self._response_cost_calculator(result=logging_result)
self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["response_cost"] = self._response_cost_calculator(
result=logging_result
)
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
def _transform_usage_objects(self, result):
if isinstance(result, ResponsesAPIResponse):
result = result.model_copy()
transformed_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(result.usage)
setattr(result, "usage", transformed_usage.model_dump() if hasattr(transformed_usage, "model_dump") else dict(transformed_usage))
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
standard_logging_payload["response"] = result.model_dump() if hasattr(result, "model_dump") else dict(result)
transformed_usage = (
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
result.usage
)
)
setattr(
result,
"usage",
(
transformed_usage.model_dump()
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
),
)
if (
standard_logging_payload := self.model_call_details.get(
"standard_logging_object"
)
) is not None:
standard_logging_payload["response"] = (
result.model_dump()
if hasattr(result, "model_dump")
else dict(result)
)
elif isinstance(result, TranscriptionResponse):
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
)
result = result.model_copy()
transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore
setattr(result, "usage", transformed_usage)
@ -1522,40 +1612,67 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details["completion_start_time"] = self.completion_start_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
self.model_call_details["cache_hit"] = cache_hit
if self.call_type == CallTypes.anthropic_messages.value:
result = self._handle_anthropic_messages_response_logging(result=result)
elif self.call_type == CallTypes.generate_content.value or self.call_type == CallTypes.agenerate_content.value:
result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
elif (
self.call_type == CallTypes.generate_content.value
or self.call_type == CallTypes.agenerate_content.value
):
result = self._handle_non_streaming_google_genai_generate_content_response_logging(
result=result
)
logging_result = self.normalize_logging_result(result=result)
if standard_logging_object is None and result is not None and self.stream is not True:
if self._is_recognized_call_type_for_logging(logging_result=logging_result):
self._process_hidden_params_and_response_cost(logging_result=logging_result, start_time=start_time, end_time=end_time)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
if (
standard_logging_object is None
and result is not None
and self.stream is not True
):
if self._is_recognized_call_type_for_logging(
logging_result=logging_result
):
self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = standard_logging_object
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
else:
self.model_call_details["response_cost"] = None
result = self._transform_usage_objects(result=result)
if litellm.max_budget and self.stream is False and result is not None and isinstance(result, dict) and "content" in result:
if (
litellm.max_budget
and self.stream is False
and result is not None
and isinstance(result, dict)
and "content" in result
):
time_diff = (end_time - start_time).total_seconds()
float_diff = float(time_diff)
litellm._current_cost += litellm.completion_cost(
@ -3340,8 +3457,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_in_memory_loggers.append(_literalai_logger)
return _literalai_logger # type: ignore
elif logging_integration == "prometheus":
if PrometheusLogger is None:
raise ValueError("PrometheusLogger is not initialized")
for callback in _in_memory_loggers:
if isinstance(callback, PrometheusLogger):
return callback # type: ignore

3
litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py
View file

@ -4,6 +4,7 @@ from typing import List, Literal
def get_formatted_prompt(
data: dict,
call_type: Literal[
"acompletion",
"completion",
"embedding",
"image_generation",
@ -18,7 +19,7 @@ def get_formatted_prompt(
Returns a string.
"""
prompt = ""
if call_type == "completion":
if call_type == "acompletion" or call_type == "completion":
for message in data["messages"]:
if message.get("content", None) is not None:
content = message.get("content")

Some files were not shown because too many files have changed in this diff Show more