mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge branch 'litellm_oss_staging_01_27_2026' into litellm_staging_12_18_2025
This commit is contained in:
Krish Dholakia
GitHub
commit
fbb9dc1358
1929 changed files with 190113 additions and 22430 deletions
|
|
@ -44,8 +44,8 @@ commands:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "hypercorn==0.17.3"
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "mcp==1.10.1"
|
||||
pip install "pydantic==2.11.0"
|
||||
pip install "mcp==1.25.0"
|
||||
pip install "requests-mock>=1.12.1"
|
||||
pip install "responses==0.25.7"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
|
|
@ -112,14 +112,14 @@ jobs:
|
|||
python -m mypy .
|
||||
cd ..
|
||||
no_output_timeout: 10m
|
||||
local_testing:
|
||||
local_testing_part1:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
|
|
@ -178,6 +178,7 @@ jobs:
|
|||
pip install "Pillow==10.3.0"
|
||||
pip install "jsonschema==4.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install "pytest-timeout==2.2.0"
|
||||
pip install "websockets==13.1.0"
|
||||
pip install semantic_router --no-deps
|
||||
pip install aurelio_sdk --no-deps
|
||||
|
|
@ -204,17 +205,32 @@ jobs:
|
|||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
name: Run tests (Part 1 - A-M)
|
||||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files (A-M)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest \
|
||||
-vv \
|
||||
--cov=litellm \
|
||||
--cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=20 \
|
||||
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
||||
-n 4 \
|
||||
--timeout=300 \
|
||||
--timeout_method=thread"
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_coverage.xml
|
||||
mv .coverage local_testing_coverage
|
||||
mv coverage.xml local_testing_part1_coverage.xml
|
||||
mv .coverage local_testing_part1_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
|
|
@ -222,8 +238,136 @@ jobs:
|
|||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_coverage.xml
|
||||
- local_testing_coverage
|
||||
- local_testing_part1_coverage.xml
|
||||
- local_testing_part1_coverage
|
||||
local_testing_part2:
|
||||
docker:
|
||||
- image: cimg/python:3.12
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
- run:
|
||||
name: Show git commit hash
|
||||
command: |
|
||||
echo "Git commit hash: $CIRCLE_SHA1"
|
||||
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -r .circleci/requirements.txt
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-retry==1.6.3"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "pytest-cov==5.0.0"
|
||||
pip install "mypy==1.18.2"
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
pip install "langfuse==2.59.7"
|
||||
pip install "logfire==0.29.0"
|
||||
pip install numpydoc
|
||||
pip install traceloop-sdk==0.21.1
|
||||
pip install opentelemetry-api==1.25.0
|
||||
pip install opentelemetry-sdk==1.25.0
|
||||
pip install opentelemetry-exporter-otlp==1.25.0
|
||||
pip install openai==1.100.1
|
||||
pip install prisma==0.11.0
|
||||
pip install "detect_secrets==1.5.0"
|
||||
pip install "httpx==0.24.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install fastapi
|
||||
pip install "gunicorn==21.2.0"
|
||||
pip install "anyio==4.2.0"
|
||||
pip install "aiodynamo==23.10.1"
|
||||
pip install "asyncio==3.4.3"
|
||||
pip install "apscheduler==3.10.4"
|
||||
pip install "PyGithub==1.59.1"
|
||||
pip install argon2-cffi
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install python-multipart
|
||||
pip install google-cloud-aiplatform
|
||||
pip install prometheus-client==0.20.0
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "diskcache==5.6.1"
|
||||
pip install "Pillow==10.3.0"
|
||||
pip install "jsonschema==4.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install "pytest-timeout==2.2.0"
|
||||
pip install "websockets==13.1.0"
|
||||
pip install semantic_router --no-deps
|
||||
pip install aurelio_sdk --no-deps
|
||||
pip uninstall posthog -y
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./venv
|
||||
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
||||
- run:
|
||||
name: Run prisma ./docker/entrypoint.sh
|
||||
command: |
|
||||
set +e
|
||||
chmod +x docker/entrypoint.sh
|
||||
./docker/entrypoint.sh
|
||||
set -e
|
||||
- run:
|
||||
name: Black Formatting
|
||||
command: |
|
||||
cd litellm
|
||||
python -m pip install black
|
||||
python -m black .
|
||||
cd ..
|
||||
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests (Part 2 - N-Z)
|
||||
command: |
|
||||
mkdir test-results
|
||||
|
||||
# Discover test files (N-Z)
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="xargs python -m pytest \
|
||||
-vv \
|
||||
--cov=litellm \
|
||||
--cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=20 \
|
||||
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
||||
-n 4 \
|
||||
--timeout=300 \
|
||||
--timeout_method=thread"
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part2_coverage.xml
|
||||
mv .coverage local_testing_part2_coverage
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- local_testing_part2_coverage.xml
|
||||
- local_testing_part2_coverage
|
||||
langfuse_logging_unit_tests:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
|
|
@ -495,7 +639,6 @@ jobs:
|
|||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
|
|
@ -509,6 +652,7 @@ jobs:
|
|||
pip install "pytest-cov==5.0.0"
|
||||
pip install "pytest-retry==1.6.3"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install semantic_router --no-deps
|
||||
pip install aurelio_sdk --no-deps
|
||||
# Run pytest and generate JUnit XML report
|
||||
|
|
@ -614,6 +758,12 @@ jobs:
|
|||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
export PATH="$HOME/miniconda/bin:$PATH"
|
||||
source $HOME/miniconda/etc/profile.d/conda.sh
|
||||
conda activate myenv
|
||||
python --version
|
||||
which python
|
||||
pip install --upgrade typing-extensions>=4.12.0
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install aiohttp
|
||||
|
|
@ -677,6 +827,9 @@ jobs:
|
|||
- run:
|
||||
name: Run prisma ./docker/entrypoint.sh
|
||||
command: |
|
||||
export PATH="$HOME/miniconda/bin:$PATH"
|
||||
source $HOME/miniconda/etc/profile.d/conda.sh
|
||||
conda activate myenv
|
||||
set +e
|
||||
chmod +x docker/entrypoint.sh
|
||||
./docker/entrypoint.sh
|
||||
|
|
@ -685,6 +838,9 @@ jobs:
|
|||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
export PATH="$HOME/miniconda/bin:$PATH"
|
||||
source $HOME/miniconda/etc/profile.d/conda.sh
|
||||
conda activate myenv
|
||||
pwd
|
||||
ls
|
||||
python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5
|
||||
|
|
@ -1090,13 +1246,16 @@ jobs:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
pip install "pytest-timeout==2.2.0"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=5 -n 4
|
||||
# Add --timeout to kill hanging tests after 120s (2 min)
|
||||
# Add --durations=20 to show 20 slowest tests for debugging
|
||||
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -1133,8 +1292,8 @@ jobs:
|
|||
pip install "pytest-cov==5.0.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "mcp==1.10.1"
|
||||
pip install "pydantic==2.11.0"
|
||||
pip install "mcp==1.25.0"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
|
|
@ -1446,7 +1605,7 @@ jobs:
|
|||
- run:
|
||||
name: Run core tests
|
||||
command: |
|
||||
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
|
||||
python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --ignore=tests/test_litellm/integrations --ignore=tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -1460,6 +1619,60 @@ jobs:
|
|||
paths:
|
||||
- litellm_core_tests_coverage.xml
|
||||
- litellm_core_tests_coverage
|
||||
litellm_mapped_tests_litellm_core_utils:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Run litellm_core_utils tests
|
||||
command: |
|
||||
python -m pytest tests/test_litellm/litellm_core_utils --cov=litellm --cov-report=xml --junitxml=test-results/junit-litellm-core-utils.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml litellm_core_utils_tests_coverage.xml
|
||||
mv .coverage litellm_core_utils_tests_coverage
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- litellm_core_utils_tests_coverage.xml
|
||||
- litellm_core_utils_tests_coverage
|
||||
litellm_mapped_tests_integrations:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
auth:
|
||||
username: ${DOCKERHUB_USERNAME}
|
||||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Run integrations tests
|
||||
command: |
|
||||
python -m pytest tests/test_litellm/integrations --cov=litellm --cov-report=xml --junitxml=test-results/junit-integrations.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml litellm_integrations_tests_coverage.xml
|
||||
mv .coverage litellm_integrations_tests_coverage
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- litellm_integrations_tests_coverage.xml
|
||||
- litellm_integrations_tests_coverage
|
||||
litellm_mapped_enterprise_tests:
|
||||
docker:
|
||||
- image: cimg/python:3.11
|
||||
|
|
@ -1483,8 +1696,8 @@ jobs:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "hypercorn==0.17.3"
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "mcp==1.10.1"
|
||||
pip install "pydantic==2.11.0"
|
||||
pip install "mcp==1.25.0"
|
||||
pip install "requests-mock>=1.12.1"
|
||||
pip install "responses==0.25.7"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
|
|
@ -1670,13 +1883,14 @@ jobs:
|
|||
pip install "pytest-cov==5.0.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
|
||||
python -m pytest -vv tests/image_gen_tests -n 4 --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -1719,6 +1933,7 @@ jobs:
|
|||
pip install "mlflow==2.17.2"
|
||||
pip install "anthropic==0.52.0"
|
||||
pip install "blockbuster==1.5.24"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- setup_litellm_enterprise_pip
|
||||
- run:
|
||||
|
|
@ -1726,7 +1941,7 @@ jobs:
|
|||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
|
||||
python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
|
||||
no_output_timeout: 120m
|
||||
- run:
|
||||
name: Rename the coverage files
|
||||
|
|
@ -1842,7 +2057,7 @@ jobs:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "pytest-cov==5.0.0"
|
||||
pip install "tomli==2.2.1"
|
||||
pip install "mcp==1.10.1"
|
||||
pip install "mcp==1.25.0"
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
|
|
@ -1886,6 +2101,18 @@ jobs:
|
|||
command: |
|
||||
kind create cluster --name litellm-test
|
||||
|
||||
- run:
|
||||
name: Build Docker image for helm tests
|
||||
command: |
|
||||
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
||||
docker build -t litellm-ci:${IMAGE_TAG} -f docker/Dockerfile.database .
|
||||
|
||||
- run:
|
||||
name: Load Docker image into Kind
|
||||
command: |
|
||||
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
||||
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
|
||||
|
||||
# Run helm lint
|
||||
- run:
|
||||
name: Run helm lint
|
||||
|
|
@ -1896,7 +2123,11 @@ jobs:
|
|||
- run:
|
||||
name: Run helm tests
|
||||
command: |
|
||||
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml
|
||||
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
||||
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
|
||||
--set image.repository=litellm-ci \
|
||||
--set image.tag=${IMAGE_TAG} \
|
||||
--set image.pullPolicy=Never
|
||||
# Wait for pod to be ready
|
||||
echo "Waiting 30 seconds for pod to be ready..."
|
||||
sleep 30
|
||||
|
|
@ -1941,11 +2172,13 @@ jobs:
|
|||
- run: ruff check ./litellm
|
||||
# - run: python ./tests/documentation_tests/test_general_setting_keys.py
|
||||
- run: python ./tests/code_coverage_tests/check_licenses.py
|
||||
- run: python ./tests/code_coverage_tests/check_provider_folders_documented.py
|
||||
- run: python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
- run: python ./tests/code_coverage_tests/info_log_check.py
|
||||
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
|
||||
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
|
||||
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
||||
- run: python ./tests/code_coverage_tests/test_proxy_types_import.py
|
||||
- run: python ./tests/code_coverage_tests/callback_manager_test.py
|
||||
- run: python ./tests/code_coverage_tests/recursive_detector.py
|
||||
|
|
@ -1961,6 +2194,7 @@ jobs:
|
|||
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
|
||||
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
|
||||
- run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
|
||||
- run: python ./tests/code_coverage_tests/memory_test.py
|
||||
- run: helm lint ./deploy/charts/litellm-helm
|
||||
|
||||
db_migration_disable_update_check:
|
||||
|
|
@ -1989,10 +2223,13 @@ jobs:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install aiohttp
|
||||
pip install apscheduler
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
docker build -t myapp . -f ./docker/Dockerfile.database
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
command: |
|
||||
|
|
@ -2005,7 +2242,7 @@ jobs:
|
|||
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
||||
--name my-app \
|
||||
myapp:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000
|
||||
- run:
|
||||
|
|
@ -2024,10 +2261,11 @@ jobs:
|
|||
name: Check container logs for expected message
|
||||
command: |
|
||||
echo "=== Printing Full Container Startup Logs ==="
|
||||
docker logs my-app
|
||||
LOG_OUTPUT="$(docker logs my-app 2>&1)"
|
||||
printf '%s\n' "$LOG_OUTPUT"
|
||||
echo "=== End of Full Container Startup Logs ==="
|
||||
|
||||
if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
||||
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
||||
echo "Expected message found in logs. Test passed."
|
||||
else
|
||||
echo "Expected message not found in logs. Test failed."
|
||||
|
|
@ -2096,6 +2334,8 @@ jobs:
|
|||
pip install "asyncio==3.4.3"
|
||||
pip install "PyGithub==1.59.1"
|
||||
pip install "openai==1.100.1"
|
||||
pip install "litellm[proxy]"
|
||||
pip install "pytest-xdist==3.6.1"
|
||||
- run:
|
||||
name: Install dockerize
|
||||
command: |
|
||||
|
|
@ -2172,7 +2412,7 @@ jobs:
|
|||
command: |
|
||||
pwd
|
||||
ls
|
||||
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
|
||||
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
|
||||
no_output_timeout: 120m
|
||||
|
||||
# Store test results
|
||||
|
|
@ -2257,9 +2497,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
command: |
|
||||
|
|
@ -2294,7 +2538,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -2397,9 +2641,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
# intentionally give bad redis credentials here
|
||||
|
|
@ -2432,7 +2680,7 @@ jobs:
|
|||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -2483,7 +2731,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app-3 \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug
|
||||
|
|
@ -2558,9 +2806,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
# intentionally give bad redis credentials here
|
||||
|
|
@ -2584,7 +2836,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -2671,9 +2923,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container 1
|
||||
# intentionally give bad redis credentials here
|
||||
|
|
@ -2693,7 +2949,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -2714,7 +2970,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app-2 \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4001 \
|
||||
--detailed_debug
|
||||
|
|
@ -2807,9 +3063,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
# intentionally give bad redis credentials here
|
||||
|
|
@ -2824,7 +3084,7 @@ jobs:
|
|||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -3039,10 +3299,13 @@ jobs:
|
|||
- run:
|
||||
name: Wait for PostgreSQL to be ready
|
||||
command: dockerize -wait tcp://localhost:5432 -timeout 1m
|
||||
# Run pytest and generate JUnit XML report
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Run Docker container
|
||||
command: |
|
||||
|
|
@ -3064,7 +3327,7 @@ jobs:
|
|||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
|
|
@ -3165,7 +3428,7 @@ jobs:
|
|||
python -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
|
||||
coverage xml
|
||||
- codecov/upload:
|
||||
file: ./coverage.xml
|
||||
|
|
@ -3402,6 +3665,37 @@ jobs:
|
|||
--coverage.reporter=html \
|
||||
--coverage.reportsDirectory=coverage/html
|
||||
|
||||
build_docker_database_image:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.10.1
|
||||
resource_class: xlarge
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- run:
|
||||
name: Upgrade Docker
|
||||
command: |
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
docker version
|
||||
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: |
|
||||
docker build \
|
||||
-t litellm-docker-database:ci \
|
||||
-f docker/Dockerfile.database .
|
||||
|
||||
- run:
|
||||
name: Save Docker image to workspace root
|
||||
command: |
|
||||
docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz
|
||||
|
||||
- persist_to_workspace:
|
||||
root: .
|
||||
paths:
|
||||
- litellm-docker-database.tar.gz
|
||||
|
||||
e2e_ui_testing:
|
||||
machine:
|
||||
image: ubuntu-2204:2023.10.1
|
||||
|
|
@ -3413,68 +3707,54 @@ jobs:
|
|||
- attach_workspace:
|
||||
at: ~/project
|
||||
- run:
|
||||
name: Upgrade Docker to v24.x (API 1.44+)
|
||||
name: Load Docker Database Image
|
||||
command: |
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
sudo usermod -aG docker $USER
|
||||
docker version
|
||||
- run:
|
||||
name: Install Python 3.9
|
||||
command: |
|
||||
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
||||
bash miniconda.sh -b -p $HOME/miniconda
|
||||
export PATH="$HOME/miniconda/bin:$PATH"
|
||||
conda init bash
|
||||
source ~/.bashrc
|
||||
conda create -n myenv python=3.9 -y
|
||||
conda activate myenv
|
||||
python --version
|
||||
gunzip -c litellm-docker-database.tar.gz | docker load
|
||||
docker images | grep litellm-docker-database
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
npm install -D @playwright/test
|
||||
npm install @google-cloud/vertexai
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-retry==1.6.3"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install aiohttp
|
||||
pip install "openai==1.100.1"
|
||||
python -m pip install --upgrade pip
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "mypy==1.18.2"
|
||||
pip install pyarrow
|
||||
pip install numpydoc
|
||||
pip install prisma
|
||||
pip install fastapi
|
||||
pip install jsonschema
|
||||
pip install "httpx==0.24.1"
|
||||
pip install "anyio==3.7.1"
|
||||
pip install "asyncio==3.4.3"
|
||||
- run:
|
||||
name: Install Playwright Browsers
|
||||
command: |
|
||||
npx playwright install
|
||||
|
||||
- run:
|
||||
name: Build Docker image
|
||||
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||
name: Install Neon CLI
|
||||
command: |
|
||||
npm i -g neonctl
|
||||
- run:
|
||||
name: Create Neon branch
|
||||
command: |
|
||||
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
|
||||
echo "Expires at: $EXPIRES_AT"
|
||||
neon branches create \
|
||||
--project-id $NEON_PROJECT_ID \
|
||||
--name preview/commit-${CIRCLE_SHA1:0:7} \
|
||||
--expires-at $EXPIRES_AT \
|
||||
--parent br-fancy-paper-ad1olsb3 \
|
||||
--api-key $NEON_API_KEY || true
|
||||
- run:
|
||||
name: Run Docker container
|
||||
command: |
|
||||
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
|
||||
--project-id $NEON_PROJECT_ID \
|
||||
--api-key $NEON_API_KEY \
|
||||
--branch preview/commit-${CIRCLE_SHA1:0:7} \
|
||||
--database-name yuneng-trial-db \
|
||||
--role neondb_owner)
|
||||
echo $E2E_UI_TEST_DATABASE_URL
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL=$SMALL_DATABASE_URL \
|
||||
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
|
||||
-e LITELLM_MASTER_KEY="sk-1234" \
|
||||
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-e UI_USERNAME="admin" \
|
||||
-e UI_PASSWORD="gm" \
|
||||
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||
--name my-app \
|
||||
--name litellm-docker-database \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug
|
||||
|
|
@ -3488,7 +3768,7 @@ jobs:
|
|||
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
command: docker logs -f litellm-docker-database
|
||||
background: true
|
||||
- run:
|
||||
name: Wait for app to be ready
|
||||
|
|
@ -3496,7 +3776,10 @@ jobs:
|
|||
- run:
|
||||
name: Run Playwright Tests
|
||||
command: |
|
||||
npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
|
||||
npx playwright test \
|
||||
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
|
||||
--reporter=html \
|
||||
--output=test-results
|
||||
no_output_timeout: 120m
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
|
@ -3600,7 +3883,13 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing:
|
||||
- local_testing_part1:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- local_testing_part2:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -3686,9 +3975,17 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- build_docker_database_image:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- e2e_ui_testing:
|
||||
context: e2e_ui_tests
|
||||
requires:
|
||||
- ui_build
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -3701,30 +3998,40 @@ workflows:
|
|||
- main
|
||||
- /litellm_.*/
|
||||
- e2e_openai_endpoints:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- proxy_logging_guardrails_model_info_tests:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- proxy_spend_accuracy_tests:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- proxy_multi_instance_tests:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- proxy_store_model_in_db_tests:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -3737,6 +4044,8 @@ workflows:
|
|||
- main
|
||||
- /litellm_.*/
|
||||
- proxy_pass_through_endpoint_tests:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -3808,6 +4117,18 @@ workflows:
|
|||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- litellm_mapped_tests_integrations:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- litellm_mapped_tests_litellm_core_utils:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- main
|
||||
- /litellm_.*/
|
||||
- batches_testing:
|
||||
filters:
|
||||
branches:
|
||||
|
|
@ -3856,6 +4177,8 @@ workflows:
|
|||
- litellm_mapped_tests_proxy
|
||||
- litellm_mapped_tests_llms
|
||||
- litellm_mapped_tests_core
|
||||
- litellm_mapped_tests_integrations
|
||||
- litellm_mapped_tests_litellm_core_utils
|
||||
- litellm_mapped_enterprise_tests
|
||||
- batches_testing
|
||||
- litellm_utils_testing
|
||||
|
|
@ -3871,10 +4194,13 @@ workflows:
|
|||
- litellm_proxy_unit_testing_part2
|
||||
- litellm_security_tests
|
||||
- langfuse_logging_unit_tests
|
||||
- local_testing
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- litellm_assistants_api_testing
|
||||
- auth_ui_unit_tests
|
||||
- db_migration_disable_update_check:
|
||||
requires:
|
||||
- build_docker_database_image
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
|
|
@ -3912,7 +4238,8 @@ workflows:
|
|||
- publish_to_pypi:
|
||||
requires:
|
||||
- mypy_linting
|
||||
- local_testing
|
||||
- local_testing_part1
|
||||
- local_testing_part2
|
||||
- build_and_test
|
||||
- e2e_openai_endpoints
|
||||
- test_bad_database_url
|
||||
|
|
@ -3925,6 +4252,8 @@ workflows:
|
|||
- litellm_mapped_tests_proxy
|
||||
- litellm_mapped_tests_llms
|
||||
- litellm_mapped_tests_core
|
||||
- litellm_mapped_tests_integrations
|
||||
- litellm_mapped_tests_litellm_core_utils
|
||||
- litellm_mapped_enterprise_tests
|
||||
- batches_testing
|
||||
- litellm_utils_testing
|
||||
|
|
|
|||
|
|
@ -8,12 +8,12 @@ redis==5.2.1
|
|||
redisvl==0.4.1
|
||||
anthropic
|
||||
orjson==3.10.12 # fast /embedding responses
|
||||
pydantic==2.10.2
|
||||
pydantic==2.11.0
|
||||
google-cloud-aiplatform==1.43.0
|
||||
google-cloud-iam==2.19.1
|
||||
fastapi-sso==0.16.0
|
||||
uvloop==0.21.0
|
||||
mcp==1.10.1 # for MCP server
|
||||
mcp==1.25.0 # for MCP server
|
||||
semantic_router==0.1.10 # for auto-routing with litellm
|
||||
fastuuid==0.12.0
|
||||
responses==0.25.7 # for proxy client tests
|
||||
111
.gitguardian.yaml
Normal file
111
.gitguardian.yaml
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
version: 2
|
||||
|
||||
secret:
|
||||
# Exclude files and paths by globbing
|
||||
ignored_paths:
|
||||
- "**/*.whl"
|
||||
- "**/*.pyc"
|
||||
- "**/__pycache__/**"
|
||||
- "**/node_modules/**"
|
||||
- "**/dist/**"
|
||||
- "**/build/**"
|
||||
- "**/.git/**"
|
||||
- "**/venv/**"
|
||||
- "**/.venv/**"
|
||||
|
||||
# Large data/metadata files that don't need scanning
|
||||
- "**/model_prices_and_context_window*.json"
|
||||
- "**/*_metadata/*.txt"
|
||||
- "**/tokenizers/*.json"
|
||||
- "**/tokenizers/*"
|
||||
- "miniconda.sh"
|
||||
|
||||
# Build outputs and static assets
|
||||
- "litellm/proxy/_experimental/out/**"
|
||||
- "ui/litellm-dashboard/public/**"
|
||||
- "**/swagger/*.js"
|
||||
- "**/*.woff"
|
||||
- "**/*.woff2"
|
||||
- "**/*.avif"
|
||||
- "**/*.webp"
|
||||
|
||||
# Test data files
|
||||
- "**/tests/**/data_map.txt"
|
||||
- "tests/**/*.txt"
|
||||
|
||||
# Documentation and other non-code files
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
- "**/*.lock"
|
||||
- "poetry.lock"
|
||||
- "package-lock.json"
|
||||
|
||||
# Ignore security incidents with the SHA256 of the occurrence (false positives)
|
||||
ignored_matches:
|
||||
# === Current detected false positives (SHA-based) ===
|
||||
|
||||
# gcs_pub_sub_body - folder name, not a password
|
||||
- name: GCS pub/sub test folder name
|
||||
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
|
||||
|
||||
# os.environ/APORIA_API_KEY_1 - environment variable reference
|
||||
- name: Environment variable reference APORIA_API_KEY_1
|
||||
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
|
||||
|
||||
# os.environ/APORIA_API_KEY_2 - environment variable reference
|
||||
- name: Environment variable reference APORIA_API_KEY_2
|
||||
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
|
||||
|
||||
# oidc/circleci_v2/ - test authentication path, not a secret
|
||||
- name: OIDC CircleCI test path
|
||||
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
|
||||
|
||||
# text-davinci-003 - OpenAI model identifier, not a secret
|
||||
- name: OpenAI model identifier text-davinci-003
|
||||
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
|
||||
|
||||
# Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
|
||||
- name: Test Base64 Basic Auth header in pass_through_endpoints test
|
||||
match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
|
||||
|
||||
# PostgreSQL password "postgres" in CI configs - standard test database password
|
||||
- name: Test PostgreSQL password in CI configurations
|
||||
match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
|
||||
|
||||
# Bearer token in locustfile.py - test/example API key for load testing
|
||||
- name: Test Bearer token in locustfile load test
|
||||
match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
|
||||
|
||||
# Inkeep API key in docusaurus.config.js - public documentation site key
|
||||
- name: Inkeep API key in documentation config
|
||||
match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
|
||||
|
||||
# Langfuse credentials in test_completion.py - test credentials for integration test
|
||||
- name: Langfuse test credentials in test_completion
|
||||
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
|
||||
|
||||
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
|
||||
- name: Test password in e2e test fixtures
|
||||
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
|
||||
|
||||
# === Preventive patterns for test keys (pattern-based) ===
|
||||
|
||||
# Test API keys (124 instances across 45 files)
|
||||
- name: Test API keys with sk-test prefix
|
||||
match: sk-test-
|
||||
|
||||
# Mock API keys
|
||||
- name: Mock API keys with sk-mock prefix
|
||||
match: sk-mock-
|
||||
|
||||
# Fake API keys
|
||||
- name: Fake API keys with sk-fake prefix
|
||||
match: sk-fake-
|
||||
|
||||
# Generic test API key patterns
|
||||
- name: Test API key patterns
|
||||
match: test-api-key
|
||||
|
||||
- name: Short fake sk keys (1–9 digits only)
|
||||
match: \bsk-\d{1,9}\b
|
||||
|
||||
26
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
26
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
|
|
@ -7,6 +7,16 @@ body:
|
|||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to fill out this bug report!
|
||||
|
||||
**💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
|
||||
- type: checkboxes
|
||||
id: duplicate-check
|
||||
attributes:
|
||||
label: Check for existing issues
|
||||
description: Please search to see if an issue already exists for the bug you encountered.
|
||||
options:
|
||||
- label: I have searched the existing issues and checked that my issue is not a duplicate.
|
||||
required: true
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
|
|
@ -16,6 +26,21 @@ body:
|
|||
value: "A bug happened!"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: steps-to-reproduce
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
|
||||
placeholder: |
|
||||
1. config.yaml file/ .env file/ etc.
|
||||
2. Run the following code...
|
||||
3. Observe the error...
|
||||
value: |
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
|
|
@ -27,6 +52,7 @@ body:
|
|||
attributes:
|
||||
label: What part of LiteLLM is this about?
|
||||
options:
|
||||
- ''
|
||||
- "SDK (litellm Python package)"
|
||||
- "Proxy"
|
||||
- "UI Dashboard"
|
||||
|
|
|
|||
9
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
9
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
|
|
@ -7,6 +7,14 @@ body:
|
|||
attributes:
|
||||
value: |
|
||||
Thanks for making LiteLLM better!
|
||||
- type: checkboxes
|
||||
id: duplicate-check
|
||||
attributes:
|
||||
label: Check for existing issues
|
||||
description: Please search to see if an issue already exists for the feature you are requesting.
|
||||
options:
|
||||
- label: I have searched the existing issues and checked that my issue is not a duplicate.
|
||||
required: true
|
||||
- type: textarea
|
||||
id: the-feature
|
||||
attributes:
|
||||
|
|
@ -27,6 +35,7 @@ body:
|
|||
attributes:
|
||||
label: What part of LiteLLM is this about?
|
||||
options:
|
||||
- ''
|
||||
- "SDK (litellm Python package)"
|
||||
- "Proxy"
|
||||
- "UI Dashboard"
|
||||
|
|
|
|||
29
.github/workflows/check_duplicate_issues.yml
vendored
Normal file
29
.github/workflows/check_duplicate_issues.yml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
name: Check Duplicate Issues
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
|
||||
jobs:
|
||||
check-duplicate:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
steps:
|
||||
- name: Check for potential duplicates
|
||||
uses: wow-actions/potential-duplicates@v1
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
label: potential-duplicate
|
||||
threshold: 0.6
|
||||
reaction: eyes
|
||||
comment: |
|
||||
**⚠️ Potential duplicate detected**
|
||||
|
||||
This issue appears similar to existing issue(s):
|
||||
{{#issues}}
|
||||
- [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
|
||||
{{/issues}}
|
||||
|
||||
Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
|
||||
|
|
@ -2,7 +2,7 @@ name: Create Daily Staging Branch
|
|||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # Runs daily at midnight UTC
|
||||
- cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
jobs:
|
||||
|
|
@ -24,7 +24,7 @@ jobs:
|
|||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
# Generate branch name with MM_DD_YYYY format
|
||||
BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')"
|
||||
BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
|
||||
echo "Creating branch: $BRANCH_NAME"
|
||||
|
||||
# Fetch all branches
|
||||
|
|
|
|||
65
.github/workflows/ghcr_deploy.yml
vendored
65
.github/workflows/ghcr_deploy.yml
vendored
|
|
@ -5,6 +5,7 @@ on:
|
|||
inputs:
|
||||
tag:
|
||||
description: "The tag version you want to build"
|
||||
required: true
|
||||
release_type:
|
||||
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
|
||||
type: string
|
||||
|
|
@ -319,59 +320,37 @@ jobs:
|
|||
run: |
|
||||
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
|
||||
|
||||
- name: Get LiteLLM Latest Tag
|
||||
id: current_app_tag
|
||||
shell: bash
|
||||
run: |
|
||||
LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
|
||||
if [ -z "${LATEST_TAG}" ]; then
|
||||
echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
|
||||
else
|
||||
echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Get last published chart version
|
||||
id: current_version
|
||||
shell: bash
|
||||
run: |
|
||||
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
|
||||
if [ -z "${CHART_LIST}" ]; then
|
||||
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
|
||||
else
|
||||
# Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
|
||||
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
|
||||
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
fi
|
||||
env:
|
||||
HELM_EXPERIMENTAL_OCI: '1'
|
||||
|
||||
# Automatically update the helm chart version one "patch" level
|
||||
- name: Bump release version
|
||||
id: bump_version
|
||||
uses: christian-draeger/increment-semantic-version@1.1.0
|
||||
with:
|
||||
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
|
||||
version-fragment: 'bug'
|
||||
|
||||
# Add suffix for non-stable releases (semantic versioning)
|
||||
- name: Calculate chart version with prerelease suffix
|
||||
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
|
||||
# This allows users to easily map Helm chart versions to LiteLLM versions
|
||||
# See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
|
||||
- name: Calculate chart and app versions
|
||||
id: chart_version
|
||||
shell: bash
|
||||
run: |
|
||||
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
|
||||
INPUT_TAG="${{ github.event.inputs.tag }}"
|
||||
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
|
||||
if [ "$RELEASE_TYPE" = "stable" ]; then
|
||||
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
# Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
|
||||
# v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
|
||||
CHART_VERSION="${INPUT_TAG#v}"
|
||||
|
||||
# Add suffix for 'latest' releases (rc already has suffix in tag)
|
||||
if [ "$RELEASE_TYPE" = "latest" ]; then
|
||||
CHART_VERSION="${CHART_VERSION}-latest"
|
||||
fi
|
||||
|
||||
# App version = Docker tag (keeps 'v' prefix to match Docker image tags)
|
||||
APP_VERSION="${INPUT_TAG}"
|
||||
|
||||
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
- uses: ./.github/actions/helm-oci-chart-releaser
|
||||
with:
|
||||
name: ${{ env.CHART_NAME }}
|
||||
repository: ${{ env.REPO_OWNER }}
|
||||
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
|
||||
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
|
||||
tag: ${{ steps.chart_version.outputs.version }}
|
||||
app_version: ${{ steps.chart_version.outputs.app_version }}
|
||||
path: deploy/charts/${{ env.CHART_NAME }}
|
||||
registry: ${{ env.REGISTRY }}
|
||||
registry_username: ${{ github.actor }}
|
||||
|
|
|
|||
42
.github/workflows/ghcr_helm_deploy.yml
vendored
42
.github/workflows/ghcr_helm_deploy.yml
vendored
|
|
@ -1,10 +1,12 @@
|
|||
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
|
||||
# Standalone workflow to publish LiteLLM Helm Chart
|
||||
# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
|
||||
name: Build, Publish LiteLLM Helm Chart. New Release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
chartVersion:
|
||||
description: "Update the helm chart's version to this"
|
||||
tag:
|
||||
description: "LiteLLM version tag (e.g., v1.81.0)"
|
||||
required: true
|
||||
|
||||
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
|
||||
env:
|
||||
|
|
@ -31,24 +33,22 @@ jobs:
|
|||
run: |
|
||||
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
|
||||
|
||||
- name: Get LiteLLM Latest Tag
|
||||
id: current_app_tag
|
||||
uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
|
||||
|
||||
- name: Get last published chart version
|
||||
id: current_version
|
||||
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
|
||||
- name: Calculate chart and app versions
|
||||
id: chart_version
|
||||
shell: bash
|
||||
run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
|
||||
env:
|
||||
HELM_EXPERIMENTAL_OCI: '1'
|
||||
run: |
|
||||
INPUT_TAG="${{ github.event.inputs.tag }}"
|
||||
|
||||
# Automatically update the helm chart version one "patch" level
|
||||
- name: Bump release version
|
||||
id: bump_version
|
||||
uses: christian-draeger/increment-semantic-version@1.1.0
|
||||
with:
|
||||
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
|
||||
version-fragment: 'bug'
|
||||
# Chart version = LiteLLM version without 'v' prefix
|
||||
# v1.81.0 -> 1.81.0
|
||||
CHART_VERSION="${INPUT_TAG#v}"
|
||||
|
||||
# App version = Docker tag (keeps 'v' prefix)
|
||||
APP_VERSION="${INPUT_TAG}"
|
||||
|
||||
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
- name: Lint helm chart
|
||||
run: helm lint deploy/charts/litellm-helm
|
||||
|
|
@ -57,8 +57,8 @@ jobs:
|
|||
with:
|
||||
name: litellm-helm
|
||||
repository: ${{ env.REPO_OWNER }}
|
||||
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
|
||||
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
|
||||
tag: ${{ steps.chart_version.outputs.version }}
|
||||
app_version: ${{ steps.chart_version.outputs.app_version }}
|
||||
path: deploy/charts/litellm-helm
|
||||
registry: ${{ env.REGISTRY }}
|
||||
registry_username: ${{ github.actor }}
|
||||
|
|
|
|||
208
.github/workflows/label-component.yml
vendored
208
.github/workflows/label-component.yml
vendored
|
|
@ -11,134 +11,106 @@ jobs:
|
|||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Add SDK label
|
||||
if: contains(github.event.issue.body, 'SDK (litellm Python package)')
|
||||
- name: Add component labels
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const labelName = 'sdk';
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName,
|
||||
color: '0E7C86',
|
||||
description: 'Issues related to the litellm Python SDK'
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [labelName]
|
||||
});
|
||||
const body = context.payload.issue.body;
|
||||
if (!body) return;
|
||||
|
||||
- name: Add Proxy label
|
||||
if: contains(github.event.issue.body, 'Proxy')
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const labelName = 'proxy';
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName,
|
||||
color: '5319E7',
|
||||
description: 'Issues related to the LiteLLM Proxy'
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
// Define component mappings with regex patterns that handle flexible whitespace
|
||||
const components = [
|
||||
{
|
||||
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
|
||||
label: 'sdk',
|
||||
color: '0E7C86',
|
||||
description: 'Issues related to the litellm Python SDK'
|
||||
},
|
||||
{
|
||||
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
|
||||
label: 'proxy',
|
||||
color: '5319E7',
|
||||
description: 'Issues related to the LiteLLM Proxy'
|
||||
},
|
||||
{
|
||||
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
|
||||
label: 'ui-dashboard',
|
||||
color: 'D876E3',
|
||||
description: 'Issues related to the LiteLLM UI Dashboard'
|
||||
},
|
||||
{
|
||||
pattern: /What part of LiteLLM is this about\?\s*Docs/,
|
||||
label: 'docs',
|
||||
color: 'FBCA04',
|
||||
description: 'Issues related to LiteLLM documentation'
|
||||
}
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [labelName]
|
||||
});
|
||||
];
|
||||
|
||||
- name: Add UI Dashboard label
|
||||
if: contains(github.event.issue.body, 'UI Dashboard')
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const labelName = 'ui-dashboard';
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName,
|
||||
color: 'D876E3',
|
||||
description: 'Issues related to the LiteLLM UI Dashboard'
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [labelName]
|
||||
});
|
||||
// Find matching component
|
||||
for (const component of components) {
|
||||
if (component.pattern.test(body)) {
|
||||
// Ensure label exists
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: component.label
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: component.label,
|
||||
color: component.color,
|
||||
description: component.description
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
- name: Add Docs label
|
||||
if: contains(github.event.issue.body, 'Docs')
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const labelName = 'docs';
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
// Add label to issue
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: labelName,
|
||||
color: 'FBCA04',
|
||||
description: 'Issues related to LiteLLM documentation'
|
||||
issue_number: context.issue.number,
|
||||
labels: [component.label]
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [labelName]
|
||||
});
|
||||
|
||||
// Check for 'claude code' keyword (can be applied alongside component labels)
|
||||
if (/claude code/i.test(body)) {
|
||||
const claudeLabel = {
|
||||
name: 'claude code',
|
||||
color: '7c3aed',
|
||||
description: 'Issues related to Claude Code usage'
|
||||
};
|
||||
|
||||
try {
|
||||
await github.rest.issues.getLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: claudeLabel.name
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
await github.rest.issues.createLabel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
name: claudeLabel.name,
|
||||
color: claudeLabel.color,
|
||||
description: claudeLabel.description
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
labels: [claudeLabel.name]
|
||||
});
|
||||
}
|
||||
|
|
|
|||
1
.github/workflows/publish-migrations.yml
vendored
1
.github/workflows/publish-migrations.yml
vendored
|
|
@ -13,6 +13,7 @@ on:
|
|||
|
||||
jobs:
|
||||
publish-migrations:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
|
|
|
|||
1
.github/workflows/test-litellm.yml
vendored
1
.github/workflows/test-litellm.yml
vendored
|
|
@ -35,6 +35,7 @@ jobs:
|
|||
poetry run pip install "google-cloud-aiplatform>=1.38"
|
||||
poetry run pip install "fastapi-offline==1.7.3"
|
||||
poetry run pip install "python-multipart==0.0.18"
|
||||
poetry run pip install "openapi-core"
|
||||
- name: Setup litellm-enterprise as local package
|
||||
run: |
|
||||
cd enterprise
|
||||
|
|
|
|||
4
.github/workflows/test-mcp.yml
vendored
4
.github/workflows/test-mcp.yml
vendored
|
|
@ -34,8 +34,8 @@ jobs:
|
|||
poetry run pip install "pytest-cov==5.0.0"
|
||||
poetry run pip install "pytest-asyncio==0.21.1"
|
||||
poetry run pip install "respx==0.22.0"
|
||||
poetry run pip install "pydantic==2.10.2"
|
||||
poetry run pip install "mcp==1.10.1"
|
||||
poetry run pip install "pydantic==2.11.0"
|
||||
poetry run pip install "mcp==1.25.0"
|
||||
poetry run pip install pytest-xdist
|
||||
|
||||
- name: Setup litellm-enterprise as local package
|
||||
|
|
|
|||
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -1,5 +1,6 @@
|
|||
.python-version
|
||||
.venv
|
||||
.venv_policy_test
|
||||
.env
|
||||
.newenv
|
||||
newenv/*
|
||||
|
|
@ -59,6 +60,7 @@ litellm/proxy/_super_secret_config.yaml
|
|||
litellm/proxy/myenv/bin/activate
|
||||
litellm/proxy/myenv/bin/Activate.ps1
|
||||
myenv/*
|
||||
litellm/proxy/_experimental/out/_next/
|
||||
litellm/proxy/_experimental/out/404/index.html
|
||||
litellm/proxy/_experimental/out/model_hub/index.html
|
||||
litellm/proxy/_experimental/out/onboarding/index.html
|
||||
|
|
@ -100,3 +102,8 @@ update_model_cost_map.py
|
|||
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
litellm/proxy/_experimental/out/guardrails/index.html
|
||||
scripts/test_vertex_ai_search.py
|
||||
LAZY_LOADING_IMPROVEMENTS.md
|
||||
**/test-results
|
||||
**/playwright-report
|
||||
**/*.storageState.json
|
||||
**/coverage
|
||||
21
AGENTS.md
21
AGENTS.md
|
|
@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that:
|
|||
- Test provider-specific functionality thoroughly
|
||||
- Consider adding load tests for performance-critical changes
|
||||
|
||||
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
|
||||
|
||||
1. **Use Common Components as much as possible**:
|
||||
- These are usually defined in the `common_components` directory
|
||||
- Use these components as much as possible and avoid building new components unless needed
|
||||
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
|
||||
|
||||
2. **Testing**:
|
||||
- The codebase uses **Vitest** and **React Testing Library**
|
||||
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
|
||||
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
|
||||
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
|
||||
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
|
||||
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
|
||||
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
|
||||
- **Structure tests properly**:
|
||||
- First test should verify the component renders successfully
|
||||
- Subsequent tests should focus on functionality and user interactions
|
||||
- Use `waitFor` for async operations that aren't already awaited
|
||||
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
|
||||
|
||||
### IMPORTANT PATTERNS
|
||||
|
||||
1. **Function/Tool Calling**:
|
||||
|
|
|
|||
398
ARCHITECTURE.md
Normal file
398
ARCHITECTURE.md
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
# LiteLLM Architecture - LiteLLM SDK + AI Gateway
|
||||
|
||||
This document helps contributors understand where to make changes in LiteLLM.
|
||||
|
||||
---
|
||||
|
||||
## How It Works
|
||||
|
||||
The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
|
||||
|
||||
```
|
||||
OpenAI SDK (client) ──▶ LiteLLM AI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
|
||||
Anthropic SDK (client) ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
|
||||
Any HTTP client ──▶ LiteLLMAI Gateway (proxy/) ──▶ LiteLLM SDK (litellm/) ──▶ LLM API
|
||||
```
|
||||
|
||||
The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
|
||||
The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
|
||||
|
||||
---
|
||||
|
||||
## 1. AI Gateway (Proxy) Request Flow
|
||||
|
||||
The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Client
|
||||
participant ProxyServer as proxy/proxy_server.py
|
||||
participant Auth as proxy/auth/user_api_key_auth.py
|
||||
participant Redis as Redis Cache
|
||||
participant Hooks as proxy/hooks/
|
||||
participant Router as router.py
|
||||
participant Main as main.py + utils.py
|
||||
participant Handler as llms/custom_httpx/llm_http_handler.py
|
||||
participant Transform as llms/{provider}/chat/transformation.py
|
||||
participant Provider as LLM Provider API
|
||||
participant CostCalc as cost_calculator.py
|
||||
participant LoggingObj as litellm_logging.py
|
||||
participant DBWriter as db/db_spend_update_writer.py
|
||||
participant Postgres as PostgreSQL
|
||||
|
||||
%% Request Flow
|
||||
Client->>ProxyServer: POST /v1/chat/completions
|
||||
ProxyServer->>Auth: user_api_key_auth()
|
||||
Auth->>Redis: Check API key cache
|
||||
Redis-->>Auth: Key info + spend limits
|
||||
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
|
||||
Hooks->>Redis: Check/increment rate limit counters
|
||||
ProxyServer->>Router: route_request()
|
||||
Router->>Main: litellm.acompletion()
|
||||
Main->>Handler: BaseLLMHTTPHandler.completion()
|
||||
Handler->>Transform: ProviderConfig.transform_request()
|
||||
Handler->>Provider: HTTP Request
|
||||
Provider-->>Handler: Response
|
||||
Handler->>Transform: ProviderConfig.transform_response()
|
||||
Transform-->>Handler: ModelResponse
|
||||
Handler-->>Main: ModelResponse
|
||||
|
||||
%% Cost Attribution (in utils.py wrapper)
|
||||
Main->>LoggingObj: update_response_metadata()
|
||||
LoggingObj->>CostCalc: _response_cost_calculator()
|
||||
CostCalc->>CostCalc: completion_cost(tokens × price)
|
||||
CostCalc-->>LoggingObj: response_cost
|
||||
LoggingObj-->>Main: Set response._hidden_params["response_cost"]
|
||||
Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
|
||||
|
||||
%% Response Headers + Async Logging
|
||||
ProxyServer->>ProxyServer: Extract cost from hidden_params
|
||||
ProxyServer->>LoggingObj: async_success_handler()
|
||||
LoggingObj->>Hooks: async_log_success_event()
|
||||
Hooks->>DBWriter: update_database(response_cost)
|
||||
DBWriter->>Redis: Queue spend increment
|
||||
DBWriter->>Postgres: Batch write spend logs (async)
|
||||
ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
|
||||
```
|
||||
|
||||
### Proxy Components
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph "Incoming Request"
|
||||
Client["POST /v1/chat/completions"]
|
||||
end
|
||||
|
||||
subgraph "proxy/proxy_server.py"
|
||||
Endpoint["chat_completion()"]
|
||||
end
|
||||
|
||||
subgraph "proxy/auth/"
|
||||
Auth["user_api_key_auth()"]
|
||||
end
|
||||
|
||||
subgraph "proxy/"
|
||||
PreCall["litellm_pre_call_utils.py"]
|
||||
RouteRequest["route_llm_request.py"]
|
||||
end
|
||||
|
||||
subgraph "litellm/"
|
||||
Router["router.py"]
|
||||
Main["main.py"]
|
||||
end
|
||||
|
||||
subgraph "Infrastructure"
|
||||
DualCache["DualCache<br/>(in-memory + Redis)"]
|
||||
Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
|
||||
end
|
||||
|
||||
Client --> Endpoint
|
||||
Endpoint --> Auth
|
||||
Auth --> DualCache
|
||||
DualCache -.->|cache miss| Postgres
|
||||
Auth --> PreCall
|
||||
PreCall --> RouteRequest
|
||||
RouteRequest --> Router
|
||||
Router --> DualCache
|
||||
Router --> Main
|
||||
Main --> Client
|
||||
```
|
||||
|
||||
**Key proxy files:**
|
||||
- `proxy/proxy_server.py` - Main API endpoints
|
||||
- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
|
||||
- `proxy/hooks/` - Proxy-level callbacks
|
||||
- `router.py` - Load balancing, fallbacks
|
||||
- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
|
||||
|
||||
**LLM-specific proxy endpoints:**
|
||||
|
||||
| Endpoint | Directory | Purpose |
|
||||
|----------|-----------|---------|
|
||||
| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
|
||||
| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
|
||||
| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
|
||||
| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
|
||||
| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
|
||||
| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
|
||||
| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
|
||||
| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
|
||||
| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
|
||||
| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
|
||||
| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
|
||||
|
||||
**Proxy Hooks** (`proxy/hooks/__init__.py`):
|
||||
|
||||
| Hook | File | Purpose |
|
||||
|------|------|---------|
|
||||
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
|
||||
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
|
||||
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
|
||||
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
|
||||
| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
|
||||
|
||||
To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
|
||||
|
||||
### Infrastructure Components
|
||||
|
||||
The AI Gateway uses external infrastructure for persistence and caching:
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph "AI Gateway (proxy/)"
|
||||
Proxy["proxy_server.py"]
|
||||
Auth["auth/user_api_key_auth.py"]
|
||||
DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
|
||||
InternalCache["utils.py<br/>InternalUsageCache"]
|
||||
CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
|
||||
Scheduler["APScheduler<br/>ProxyStartupEvent"]
|
||||
end
|
||||
|
||||
subgraph "SDK (litellm/)"
|
||||
Router["router.py<br/>Router.cache (DualCache)"]
|
||||
LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
|
||||
CacheClass["caching/caching.py<br/>Cache"]
|
||||
end
|
||||
|
||||
subgraph "Redis (caching/redis_cache.py)"
|
||||
RateLimit["Rate Limit Counters"]
|
||||
SpendQueue["Spend Increment Queue"]
|
||||
KeyCache["API Key Cache"]
|
||||
TPM_RPM["TPM/RPM Tracking"]
|
||||
Cooldowns["Deployment Cooldowns"]
|
||||
LLMResponseCache["LLM Response Cache"]
|
||||
end
|
||||
|
||||
subgraph "PostgreSQL (proxy/schema.prisma)"
|
||||
Keys["LiteLLM_VerificationToken"]
|
||||
Teams["LiteLLM_TeamTable"]
|
||||
SpendLogs["LiteLLM_SpendLogs"]
|
||||
Users["LiteLLM_UserTable"]
|
||||
end
|
||||
|
||||
Auth --> InternalCache
|
||||
InternalCache --> KeyCache
|
||||
InternalCache -.->|cache miss| Keys
|
||||
InternalCache --> RateLimit
|
||||
Router --> TPM_RPM
|
||||
Router --> Cooldowns
|
||||
LLMCache --> CacheClass
|
||||
CacheClass --> LLMResponseCache
|
||||
CostCallback --> DBWriter
|
||||
DBWriter --> SpendQueue
|
||||
DBWriter --> SpendLogs
|
||||
Scheduler --> SpendLogs
|
||||
Scheduler --> Keys
|
||||
```
|
||||
|
||||
| Component | Purpose | Key Files/Classes |
|
||||
|-----------|---------|-------------------|
|
||||
| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
|
||||
| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
|
||||
| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
|
||||
| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
|
||||
| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
|
||||
| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
|
||||
| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
|
||||
|
||||
**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
|
||||
|
||||
| Job | Interval | Purpose | Key Files |
|
||||
|-----|----------|---------|-----------|
|
||||
| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
|
||||
| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
|
||||
| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
|
||||
| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
|
||||
| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
|
||||
| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
|
||||
| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
|
||||
| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
|
||||
| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
|
||||
| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
|
||||
|
||||
**Cost Attribution Flow:**
|
||||
1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
|
||||
2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
|
||||
3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
|
||||
4. Cost is stored in `response._hidden_params["response_cost"]`
|
||||
5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
|
||||
6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
|
||||
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
|
||||
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
|
||||
|
||||
---
|
||||
|
||||
## 2. SDK Request Flow
|
||||
|
||||
The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph "SDK Entry Points"
|
||||
Completion["litellm.completion()"]
|
||||
Messages["litellm.messages()"]
|
||||
end
|
||||
|
||||
subgraph "main.py"
|
||||
Main["completion()<br/>acompletion()"]
|
||||
end
|
||||
|
||||
subgraph "utils.py"
|
||||
GetProvider["get_llm_provider()"]
|
||||
end
|
||||
|
||||
subgraph "llms/custom_httpx/"
|
||||
Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
|
||||
HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
|
||||
end
|
||||
|
||||
subgraph "llms/{provider}/chat/"
|
||||
TransformReq["transform_request()"]
|
||||
TransformResp["transform_response()"]
|
||||
end
|
||||
|
||||
subgraph "litellm_core_utils/"
|
||||
Streaming["streaming_handler.py"]
|
||||
end
|
||||
|
||||
subgraph "integrations/ (async, off main thread)"
|
||||
Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
|
||||
end
|
||||
|
||||
Completion --> Main
|
||||
Messages --> Main
|
||||
Main --> GetProvider
|
||||
GetProvider --> Handler
|
||||
Handler --> TransformReq
|
||||
TransformReq --> HTTP
|
||||
HTTP --> Provider["LLM Provider API"]
|
||||
Provider --> HTTP
|
||||
HTTP --> TransformResp
|
||||
TransformResp --> Streaming
|
||||
Streaming --> Response["ModelResponse"]
|
||||
Response -.->|async| Callbacks
|
||||
```
|
||||
|
||||
**Key SDK files:**
|
||||
- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
|
||||
- `utils.py` - `get_llm_provider()` resolves model → provider
|
||||
- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
|
||||
- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
|
||||
- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
|
||||
- `litellm_core_utils/streaming_handler.py` - Streaming response handling
|
||||
- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
|
||||
|
||||
---
|
||||
|
||||
## 3. Translation Layer
|
||||
|
||||
When a request comes in, it goes through a **translation layer** that converts between API formats.
|
||||
Each translation is isolated in its own file, making it easy to test and modify independently.
|
||||
|
||||
### Where to find translations
|
||||
|
||||
| Incoming API | Provider | Translation File |
|
||||
|--------------|----------|------------------|
|
||||
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
|
||||
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
|
||||
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
|
||||
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
|
||||
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
|
||||
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
|
||||
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
|
||||
|
||||
### Example: Debugging prompt caching
|
||||
|
||||
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
|
||||
|
||||
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
|
||||
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
|
||||
3. Compare how each handles `cache_control` in `transform_request()`
|
||||
|
||||
### How translations work
|
||||
|
||||
Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
|
||||
|
||||
```python
|
||||
class ProviderConfig(BaseConfig):
|
||||
def transform_request(self, model, messages, optional_params, litellm_params, headers):
|
||||
# Convert OpenAI format → Provider format
|
||||
return {"messages": transformed_messages, ...}
|
||||
|
||||
def transform_response(self, model, raw_response, model_response, logging_obj, ...):
|
||||
# Convert Provider format → OpenAI format
|
||||
return ModelResponse(choices=[...], usage=Usage(...))
|
||||
```
|
||||
|
||||
The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
|
||||
|
||||
---
|
||||
|
||||
## 4. Adding/Modifying Providers
|
||||
|
||||
### To add a new provider:
|
||||
|
||||
1. Create `llms/{provider}/chat/transformation.py`
|
||||
2. Implement `Config` class with `transform_request()` and `transform_response()`
|
||||
3. Add tests in `tests/llm_translation/test_{provider}.py`
|
||||
|
||||
### To add a feature (e.g., prompt caching):
|
||||
|
||||
1. Find the translation file from the table above
|
||||
2. Modify `transform_request()` to handle the new parameter
|
||||
3. Add unit tests that verify the transformation
|
||||
|
||||
### Testing checklist
|
||||
|
||||
When adding a feature, verify it works across all paths:
|
||||
|
||||
| Test | File Pattern |
|
||||
|------|--------------|
|
||||
| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
|
||||
| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
|
||||
| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
|
||||
| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
|
||||
| Vertex AI | `tests/llm_translation/test_vertex*.py` |
|
||||
| Gemini | `tests/llm_translation/test_gemini*.py` |
|
||||
|
||||
### Unit testing translations
|
||||
|
||||
Translations are designed to be unit testable without making API calls:
|
||||
|
||||
```python
|
||||
from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
|
||||
|
||||
def test_prompt_caching_transform():
|
||||
config = BedrockConverseConfig()
|
||||
result = config.transform_request(
|
||||
model="anthropic.claude-3-opus",
|
||||
messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
assert "cachePoint" in str(result) # Verify cache_control was translated
|
||||
```
|
||||
11
Dockerfile
11
Dockerfile
|
|
@ -20,7 +20,8 @@ RUN python -m pip install build
|
|||
COPY . .
|
||||
|
||||
# Build Admin UI
|
||||
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Build the package
|
||||
RUN rm -rf dist/* && python -m build
|
||||
|
|
@ -65,12 +66,14 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
|
|||
find /usr/lib -type d -path "*/tornado/test" -delete
|
||||
|
||||
# Install semantic_router and aurelio-sdk using script
|
||||
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
|
||||
|
||||
# Generate prisma client
|
||||
RUN prisma generate
|
||||
RUN chmod +x docker/entrypoint.sh
|
||||
RUN chmod +x docker/prod_entrypoint.sh
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
|
||||
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
|
|
|
|||
3
Makefile
3
Makefile
|
|
@ -45,6 +45,7 @@ install-proxy-dev-ci:
|
|||
install-test-deps: install-proxy-dev
|
||||
poetry run pip install "pytest-retry==1.6.3"
|
||||
poetry run pip install pytest-xdist
|
||||
poetry run pip install openapi-core
|
||||
cd enterprise && poetry run pip install -e . && cd ..
|
||||
|
||||
install-helm-unittest:
|
||||
|
|
@ -100,4 +101,4 @@ test-llm-translation-single: install-test-deps
|
|||
@mkdir -p test-results
|
||||
poetry run pytest tests/llm_translation/$(FILE) \
|
||||
--junitxml=test-results/junit.xml \
|
||||
-v --tb=short --maxfail=100 --timeout=300
|
||||
-v --tb=short --maxfail=100 --timeout=300
|
||||
|
|
|
|||
496
README.md
496
README.md
|
|
@ -2,16 +2,16 @@
|
|||
🚅 LiteLLM
|
||||
</h1>
|
||||
<p align="center">
|
||||
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
|
||||
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
|
||||
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
|
||||
</a>
|
||||
</p>
|
||||
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
|
||||
<br>
|
||||
</p>
|
||||
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
|
||||
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
|
||||
<h4 align="center">
|
||||
<a href="https://pypi.org/project/litellm/" target="_blank">
|
||||
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
|
||||
|
|
@ -30,27 +30,17 @@
|
|||
</a>
|
||||
</h4>
|
||||
|
||||
LiteLLM manages:
|
||||
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
|
||||
|
||||
- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more)
|
||||
- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use
|
||||
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
|
||||
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
|
||||
|
||||
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
|
||||
## Use LiteLLM for
|
||||
|
||||
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
|
||||
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
|
||||
<details open>
|
||||
<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
|
||||
|
||||
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
|
||||
[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
|
||||
|
||||
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
|
||||
|
||||
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
|
||||
|
||||
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
|
||||
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
|
||||
</a>
|
||||
### Python SDK
|
||||
|
||||
```shell
|
||||
pip install litellm
|
||||
|
|
@ -60,309 +50,236 @@ pip install litellm
|
|||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
||||
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
|
||||
|
||||
messages = [{ "content": "Hello, how are you?","role": "user"}]
|
||||
|
||||
# openai call
|
||||
response = completion(model="openai/gpt-4o", messages=messages)
|
||||
|
||||
# anthropic call
|
||||
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Response (OpenAI Chat Completions Format)
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
|
||||
"created": 1751494488,
|
||||
"model": "claude-sonnet-4-20250514",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": null,
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
|
||||
"role": "assistant",
|
||||
"tool_calls": null,
|
||||
"function_call": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"completion_tokens": 39,
|
||||
"prompt_tokens": 13,
|
||||
"total_tokens": 52,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": {
|
||||
"audio_tokens": null,
|
||||
"cached_tokens": 0
|
||||
},
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Responses API ([Docs](https://docs.litellm.ai/docs/response_api))
|
||||
|
||||
LiteLLM also supports OpenAI's `/responses` format. Works with **all providers** - LiteLLM handles the translation automatically.
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# OpenAI
|
||||
response = litellm.responses(
|
||||
model="openai/gpt-4o",
|
||||
input="Hello, how are you?"
|
||||
)
|
||||
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
|
||||
|
||||
# Anthropic
|
||||
response = litellm.responses(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
input="Hello, how are you?"
|
||||
)
|
||||
|
||||
print(response)
|
||||
# Anthropic
|
||||
response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
|
||||
```
|
||||
|
||||
### Response (OpenAI Responses API Format)
|
||||
### AI Gateway (Proxy Server)
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "resp_abc123",
|
||||
"object": "response",
|
||||
"created_at": 1764682691,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini-2024-07-18",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_abc123",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Hello! I'm here and ready to help you. How can I assist you today?",
|
||||
"annotations": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 13,
|
||||
"output_tokens": 18,
|
||||
"total_tokens": 31
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
|
||||
|
||||
## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
|
||||
|
||||
```python
|
||||
from litellm import acompletion
|
||||
import asyncio
|
||||
|
||||
async def test_get_response():
|
||||
user_message = "Hello, how are you?"
|
||||
messages = [{"content": user_message, "role": "user"}]
|
||||
response = await acompletion(model="openai/gpt-4o", messages=messages)
|
||||
return response
|
||||
|
||||
response = asyncio.run(test_get_response())
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
|
||||
|
||||
LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
|
||||
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
messages = [{"content": "Hello, how are you?", "role": "user"}]
|
||||
|
||||
# gpt-4o
|
||||
response = completion(model="openai/gpt-4o", messages=messages, stream=True)
|
||||
for part in response:
|
||||
print(part.choices[0].delta.content or "")
|
||||
|
||||
# claude sonnet 4
|
||||
response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
|
||||
for part in response:
|
||||
print(part)
|
||||
```
|
||||
|
||||
### Response chunk (OpenAI Format)
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
|
||||
"created": 1751494808,
|
||||
"model": "claude-sonnet-4-20250514",
|
||||
"object": "chat.completion.chunk",
|
||||
"system_fingerprint": null,
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": null,
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"provider_specific_fields": null,
|
||||
"content": "Hello",
|
||||
"role": "assistant",
|
||||
"function_call": null,
|
||||
"tool_calls": null,
|
||||
"audio": null
|
||||
},
|
||||
"logprobs": null
|
||||
}
|
||||
],
|
||||
"provider_specific_fields": null,
|
||||
"stream_options": null,
|
||||
"citations": null
|
||||
}
|
||||
```
|
||||
|
||||
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
|
||||
|
||||
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
## set env variables for logging tools (when using MLflow, no API key set up is required)
|
||||
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
|
||||
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
|
||||
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
|
||||
os.environ["LANGFUSE_SECRET_KEY"] = ""
|
||||
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
|
||||
|
||||
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
||||
|
||||
# set callbacks
|
||||
litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
|
||||
|
||||
#openai call
|
||||
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
|
||||
```
|
||||
|
||||
# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
|
||||
|
||||
Track spend + Load Balance across multiple projects
|
||||
|
||||
[Hosted Proxy](https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy)
|
||||
|
||||
The proxy provides:
|
||||
|
||||
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
|
||||
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
|
||||
3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
|
||||
4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
|
||||
|
||||
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
|
||||
|
||||
|
||||
## Quick Start Proxy - CLI
|
||||
[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
|
||||
|
||||
```shell
|
||||
pip install 'litellm[proxy]'
|
||||
litellm --model gpt-4o
|
||||
```
|
||||
|
||||
### Step 1: Start litellm proxy
|
||||
|
||||
```shell
|
||||
$ litellm --model huggingface/bigcode/starcoder
|
||||
|
||||
#INFO: Proxy running on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
### Step 2: Make ChatCompletions Request to Proxy
|
||||
|
||||
|
||||
> [!IMPORTANT]
|
||||
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
|
||||
|
||||
```python
|
||||
import openai # openai v1.0.0+
|
||||
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
|
||||
# request sent to model set on litellm proxy, `litellm --model`
|
||||
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "this is a test request, write a short poem"
|
||||
}
|
||||
])
|
||||
import openai
|
||||
|
||||
print(response)
|
||||
client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
```
|
||||
|
||||
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
|
||||
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
|
||||
|
||||
Connect the proxy with a Postgres DB to create proxy keys
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
|
||||
|
||||
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
|
||||
|
||||
### Python SDK - A2A Protocol
|
||||
|
||||
```python
|
||||
from litellm.a2a_protocol import A2AClient
|
||||
from a2a.types import SendMessageRequest, MessageSendParams
|
||||
from uuid import uuid4
|
||||
|
||||
client = A2AClient(base_url="http://localhost:10001")
|
||||
|
||||
request = SendMessageRequest(
|
||||
id=str(uuid4()),
|
||||
params=MessageSendParams(
|
||||
message={
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "Hello!"}],
|
||||
"messageId": uuid4().hex,
|
||||
}
|
||||
)
|
||||
)
|
||||
response = await client.send_message(request)
|
||||
```
|
||||
|
||||
### AI Gateway (Proxy Server)
|
||||
|
||||
**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
|
||||
|
||||
**Step 2.** Call Agent via A2A SDK
|
||||
|
||||
```python
|
||||
from a2a.client import A2ACardResolver, A2AClient
|
||||
from a2a.types import MessageSendParams, SendMessageRequest
|
||||
from uuid import uuid4
|
||||
import httpx
|
||||
|
||||
base_url = "http://localhost:4000/a2a/my-agent" # LiteLLM proxy + agent name
|
||||
headers = {"Authorization": "Bearer sk-1234"} # LiteLLM Virtual Key
|
||||
|
||||
async with httpx.AsyncClient(headers=headers) as httpx_client:
|
||||
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
|
||||
agent_card = await resolver.get_agent_card()
|
||||
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
|
||||
|
||||
request = SendMessageRequest(
|
||||
id=str(uuid4()),
|
||||
params=MessageSendParams(
|
||||
message={
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "Hello!"}],
|
||||
"messageId": uuid4().hex,
|
||||
}
|
||||
)
|
||||
)
|
||||
response = await client.send_message(request)
|
||||
```
|
||||
|
||||
[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
|
||||
|
||||
### Python SDK - MCP Bridge
|
||||
|
||||
```python
|
||||
from mcp import ClientSession, StdioServerParameters
|
||||
from mcp.client.stdio import stdio_client
|
||||
from litellm import experimental_mcp_client
|
||||
import litellm
|
||||
|
||||
server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
|
||||
|
||||
async with stdio_client(server_params) as (read, write):
|
||||
async with ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
|
||||
# Load MCP tools in OpenAI format
|
||||
tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
|
||||
|
||||
# Use with any LiteLLM model
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "What's 3 + 5?"}],
|
||||
tools=tools
|
||||
)
|
||||
```
|
||||
|
||||
### AI Gateway - MCP Gateway
|
||||
|
||||
**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
|
||||
|
||||
**Step 2.** Call MCP tools via `/chat/completions`
|
||||
|
||||
```bash
|
||||
# Get the code
|
||||
git clone https://github.com/BerriAI/litellm
|
||||
|
||||
# Go to folder
|
||||
cd litellm
|
||||
|
||||
# Add the master key - you can change this after setup
|
||||
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
|
||||
|
||||
# Add the litellm salt key - you cannot change this after adding a model
|
||||
# It is used to encrypt / decrypt your LLM API Key credentials
|
||||
# We recommend - https://1password.com/password-generator/
|
||||
# password generator to get a random hash for litellm salt key
|
||||
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
|
||||
|
||||
# Start
|
||||
docker compose up
|
||||
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"messages": [{"role": "user", "content": "Summarize the latest open PR"}],
|
||||
"tools": [{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy/mcp/github",
|
||||
"server_label": "github_mcp",
|
||||
"require_approval": "never"
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
### Use with Cursor IDE
|
||||
|
||||
UI on `/ui` on your proxy server
|
||||
|
||||
|
||||
Set budgets and rate limits across multiple projects
|
||||
`POST /key/generate`
|
||||
|
||||
### Request
|
||||
|
||||
```shell
|
||||
curl 'http://0.0.0.0:4000/key/generate' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
|
||||
```
|
||||
|
||||
### Expected Response
|
||||
|
||||
```shell
|
||||
```json
|
||||
{
|
||||
"key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
|
||||
"expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
|
||||
"mcpServers": {
|
||||
"LiteLLM": {
|
||||
"url": "http://localhost:4000/mcp",
|
||||
"headers": {
|
||||
"x-litellm-api-key": "Bearer sk-1234"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
## How to use LiteLLM
|
||||
|
||||
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
|
||||
|
||||
<table style={{width: '100%', tableLayout: 'fixed'}}>
|
||||
<thead>
|
||||
<tr>
|
||||
<th style={{width: '14%'}}></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
|
||||
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style={{width: '14%'}}><strong>Use Case</strong></td>
|
||||
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
|
||||
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
|
||||
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
|
||||
<td style={{width: '43%'}}>Developers building LLM projects</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{width: '14%'}}><strong>Key Features</strong></td>
|
||||
<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
|
||||
<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
|
||||
|
||||
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
|
||||
[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
|
||||
|
||||
**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
|
||||
|
||||
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
|
||||
|
||||
## OSS Adopters
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
|
||||
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
|
||||
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
|
||||
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
|
||||
<td><h2>Netflix</h2></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
|
||||
|
||||
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|
||||
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
|
||||
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
|
||||
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
|
||||
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
|
||||
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
|
||||
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
@ -469,7 +386,9 @@ curl 'http://0.0.0.0:4000/key/generate' \
|
|||
1. (In root) create virtual environment `python -m venv .venv`
|
||||
2. Activate virtual environment `source .venv/bin/activate`
|
||||
3. Install dependencies `pip install -e ".[all]"`
|
||||
4. Start proxy backend `python litellm/proxy_cli.py`
|
||||
4. `pip install prisma`
|
||||
5. `prisma generate`
|
||||
6. Start proxy backend `python litellm/proxy/proxy_cli.py`
|
||||
|
||||
### Frontend
|
||||
1. Navigate to `ui/litellm-dashboard`
|
||||
|
|
@ -551,4 +470,3 @@ All these checks must pass before your PR can be merged.
|
|||
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
|
||||
</a>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
|
||||
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
|
||||
{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}
|
||||
|
||||
3
ci_cd/.grype.yaml
Normal file
3
ci_cd/.grype.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
ignore:
|
||||
- vulnerability: CVE-2026-22184
|
||||
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
|
||||
40
ci_cd/TEST_KEY_PATTERNS.md
Normal file
40
ci_cd/TEST_KEY_PATTERNS.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Test Key Patterns Standard
|
||||
|
||||
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
|
||||
|
||||
## How GitGuardian Works
|
||||
|
||||
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
|
||||
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
|
||||
- **High entropy** values (realistic-looking secrets) trigger detection
|
||||
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
|
||||
|
||||
## Recommended Test Key Patterns
|
||||
|
||||
### Option 1: Low Entropy Values (Simplest)
|
||||
These won't trigger GitGuardian's ML detector:
|
||||
|
||||
```python
|
||||
api_key = "sk-1234"
|
||||
api_key = "sk-12345"
|
||||
database_password = "postgres"
|
||||
token = "test123"
|
||||
```
|
||||
|
||||
### Option 2: High Entropy with Test Prefixes
|
||||
If you need realistic-looking test keys with high entropy, use these prefixes:
|
||||
|
||||
```python
|
||||
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
|
||||
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
|
||||
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
|
||||
token = "test-api-key-with-high-entropy"
|
||||
```
|
||||
|
||||
## Configured Ignore Patterns
|
||||
|
||||
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
|
||||
- `sk-test-*` - OpenAI-style test keys
|
||||
- `sk-mock-*` - Mock API keys
|
||||
- `sk-fake-*` - Fake API keys
|
||||
- `test-api-key` - Generic test tokens
|
||||
|
|
@ -34,47 +34,47 @@ install_ggshield() {
|
|||
echo "ggshield installed successfully"
|
||||
}
|
||||
|
||||
# Function to run secret detection scans
|
||||
run_secret_detection() {
|
||||
echo "Running secret detection scans..."
|
||||
# # Function to run secret detection scans
|
||||
# run_secret_detection() {
|
||||
# echo "Running secret detection scans..."
|
||||
|
||||
if ! command -v ggshield &> /dev/null; then
|
||||
install_ggshield
|
||||
fi
|
||||
# if ! command -v ggshield &> /dev/null; then
|
||||
# install_ggshield
|
||||
# fi
|
||||
|
||||
# Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
|
||||
if [ -z "$GITGUARDIAN_API_KEY" ]; then
|
||||
echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
|
||||
echo "ggshield requires a GitGuardian API key to scan for secrets."
|
||||
echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
|
||||
exit 1
|
||||
fi
|
||||
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
|
||||
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
|
||||
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
|
||||
# echo "ggshield requires a GitGuardian API key to scan for secrets."
|
||||
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
echo "Scanning codebase for secrets..."
|
||||
echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
|
||||
echo "ggshield will automatically handle rate limits and retry as needed."
|
||||
echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
|
||||
# echo "Scanning codebase for secrets..."
|
||||
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
|
||||
# echo "ggshield will automatically handle rate limits and retry as needed."
|
||||
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
|
||||
|
||||
# Use --recursive for directory scanning and auto-confirm if prompted
|
||||
# .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
|
||||
# GITGUARDIAN_API_KEY environment variable will be used for authentication
|
||||
echo y | ggshield secret scan path . --recursive || {
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "ERROR: Secret Detection Failed"
|
||||
echo "=========================================="
|
||||
echo "ggshield has detected secrets in the codebase."
|
||||
echo "Please review discovered secrets above, revoke any actively used secrets"
|
||||
echo "from underlying systems and make changes to inject secrets dynamically at runtime."
|
||||
echo ""
|
||||
echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
exit 1
|
||||
}
|
||||
# # Use --recursive for directory scanning and auto-confirm if prompted
|
||||
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
|
||||
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
|
||||
# echo y | ggshield secret scan path . --recursive || {
|
||||
# echo ""
|
||||
# echo "=========================================="
|
||||
# echo "ERROR: Secret Detection Failed"
|
||||
# echo "=========================================="
|
||||
# echo "ggshield has detected secrets in the codebase."
|
||||
# echo "Please review discovered secrets above, revoke any actively used secrets"
|
||||
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
|
||||
# echo ""
|
||||
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
|
||||
# echo "=========================================="
|
||||
# echo ""
|
||||
# exit 1
|
||||
# }
|
||||
|
||||
echo "Secret detection scans completed successfully"
|
||||
}
|
||||
# echo "Secret detection scans completed successfully"
|
||||
# }
|
||||
|
||||
# Function to run Trivy scans
|
||||
run_trivy_scans() {
|
||||
|
|
@ -101,12 +101,12 @@ run_grype_scans() {
|
|||
# Build and scan Dockerfile.database
|
||||
echo "Building and scanning Dockerfile.database..."
|
||||
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
|
||||
grype litellm-database:latest --fail-on critical
|
||||
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
|
||||
|
||||
# Build and scan main Dockerfile
|
||||
echo "Building and scanning main Dockerfile..."
|
||||
docker build --no-cache -t litellm:latest .
|
||||
grype litellm:latest --fail-on critical
|
||||
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
|
||||
|
||||
# Restore original .dockerignore
|
||||
echo "Restoring original .dockerignore..."
|
||||
|
|
@ -128,6 +128,31 @@ run_grype_scans() {
|
|||
"GHSA-5j98-mcp5-4vw2"
|
||||
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
|
||||
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
|
||||
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
|
||||
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
|
||||
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
|
||||
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
|
||||
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
|
||||
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
|
||||
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
|
||||
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
|
||||
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
|
||||
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
|
||||
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
|
||||
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
|
||||
"CVE-2025-15281" # No fix available yet
|
||||
"CVE-2026-0865" # No fix available yet
|
||||
"CVE-2025-15282" # No fix available yet
|
||||
"CVE-2026-0672" # No fix available yet
|
||||
"CVE-2025-15366" # No fix available yet
|
||||
"CVE-2025-15367" # No fix available yet
|
||||
"CVE-2025-12781" # No fix available yet
|
||||
"CVE-2025-11468" # No fix available yet
|
||||
)
|
||||
|
||||
# Build JSON array of allowlisted CVE IDs for jq
|
||||
|
|
@ -208,8 +233,8 @@ main() {
|
|||
install_trivy
|
||||
install_grype
|
||||
|
||||
echo "Running secret detection scans..."
|
||||
run_secret_detection
|
||||
# echo "Running secret detection scans..."
|
||||
# run_secret_detection
|
||||
|
||||
echo "Running filesystem vulnerability scans..."
|
||||
run_trivy_scans
|
||||
|
|
|
|||
2
cookbook/LiteLLM_PromptLayer.ipynb
vendored
2
cookbook/LiteLLM_PromptLayer.ipynb
vendored
|
|
@ -39,7 +39,7 @@
|
|||
"import os\n",
|
||||
"os.environ['OPENAI_API_KEY'] = \"\"\n",
|
||||
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
|
||||
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
|
||||
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
|
||||
"\n",
|
||||
"# Set Promptlayer as a success callback\n",
|
||||
"litellm.success_callback =['promptlayer']\n",
|
||||
|
|
|
|||
|
|
@ -1,21 +1,10 @@
|
|||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "kccfk0mHZ4Ad"
|
||||
},
|
||||
"source": [
|
||||
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
|
||||
"\n",
|
||||
|
|
@ -32,29 +21,26 @@
|
|||
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
|
||||
"\n",
|
||||
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "kccfk0mHZ4Ad"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "nmSClzCPaGH6"
|
||||
},
|
||||
"source": [
|
||||
"## /chat/completion\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "nmSClzCPaGH6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### OpenAI Python SDK"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "_vqcjwOVaKpO"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### OpenAI Python SDK"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -94,15 +80,20 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Function Calling"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "AqkyKk9Scxgj"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## Function Calling"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "wDg10VqLczE1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from openai import OpenAI\n",
|
||||
"client = OpenAI(\n",
|
||||
|
|
@ -139,24 +130,24 @@
|
|||
")\n",
|
||||
"\n",
|
||||
"print(completion)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "wDg10VqLczE1"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Azure OpenAI Python SDK"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "YYoxLloSaNWW"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Azure OpenAI Python SDK"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "yA1XcgowaSRy"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import openai\n",
|
||||
"client = openai.AzureOpenAI(\n",
|
||||
|
|
@ -184,24 +175,24 @@
|
|||
")\n",
|
||||
"\n",
|
||||
"print(response)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "yA1XcgowaSRy"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Langchain Python"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "yl9qhDvnaTpL"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Langchain Python"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "5MUZgSquaW5t"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.chat_models import ChatOpenAI\n",
|
||||
"from langchain.prompts.chat import (\n",
|
||||
|
|
@ -239,24 +230,22 @@
|
|||
"response = chat(messages)\n",
|
||||
"\n",
|
||||
"print(response)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "5MUZgSquaW5t"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Curl"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "B9eMgnULbRaz"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Curl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "VWCCk5PFcmhS"
|
||||
},
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
|
|
@ -280,22 +269,24 @@
|
|||
"}'\n",
|
||||
"```\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "VWCCk5PFcmhS"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### LlamaIndex"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "drBAm2e1b6xe"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### LlamaIndex"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d0bZcv8fb9mL"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os, dotenv\n",
|
||||
"\n",
|
||||
|
|
@ -326,24 +317,24 @@
|
|||
"query_engine = index.as_query_engine()\n",
|
||||
"response = query_engine.query(\"What did the author do growing up?\")\n",
|
||||
"print(response)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "d0bZcv8fb9mL"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Langchain JS"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "xypvNdHnb-Yy"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Langchain JS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "R55mK2vCcBN2"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import { ChatOpenAI } from \"@langchain/openai\";\n",
|
||||
"\n",
|
||||
|
|
@ -359,24 +350,24 @@
|
|||
"const message = await model.invoke(\"Hi there!\");\n",
|
||||
"\n",
|
||||
"console.log(message);\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "R55mK2vCcBN2"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### OpenAI JS"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "nC4bLifCcCiW"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### OpenAI JS"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "MICH8kIMcFpg"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"const { OpenAI } = require('openai');\n",
|
||||
"\n",
|
||||
|
|
@ -398,24 +389,24 @@
|
|||
"}\n",
|
||||
"\n",
|
||||
"main();\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "MICH8kIMcFpg"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Anthropic SDK"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "D1Q07pEAcGTb"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Anthropic SDK"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "qBjFcAvgcI3t"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
|
|
@ -423,7 +414,7 @@
|
|||
"\n",
|
||||
"client = Anthropic(\n",
|
||||
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
|
||||
" api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
|
||||
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"message = client.messages.create(\n",
|
||||
|
|
@ -437,33 +428,33 @@
|
|||
" model=\"claude-3-opus-20240229\",\n",
|
||||
")\n",
|
||||
"print(message.content)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "qBjFcAvgcI3t"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## /embeddings"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "dFAR4AJGcONI"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"## /embeddings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### OpenAI Python SDK"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "lgNoM281cRzR"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### OpenAI Python SDK"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "NY3DJhPfcQhA"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import openai\n",
|
||||
"from openai import OpenAI\n",
|
||||
|
|
@ -478,24 +469,24 @@
|
|||
")\n",
|
||||
"\n",
|
||||
"print(response)\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "NY3DJhPfcQhA"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Langchain Embeddings"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "hmbg-DW6cUZs"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Langchain Embeddings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "lX2S8Nl1cWVP"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.embeddings import OpenAIEmbeddings\n",
|
||||
"\n",
|
||||
|
|
@ -526,24 +517,22 @@
|
|||
"\n",
|
||||
"print(f\"TITAN EMBEDDINGS\")\n",
|
||||
"print(query_result[:5])"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "lX2S8Nl1cWVP"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"### Curl Request"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "oqGbWBCQcYfd"
|
||||
}
|
||||
},
|
||||
"source": [
|
||||
"### Curl Request"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "7rkIMV9LcdwQ"
|
||||
},
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
|
|
@ -556,10 +545,21 @@
|
|||
" }'\n",
|
||||
"```\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "7rkIMV9LcdwQ"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
|
|
|
|||
295
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
Normal file
295
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
# Claude Code with LiteLLM Quickstart
|
||||
|
||||
This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
|
||||
|
||||
> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
|
||||
|
||||
## Video Walkthrough
|
||||
|
||||
Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
|
||||
- API keys for your chosen providers
|
||||
|
||||
## Installation
|
||||
|
||||
First, install LiteLLM with proxy support:
|
||||
|
||||
```bash
|
||||
pip install 'litellm[proxy]'
|
||||
```
|
||||
|
||||
## Step 1: Setup config.yaml
|
||||
|
||||
Create a secure configuration using environment variables:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
# Claude models
|
||||
- model_name: claude-3-5-sonnet-20241022
|
||||
litellm_params:
|
||||
model: anthropic/claude-3-5-sonnet-20241022
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: claude-3-5-haiku-20241022
|
||||
litellm_params:
|
||||
model: anthropic/claude-3-5-haiku-20241022
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
|
||||
litellm_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
```
|
||||
|
||||
Set your environment variables:
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_API_KEY="your-anthropic-api-key"
|
||||
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
|
||||
```
|
||||
|
||||
## Step 2: Start Proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
## Step 3: Verify Setup
|
||||
|
||||
Test that your proxy is working correctly:
|
||||
|
||||
```bash
|
||||
curl -X POST http://0.0.0.0:4000/v1/messages \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"max_tokens": 1000,
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
## Step 4: Configure Claude Code
|
||||
|
||||
### Method 1: Unified Endpoint (Recommended)
|
||||
|
||||
Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
|
||||
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
||||
> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
|
||||
|
||||
### Method 2: Provider-specific Pass-through Endpoint
|
||||
|
||||
Alternatively, use the Anthropic pass-through endpoint:
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
|
||||
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
||||
## Step 5: Use Claude Code
|
||||
|
||||
### Choosing Your Model
|
||||
|
||||
You have two options for specifying which model Claude Code uses:
|
||||
|
||||
#### Option 1: Command Line / Session Model Selection
|
||||
|
||||
Specify the model directly when starting Claude Code or during a session:
|
||||
|
||||
```bash
|
||||
# Specify model at startup
|
||||
claude --model claude-3-5-sonnet-20241022
|
||||
|
||||
# Or change model during a session
|
||||
/model claude-3-5-haiku-20241022
|
||||
```
|
||||
|
||||
This method uses the exact model you specify.
|
||||
|
||||
#### Option 2: Environment Variables
|
||||
|
||||
Configure default models using environment variables:
|
||||
|
||||
```bash
|
||||
# Tell Claude Code which models to use by default
|
||||
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
|
||||
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
|
||||
export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
|
||||
|
||||
claude # Will use the models specified above
|
||||
```
|
||||
|
||||
**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
|
||||
|
||||
**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
|
||||
|
||||
### Using 1M Context Window
|
||||
|
||||
Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
|
||||
|
||||
```bash
|
||||
# Use Sonnet 4.5 with 1M context (requires quotes for shell)
|
||||
claude --model 'claude-sonnet-4-5-20250929[1m]'
|
||||
|
||||
# Inside a Claude Code session (no quotes needed)
|
||||
/model claude-sonnet-4-5-20250929[1m]
|
||||
```
|
||||
|
||||
**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
|
||||
|
||||
Alternatively, set as default with environment variables:
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
|
||||
claude
|
||||
```
|
||||
|
||||
**How it works:**
|
||||
- Claude Code strips the `[1m]` suffix before sending to LiteLLM
|
||||
- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
|
||||
- Your LiteLLM config should **NOT** include `[1m]` in model names
|
||||
|
||||
**Verify 1M context is active:**
|
||||
```bash
|
||||
/context
|
||||
# Should show: 21k/1000k tokens (2%)
|
||||
```
|
||||
|
||||
**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Common issues and solutions:
|
||||
|
||||
**Claude Code not connecting:**
|
||||
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
|
||||
- Check that `ANTHROPIC_BASE_URL` is set correctly
|
||||
- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
|
||||
|
||||
**Authentication errors:**
|
||||
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
|
||||
- Check that your API keys are valid and have sufficient credits
|
||||
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
|
||||
|
||||
**Model not found:**
|
||||
- Check what model Claude Code is requesting in LiteLLM logs
|
||||
- Ensure your `config.yaml` has a matching `model_name` entry
|
||||
- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
|
||||
|
||||
**1M context not working (showing 200k instead of 1000k):**
|
||||
- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
|
||||
- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
|
||||
- Ensure your model supports 1M context (only certain Claude models do)
|
||||
- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
|
||||
|
||||
## Using Multiple Models and Providers
|
||||
|
||||
You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
# OpenAI models
|
||||
- model_name: codex-mini
|
||||
litellm_params:
|
||||
model: openai/codex-mini
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
api_base: https://api.openai.com/v1
|
||||
|
||||
- model_name: o3-pro
|
||||
litellm_params:
|
||||
model: openai/o3-pro
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
api_base: https://api.openai.com/v1
|
||||
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
api_base: https://api.openai.com/v1
|
||||
|
||||
# Anthropic models
|
||||
- model_name: claude-3-5-sonnet-20241022
|
||||
litellm_params:
|
||||
model: anthropic/claude-3-5-sonnet-20241022
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: claude-3-5-haiku-20241022
|
||||
litellm_params:
|
||||
model: anthropic/claude-3-5-haiku-20241022
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
# AWS Bedrock
|
||||
- model_name: claude-bedrock
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: us-east-1
|
||||
|
||||
litellm_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
```
|
||||
|
||||
**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
|
||||
|
||||
Switch between models seamlessly:
|
||||
|
||||
```bash
|
||||
# Use environment variables to set defaults
|
||||
export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
|
||||
export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
|
||||
|
||||
# Or specify directly
|
||||
claude --model claude-3-5-sonnet-20241022 # Complex reasoning
|
||||
claude --model claude-3-5-haiku-20241022 # Fast responses
|
||||
claude --model claude-bedrock # Bedrock deployment
|
||||
```
|
||||
|
||||
## Default Models Used by Claude Code
|
||||
|
||||
If you **don't** set environment variables, Claude Code uses these default model names:
|
||||
|
||||
| Purpose | Default Model Name (v2.1.14) |
|
||||
|---------|------------------------------|
|
||||
| Main model | `claude-sonnet-4-5-20250929` |
|
||||
| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
|
||||
| Planning mode | `claude-opus-4-5-20251101` |
|
||||
|
||||
Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet-4-5-20250929
|
||||
litellm_params:
|
||||
# Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
|
||||
model: anthropic/claude-sonnet-4-5-20250929
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: claude-haiku-4-5-20251001
|
||||
litellm_params:
|
||||
model: anthropic/claude-haiku-4-5-20251001
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: claude-opus-4-5-20251101
|
||||
litellm_params:
|
||||
model: anthropic/claude-opus-4-5-20251101
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [LiteLLM Documentation](https://docs.litellm.ai/)
|
||||
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
|
||||
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
|
||||
|
||||
134
cookbook/ai_coding_tool_guides/index.json
Normal file
134
cookbook/ai_coding_tool_guides/index.json
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
[{
|
||||
"title": "Claude Code Quickstart",
|
||||
"description": "This is a quickstart guide to using Claude Code with LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
|
||||
"date": "2026-01-15",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"LiteLLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Claude Code with MCPs",
|
||||
"description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
|
||||
"date": "2026-01-15",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"LiteLLM",
|
||||
"MCP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Claude Code with Non-Anthropic Models",
|
||||
"description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"LiteLLM",
|
||||
"OpenAI",
|
||||
"Gemini"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Cursor Quickstart",
|
||||
"description": "This is a quickstart guide to using Cursor with LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Cursor",
|
||||
"LiteLLM",
|
||||
"Quickstart"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Github Copilot Quickstart",
|
||||
"description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Github Copilot",
|
||||
"LiteLLM",
|
||||
"Quickstart"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "LiteLLM Gemini CLI Quickstart",
|
||||
"description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Gemini CLI",
|
||||
"Gemini",
|
||||
"LiteLLM",
|
||||
"Quickstart"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "OpenAI Codex CLI Quickstart",
|
||||
"description": "This is a quickstart guide to using OpenAI Codex CLI.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"OpenAI Codex CLI",
|
||||
"OpenAI",
|
||||
"LiteLLM",
|
||||
"Quickstart"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "OpenWebUI Quickstart",
|
||||
"description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
|
||||
"date": "2026-01-16",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"OpenWebUI",
|
||||
"LiteLLM",
|
||||
"Quickstart"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "AI Coding Tool Usage Tracking",
|
||||
"description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
|
||||
"date": "2026-01-17",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"Gemini CLI",
|
||||
"OpenAI Codex",
|
||||
"LiteLLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
|
||||
"description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
|
||||
"date": "2026-01-17",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"LiteLLM",
|
||||
"Web Search"
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Track Claude Code Usage per user via Custom Headers",
|
||||
"description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
|
||||
"url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
|
||||
"date": "2026-01-17",
|
||||
"version": "1.0.0",
|
||||
"tags": [
|
||||
"Claude Code",
|
||||
"LiteLLM"
|
||||
]
|
||||
}]
|
||||
|
|
@ -8,7 +8,8 @@ WORKDIR /app
|
|||
COPY config.yaml .
|
||||
|
||||
# Make sure your docker/entrypoint.sh is executable
|
||||
RUN chmod +x docker/entrypoint.sh
|
||||
# Convert Windows line endings to Unix
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
|
||||
|
||||
# Expose the necessary port
|
||||
EXPOSE 4000/tcp
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ type: application
|
|||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.10
|
||||
version: 1.1.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: v1.50.2
|
||||
appVersion: v1.80.12
|
||||
|
||||
dependencies:
|
||||
- name: "postgresql"
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ metadata:
|
|||
{{- toYaml .Values.deploymentLabels | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if not .Values.autoscaling.enabled }}
|
||||
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
{{- end }}
|
||||
selector:
|
||||
|
|
@ -170,7 +170,8 @@ spec:
|
|||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: litellm-config
|
||||
mountPath: /etc/litellm/
|
||||
mountPath: /etc/litellm/config.yaml
|
||||
subPath: config.yaml
|
||||
{{ if .Values.securityContext.readOnlyRootFilesystem }}
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
|
|
@ -182,6 +183,10 @@ spec:
|
|||
{{- with .Values.volumeMounts }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.lifecycle }}
|
||||
lifecycle:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.extraContainers }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
|
|
|||
37
deploy/charts/litellm-helm/templates/keda.yaml
Normal file
37
deploy/charts/litellm-helm/templates/keda.yaml
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
|
||||
apiVersion: keda.sh/v1alpha1
|
||||
kind: ScaledObject
|
||||
metadata:
|
||||
name: {{ include "litellm.fullname" . }}
|
||||
labels:
|
||||
{{- include "litellm.labels" . | nindent 4 }}
|
||||
{{- if .Values.keda.scaledObject.annotations }}
|
||||
annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
name: {{ include "litellm.fullname" . }}
|
||||
pollingInterval: {{ .Values.keda.pollingInterval }}
|
||||
cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
|
||||
minReplicaCount: {{ .Values.keda.minReplicas }}
|
||||
maxReplicaCount: {{ .Values.keda.maxReplicas }}
|
||||
{{- with .Values.keda.fallback }}
|
||||
fallback:
|
||||
failureThreshold: {{ .failureThreshold | default 3 }}
|
||||
replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
|
||||
{{- end }}
|
||||
triggers:
|
||||
{{- with .Values.keda.triggers }}
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
advanced:
|
||||
restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
|
||||
{{- if .Values.keda.behavior }}
|
||||
horizontalPodAutoscalerConfig:
|
||||
behavior:
|
||||
{{- with .Values.keda.behavior }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -136,4 +136,27 @@ tests:
|
|||
path: spec.template.spec.containers[0].volumeMounts
|
||||
content:
|
||||
name: litellm-config
|
||||
mountPath: /etc/litellm/
|
||||
mountPath: /etc/litellm/config.yaml
|
||||
subPath: config.yaml
|
||||
- it: should work with lifecycle hooks
|
||||
template: deployment.yaml
|
||||
set:
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- echo "Container stopping"
|
||||
asserts:
|
||||
- exists:
|
||||
path: spec.template.spec.containers[0].lifecycle
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
|
||||
value: /bin/sh
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
|
||||
value: -c
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
|
||||
value: echo "Container stopping"
|
||||
|
|
@ -156,6 +156,40 @@ autoscaling:
|
|||
targetCPUUtilizationPercentage: 80
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
|
||||
# Autoscaling with keda is mutually exclusive with hpa
|
||||
keda:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
pollingInterval: 30
|
||||
cooldownPeriod: 300
|
||||
# fallback:
|
||||
# failureThreshold: 3
|
||||
# replicas: 11
|
||||
restoreToOriginalReplicaCount: false
|
||||
scaledObject:
|
||||
annotations: {}
|
||||
triggers: []
|
||||
# - type: prometheus
|
||||
# metadata:
|
||||
# serverAddress: http://<prometheus-host>:9090
|
||||
# metricName: http_requests_total
|
||||
# threshold: '100'
|
||||
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
|
||||
behavior: {}
|
||||
# scaleDown:
|
||||
# stabilizationWindowSeconds: 300
|
||||
# policies:
|
||||
# - type: Pods
|
||||
# value: 1
|
||||
# periodSeconds: 180
|
||||
# scaleUp:
|
||||
# stabilizationWindowSeconds: 300
|
||||
# policies:
|
||||
# - type: Pods
|
||||
# value: 2
|
||||
# periodSeconds: 60
|
||||
|
||||
# Additional volumes on the output Deployment definition.
|
||||
volumes: []
|
||||
# - name: foo
|
||||
|
|
@ -200,6 +234,14 @@ db:
|
|||
# instance. See the "postgresql" top level key for additional configuration.
|
||||
deployStandalone: true
|
||||
|
||||
# Lifecycle hooks for the LiteLLM container
|
||||
# Example:
|
||||
# lifecycle:
|
||||
# preStop:
|
||||
# exec:
|
||||
# command: ["/bin/sh", "-c", "sleep 10"]
|
||||
lifecycle: {}
|
||||
|
||||
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
|
||||
# otherwise)
|
||||
postgresql:
|
||||
|
|
|
|||
|
|
@ -46,8 +46,9 @@ COPY --from=builder /wheels/ /wheels/
|
|||
# Install the built wheel using pip; again using a wildcard if it's the only file
|
||||
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
|
||||
|
||||
RUN chmod +x docker/entrypoint.sh
|
||||
RUN chmod +x docker/prod_entrypoint.sh
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
|
||||
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
|
|
|
|||
|
|
@ -32,8 +32,9 @@ RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
|
|||
WORKDIR /app
|
||||
|
||||
# Make sure your docker/entrypoint.sh is executable
|
||||
RUN chmod +x docker/entrypoint.sh
|
||||
RUN chmod +x docker/prod_entrypoint.sh
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
|
||||
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
# Expose the necessary port
|
||||
EXPOSE 4000/tcp
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ RUN python -m pip install build
|
|||
COPY . .
|
||||
|
||||
# Build Admin UI
|
||||
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Build the package
|
||||
RUN rm -rf dist/* && python -m build
|
||||
|
|
@ -48,7 +49,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
|
||||
|
||||
WORKDIR /app
|
||||
# Copy the current directory contents into the container at /app
|
||||
|
|
@ -63,20 +64,23 @@ COPY --from=builder /wheels/ /wheels/
|
|||
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
|
||||
|
||||
# Install semantic_router and aurelio-sdk using script
|
||||
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
|
||||
|
||||
# ensure pyjwt is used, not jwt
|
||||
RUN pip uninstall jwt -y
|
||||
RUN pip uninstall PyJWT -y
|
||||
RUN pip install PyJWT==2.9.0 --no-cache-dir
|
||||
|
||||
# Build Admin UI
|
||||
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
# Build Admin UI (runtime stage)
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Generate prisma client
|
||||
RUN prisma generate
|
||||
RUN chmod +x docker/entrypoint.sh
|
||||
RUN chmod +x docker/prod_entrypoint.sh
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
|
||||
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
RUN apk add --no-cache supervisor
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ COPY enterprise/ ./enterprise/
|
|||
COPY docker/ ./docker/
|
||||
|
||||
# Build Admin UI once
|
||||
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
# Convert Windows line endings to Unix and make executable
|
||||
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
||||
|
||||
# Build the package
|
||||
RUN rm -rf dist/* && python -m build
|
||||
|
|
@ -79,8 +80,12 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
|
|||
rm -rf /wheels
|
||||
|
||||
# Generate prisma client and set permissions
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN prisma generate && \
|
||||
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
|
||||
sed -i 's/\r$//' docker/entrypoint.sh && \
|
||||
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
|
||||
chmod +x docker/entrypoint.sh && \
|
||||
chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
|
|
|
|||
16
docker/Dockerfile.health_check
Normal file
16
docker/Dockerfile.health_check
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy health check script and requirements
|
||||
COPY scripts/health_check/health_check_client.py /app/health_check_client.py
|
||||
COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt
|
||||
|
||||
# Install dependencies
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Make script executable
|
||||
RUN chmod +x /app/health_check_client.py
|
||||
|
||||
# Set entrypoint
|
||||
ENTRYPOINT ["python", "/app/health_check_client.py"]
|
||||
|
|
@ -15,6 +15,7 @@ USER root
|
|||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache \
|
||||
python3 \
|
||||
python3-dev \
|
||||
py3-pip \
|
||||
clang \
|
||||
llvm \
|
||||
|
|
@ -40,7 +41,7 @@ COPY . .
|
|||
ENV LITELLM_NON_ROOT=true
|
||||
|
||||
# Build Admin UI using the upstream command order while keeping a single RUN layer
|
||||
RUN mkdir -p /tmp/litellm_ui && \
|
||||
RUN mkdir -p /var/lib/litellm/ui && \
|
||||
npm install -g npm@latest && npm cache clean --force && \
|
||||
cd /app/ui/litellm-dashboard && \
|
||||
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
|
||||
|
|
@ -49,10 +50,10 @@ RUN mkdir -p /tmp/litellm_ui && \
|
|||
rm -f package-lock.json && \
|
||||
npm install --legacy-peer-deps && \
|
||||
npm run build && \
|
||||
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
|
||||
mkdir -p /tmp/litellm_assets && \
|
||||
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
|
||||
( cd /tmp/litellm_ui && \
|
||||
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
|
||||
mkdir -p /var/lib/litellm/assets && \
|
||||
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
|
||||
( cd /var/lib/litellm/ui && \
|
||||
for html_file in *.html; do \
|
||||
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
|
||||
folder_name="${html_file%.html}" && \
|
||||
|
|
@ -79,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
|
|||
XDG_CACHE_HOME=/app/.cache \
|
||||
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
|
||||
|
||||
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \
|
||||
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
|
||||
&& mkdir -p /app/.cache/npm
|
||||
|
||||
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
|
||||
|
|
@ -110,9 +111,11 @@ COPY --from=builder /app/requirements.txt /app/requirements.txt
|
|||
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
|
||||
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
|
||||
COPY --from=builder /app/schema.prisma /app/
|
||||
# Copy prisma_migration.py for Helm migrations job compatibility
|
||||
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
|
||||
COPY --from=builder /wheels/ /wheels/
|
||||
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
|
||||
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
|
||||
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
|
||||
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
|
||||
COPY --from=builder /app/.cache /app/.cache
|
||||
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
|
||||
COPY --from=builder \
|
||||
|
|
@ -144,9 +147,12 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
|
|||
fi
|
||||
|
||||
# Permissions, cleanup, and Prisma prep
|
||||
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
|
||||
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
|
||||
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
|
||||
# Convert Windows line endings to Unix for entrypoint scripts
|
||||
RUN sed -i 's/\r$//' docker/entrypoint.sh && \
|
||||
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
|
||||
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
|
||||
mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
|
||||
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
|
||||
pip uninstall jwt -y || true && \
|
||||
pip uninstall PyJWT -y || true && \
|
||||
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
|
||||
|
|
@ -156,11 +162,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
|
|||
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
|
||||
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
|
||||
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
|
||||
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
|
||||
chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
|
||||
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
|
||||
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
|
||||
chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
|
||||
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
|
||||
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
|
||||
chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
|
||||
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
|
||||
chmod -R g+rX $PRISMA_PATH && \
|
||||
chmod -R g+rX /app/.cache && \
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
|
||||
export LITELLM_ARGS="$@"
|
||||
export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
|
||||
exec supervisord -c /etc/supervisord.conf
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
[supervisord]
|
||||
nodaemon=true
|
||||
loglevel=info
|
||||
logfile=/tmp/supervisord.log
|
||||
pidfile=/tmp/supervisord.pid
|
||||
|
||||
[group:litellm]
|
||||
programs=main,health
|
||||
|
|
@ -14,6 +16,7 @@ priority=1
|
|||
exitcodes=0
|
||||
stopasgroup=true
|
||||
killasgroup=true
|
||||
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
|
||||
stdout_logfile=/dev/stdout
|
||||
stderr_logfile=/dev/stderr
|
||||
stdout_logfile_maxbytes = 0
|
||||
|
|
@ -29,6 +32,7 @@ priority=2
|
|||
exitcodes=0
|
||||
stopasgroup=true
|
||||
killasgroup=true
|
||||
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
|
||||
stdout_logfile=/dev/stdout
|
||||
stderr_logfile=/dev/stderr
|
||||
stdout_logfile_maxbytes = 0
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ import TabItem from '@theme/TabItem';
|
|||
|
||||
LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
|
||||
|
||||
:::note
|
||||
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
|
||||
:::
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
|
|
@ -232,6 +236,11 @@ response = completion(
|
|||
print(response)
|
||||
```
|
||||
|
||||
:::note
|
||||
If using this model via vertex_ai, keep the location as global as this is the only supported location as of now.
|
||||
:::
|
||||
|
||||
|
||||
## `reasoning_effort` Mapping for Gemini 3+
|
||||
|
||||
| reasoning_effort | thinking_level |
|
||||
|
|
|
|||
|
|
@ -237,6 +237,27 @@ litellm_settings:
|
|||
language: "en"
|
||||
```
|
||||
|
||||
### Example: Pillar Security
|
||||
|
||||
[Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "pillar-security"
|
||||
litellm_params:
|
||||
guardrail: generic_guardrail_api
|
||||
mode: [pre_call, post_call]
|
||||
api_base: https://api.pillar.security/api/v1/integrations/litellm
|
||||
api_key: os.environ/PILLAR_API_KEY
|
||||
default_on: true
|
||||
additional_provider_specific_params:
|
||||
plr_mask: true # Enable automatic masking of sensitive data
|
||||
plr_evidence: true # Include detection evidence in response
|
||||
plr_scanners: true # Include scanner details in response
|
||||
```
|
||||
|
||||
See the [Pillar Security documentation](../proxy/guardrails/pillar_security.md) for full configuration options.
|
||||
|
||||
## Usage
|
||||
|
||||
Users apply your guardrail by name:
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ model_list:
|
|||
model: vertex_ai/claude-3-5-sonnet-v2@20241022
|
||||
vertex_project: my-project
|
||||
vertex_location: us-east5
|
||||
vertex_count_tokens_location: us-east5 # Optional: Override location for token counting (count_tokens not available on global location)
|
||||
|
||||
- model_name: claude-bedrock
|
||||
litellm_params:
|
||||
|
|
|
|||
294
docs/my-website/docs/anthropic_unified/structured_output.md
Normal file
294
docs/my-website/docs/anthropic_unified/structured_output.md
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Structured Output /v1/messages
|
||||
|
||||
Use LiteLLM to call Anthropic's structured output feature via the `/v1/messages` endpoint.
|
||||
|
||||
## Supported Providers
|
||||
|
||||
| Provider | Supported | Notes |
|
||||
|----------|-----------|-------|
|
||||
| Anthropic | ✅ | Native support |
|
||||
| Azure AI (Anthropic models) | ✅ | Claude models on Azure AI |
|
||||
| Bedrock (Converse Anthropic models) | ✅ | Claude models via Bedrock Converse API |
|
||||
| Bedrock (Invoke Anthropic models) | ✅ | Claude models via Bedrock Invoke API |
|
||||
|
||||
## Usage
|
||||
|
||||
### LiteLLM Proxy Server
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="anthropic" label="Anthropic">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5-20250514
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/messages \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-d '{
|
||||
"model": "claude-sonnet",
|
||||
"max_tokens": 1024,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
|
||||
}
|
||||
],
|
||||
"output_format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"}
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="azure_ai" label="Azure AI (Anthropic)">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: azure-claude-sonnet
|
||||
litellm_params:
|
||||
model: azure_ai/claude-sonnet-4-5-20250514
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_base: https://your-endpoint.inference.ai.azure.com
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/messages \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-d '{
|
||||
"model": "azure-claude-sonnet",
|
||||
"max_tokens": 1024,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
|
||||
}
|
||||
],
|
||||
"output_format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"}
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="bedrock" label="Bedrock (Converse)">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-claude-sonnet
|
||||
litellm_params:
|
||||
model: bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: us-west-2
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/messages \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-d '{
|
||||
"model": "bedrock-claude-sonnet",
|
||||
"max_tokens": 1024,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
|
||||
}
|
||||
],
|
||||
"output_format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"}
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="bedrock_invoke" label="Bedrock (Invoke)">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-claude-invoke
|
||||
litellm_params:
|
||||
model: bedrock/invoke/global.anthropic.claude-sonnet-4-5-20250929-v1:0
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: us-west-2
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/messages \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-d '{
|
||||
"model": "bedrock-claude-invoke",
|
||||
"max_tokens": 1024,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email: John Smith (john@example.com) is interested in our Enterprise plan and wants to schedule a demo for next Tuesday at 2pm."
|
||||
}
|
||||
],
|
||||
"output_format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"}
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Example Response
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "msg_01XFDUDYJgAACzvnptvVoYEL",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"name\":\"John Smith\",\"email\":\"john@example.com\",\"plan_interest\":\"Enterprise\",\"demo_requested\":true}"
|
||||
}
|
||||
],
|
||||
"model": "claude-sonnet-4-5-20250514",
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": null,
|
||||
"usage": {
|
||||
"input_tokens": 75,
|
||||
"output_tokens": 28
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Request Format
|
||||
|
||||
### output_format
|
||||
|
||||
The `output_format` parameter specifies the structured output format.
|
||||
|
||||
```json
|
||||
{
|
||||
"output_format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"field_name": {"type": "string"},
|
||||
"another_field": {"type": "integer"}
|
||||
},
|
||||
"required": ["field_name", "another_field"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Fields
|
||||
|
||||
- **type** (string): Must be `"json_schema"`
|
||||
- **schema** (object): A JSON Schema object defining the expected output structure
|
||||
- **type** (string): The root type, typically `"object"`
|
||||
- **properties** (object): Defines the fields and their types
|
||||
- **required** (array): List of required field names
|
||||
- **additionalProperties** (boolean): Set to `false` to enforce strict schema adherence
|
||||
|
|
@ -60,6 +60,58 @@ Each machine deploying LiteLLM had the following specs:
|
|||
- Database: PostgreSQL
|
||||
- Redis: Not used
|
||||
|
||||
## Infrastructure Recommendations
|
||||
|
||||
Recommended specifications based on benchmark results and industry standards for API gateway deployments.
|
||||
|
||||
### PostgreSQL
|
||||
|
||||
Required for authentication, key management, and usage tracking.
|
||||
|
||||
| Workload | CPU | RAM | Storage | Connections |
|
||||
|----------|-----|-----|---------|-------------|
|
||||
| 1-2K RPS | 4-8 cores | 16GB | 200GB SSD (3000+ IOPS) | 100-200 |
|
||||
| 2-5K RPS | 8 cores | 16-32GB | 500GB SSD (5000+ IOPS) | 200-500 |
|
||||
| 5K+ RPS | 16+ cores | 32-64GB | 1TB+ SSD (10000+ IOPS) | 500+ |
|
||||
|
||||
**Configuration:** Set `proxy_batch_write_at: 60` to batch writes and reduce DB load. Total connections = pool limit × instances.
|
||||
|
||||
### Redis (Recommended)
|
||||
|
||||
Redis was not used in these benchmarks but provides significant production benefits: 60-80% reduced DB load.
|
||||
|
||||
| Workload | CPU | RAM |
|
||||
|----------|-----|-----|
|
||||
| 1-2K RPS | 2-4 cores | 8GB |
|
||||
| 2-5K RPS | 4 cores | 16GB |
|
||||
| 5K+ RPS | 8+ cores | 32GB+ |
|
||||
|
||||
**Requirements:** Redis 7.0+, AOF persistence enabled, `allkeys-lru` eviction policy.
|
||||
|
||||
**Configuration:**
|
||||
```yaml
|
||||
router_settings:
|
||||
redis_host: os.environ/REDIS_HOST
|
||||
redis_port: os.environ/REDIS_PORT
|
||||
redis_password: os.environ/REDIS_PASSWORD
|
||||
|
||||
litellm_settings:
|
||||
cache: True
|
||||
cache_params:
|
||||
type: redis
|
||||
host: os.environ/REDIS_HOST
|
||||
port: os.environ/REDIS_PORT
|
||||
password: os.environ/REDIS_PASSWORD
|
||||
```
|
||||
|
||||
:::tip
|
||||
Use `redis_host`, `redis_port`, and `redis_password` instead of `redis_url` for ~80 RPS better performance.
|
||||
:::
|
||||
|
||||
**Scaling:** DB connections scale linearly with instances. Consider PostgreSQL read replicas beyond 5K RPS.
|
||||
|
||||
See [Production Configuration](./proxy/prod) for detailed best practices.
|
||||
|
||||
## Locust Settings
|
||||
|
||||
- 1000 Users
|
||||
|
|
|
|||
|
|
@ -105,6 +105,14 @@ Then simply initialize:
|
|||
litellm.cache = Cache(type="redis")
|
||||
```
|
||||
|
||||
:::info
|
||||
Use `REDIS_*` environment variables as the primary mechanism for configuring all Redis client library parameters. This approach automatically maps environment variables to Redis client kwargs and is the suggested way to toggle Redis settings.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
If you need to pass non-string Redis parameters (integers, booleans, complex objects), avoid `REDIS_*` environment variables as they may fail during Redis client initialization. Instead, pass them directly as kwargs to the `Cache()` constructor.
|
||||
:::
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="gcs" label="gcs-cache">
|
||||
|
|
|
|||
|
|
@ -142,7 +142,47 @@ def completion(
|
|||
- `tool_call_id`: *str (optional)* - Tool call that this message is responding to.
|
||||
|
||||
|
||||
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392)
|
||||
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664)
|
||||
|
||||
#### Content Types
|
||||
|
||||
`content` can be a string (text only) or a list of content blocks (multimodal):
|
||||
|
||||
| Type | Description | Docs |
|
||||
|------|-------------|------|
|
||||
| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) |
|
||||
| `image_url` | Images | [Vision](./vision.md) |
|
||||
| `input_audio` | Audio input | [Audio](./audio.md) |
|
||||
| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) |
|
||||
| `file` | Files | [Document Understanding](./document_understanding.md) |
|
||||
| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) |
|
||||
|
||||
**Examples:**
|
||||
```python
|
||||
# Text
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}]
|
||||
|
||||
# Image
|
||||
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}]
|
||||
|
||||
# Audio
|
||||
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "<base64>", "format": "wav"}}]}]
|
||||
|
||||
# Video
|
||||
messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}]
|
||||
|
||||
# File
|
||||
messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}]
|
||||
|
||||
# Document
|
||||
messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": "<base64>"}}]}]
|
||||
|
||||
# Combining multiple types (multimodal)
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": "Generate a product description based on this image"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
|
||||
]}]
|
||||
```
|
||||
|
||||
## Optional Fields
|
||||
|
||||
|
|
@ -159,6 +199,8 @@ def completion(
|
|||
- `include_usage` *boolean (optional)* - If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value.
|
||||
|
||||
- `stop`: *string/ array/ null (optional)* - Up to 4 sequences where the API will stop generating further tokens.
|
||||
|
||||
**Note**: OpenAI supports a maximum of 4 stop sequences. If you provide more than 4, LiteLLM will automatically truncate the list to the first 4 elements. To disable this automatic truncation, set `litellm.disable_stop_sequence_limit = True`.
|
||||
|
||||
- `max_completion_tokens`: *integer (optional)* - An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
|
||||
|
||||
|
|
|
|||
|
|
@ -341,4 +341,90 @@ curl http://0.0.0.0:4000/v1/chat/completions \
|
|||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
</Tabs>
|
||||
|
||||
## Gemini - Native JSON Schema Format (Gemini 2.0+)
|
||||
|
||||
Gemini 2.0+ models automatically use the native `responseJsonSchema` parameter, which provides better compatibility with standard JSON Schema format.
|
||||
|
||||
### Benefits (Gemini 2.0+):
|
||||
- Standard JSON Schema format (lowercase types like `string`, `object`)
|
||||
- Supports `additionalProperties: false` for stricter validation
|
||||
- Better compatibility with Pydantic's `model_json_schema()`
|
||||
- No `propertyOrdering` required
|
||||
|
||||
### Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
from pydantic import BaseModel
|
||||
|
||||
class UserInfo(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
|
||||
response = completion(
|
||||
model="gemini/gemini-2.0-flash",
|
||||
messages=[{"role": "user", "content": "Extract: John is 25 years old"}],
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "user_info",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"}
|
||||
},
|
||||
"required": ["name", "age"],
|
||||
"additionalProperties": False # Supported on Gemini 2.0+
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-d '{
|
||||
"model": "gemini-2.0-flash",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Extract: John is 25 years old"}
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "user_info",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"}
|
||||
},
|
||||
"required": ["name", "age"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Model Behavior
|
||||
|
||||
| Model | Format Used | `additionalProperties` Support |
|
||||
|-------|-------------|-------------------------------|
|
||||
| Gemini 2.0+ | `responseJsonSchema` (JSON Schema) | ✅ Yes |
|
||||
| Gemini 1.5 | `responseSchema` (OpenAPI) | ❌ No |
|
||||
|
||||
LiteLLM automatically selects the appropriate format based on the model version.
|
||||
|
|
@ -100,7 +100,7 @@ from litellm import cost_per_token
|
|||
|
||||
prompt_tokens = 5
|
||||
completion_tokens = 10
|
||||
prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
|
||||
prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
|
||||
|
||||
print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
|
||||
```
|
||||
|
|
@ -162,7 +162,7 @@ print(model_cost) # {'gpt-3.5-turbo': {'max_tokens': 4000, 'input_cost_per_token
|
|||
|
||||
**Dictionary**
|
||||
```python
|
||||
from litellm import register_model
|
||||
import litellm
|
||||
|
||||
litellm.register_model({
|
||||
"gpt-4": {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
|
|||
|
||||
| Endpoint | Method | Description |
|
||||
|----------|--------|-------------|
|
||||
| `/v1/containers/{container_id}/files` | POST | Upload file to container |
|
||||
| `/v1/containers/{container_id}/files` | GET | List files in container |
|
||||
| `/v1/containers/{container_id}/files/{file_id}` | GET | Get file metadata |
|
||||
| `/v1/containers/{container_id}/files/{file_id}/content` | GET | Download file content |
|
||||
|
|
@ -28,6 +29,45 @@ Looking for how to use Code Interpreter? See the [Code Interpreter Guide](/docs/
|
|||
|
||||
## LiteLLM Python SDK
|
||||
|
||||
### Upload Container File
|
||||
|
||||
Upload files directly to a container session. This is useful when `/chat/completions` or `/responses` sends files to the container but the input file type is limited to PDF. This endpoint lets you work with other file types like CSV, Excel, Python scripts, etc.
|
||||
|
||||
```python showLineNumbers title="upload_container_file.py"
|
||||
from litellm import upload_container_file
|
||||
|
||||
# Upload a CSV file
|
||||
file = upload_container_file(
|
||||
container_id="cntr_123...",
|
||||
file=("data.csv", open("data.csv", "rb").read(), "text/csv"),
|
||||
custom_llm_provider="openai"
|
||||
)
|
||||
|
||||
print(f"Uploaded: {file.id}")
|
||||
print(f"Path: {file.path}")
|
||||
```
|
||||
|
||||
**Async:**
|
||||
|
||||
```python showLineNumbers title="aupload_container_file.py"
|
||||
from litellm import aupload_container_file
|
||||
|
||||
file = await aupload_container_file(
|
||||
container_id="cntr_123...",
|
||||
file=("script.py", b"print('hello world')", "text/x-python"),
|
||||
custom_llm_provider="openai"
|
||||
)
|
||||
```
|
||||
|
||||
**Supported file formats:**
|
||||
- CSV (`.csv`)
|
||||
- Excel (`.xlsx`)
|
||||
- Python scripts (`.py`)
|
||||
- JSON (`.json`)
|
||||
- Markdown (`.md`)
|
||||
- Text files (`.txt`)
|
||||
- And more...
|
||||
|
||||
### List Container Files
|
||||
|
||||
```python showLineNumbers title="list_container_files.py"
|
||||
|
|
@ -103,6 +143,40 @@ print(f"Deleted: {result.deleted}")
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
### Upload File
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai-sdk" label="OpenAI SDK">
|
||||
|
||||
```python showLineNumbers title="upload_file.py"
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
file = client.containers.files.create(
|
||||
container_id="cntr_123...",
|
||||
file=open("data.csv", "rb")
|
||||
)
|
||||
|
||||
print(f"Uploaded: {file.id}")
|
||||
print(f"Path: {file.path}")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="curl" label="curl">
|
||||
|
||||
```bash showLineNumbers title="upload_file.sh"
|
||||
curl "http://localhost:4000/v1/containers/cntr_123.../files" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-F file="@data.csv"
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### List Files
|
||||
|
||||
<Tabs>
|
||||
|
|
@ -236,6 +310,13 @@ curl -X DELETE "http://localhost:4000/v1/containers/cntr_123.../files/cfile_456.
|
|||
|
||||
## Parameters
|
||||
|
||||
### Upload File
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `container_id` | string | Yes | Container ID |
|
||||
| `file` | FileTypes | Yes | File to upload. Can be a tuple of (filename, content, content_type), file-like object, or bytes |
|
||||
|
||||
### List Files
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|
|
|
|||
|
|
@ -1,45 +1,100 @@
|
|||
# Contributing - UI
|
||||
|
||||
Here's how to run the LiteLLM UI locally for making changes:
|
||||
Thanks for contributing to the LiteLLM UI! This guide will help you set up your local development environment.
|
||||
|
||||
|
||||
## 1. Clone the repo
|
||||
|
||||
## 1. Clone the repo
|
||||
```bash
|
||||
git clone https://github.com/BerriAI/litellm.git
|
||||
cd litellm
|
||||
```
|
||||
|
||||
## 2. Start the UI + Proxy
|
||||
## 2. Start the Proxy
|
||||
|
||||
**2.1 Start the proxy on port 4000**
|
||||
Create a config file (e.g., `config.yaml`):
|
||||
|
||||
Tell the proxy where the UI is located
|
||||
```bash
|
||||
DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
|
||||
LITELLM_MASTER_KEY = "sk-1234"
|
||||
STORE_MODEL_IN_DB = "True"
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
database_url: postgresql://<user>:<password>@<host>:<port>/<dbname>
|
||||
store_model_in_db: true
|
||||
```
|
||||
|
||||
Start the proxy on port 4000:
|
||||
|
||||
```bash
|
||||
cd litellm/litellm/proxy
|
||||
python3 proxy_cli.py --config /path/to/config.yaml --port 4000
|
||||
poetry run litellm --config config.yaml --port 4000
|
||||
```
|
||||
|
||||
**2.2 Start the UI**
|
||||
The UI comes pre-built in the repo. Access it at `http://localhost:4000/ui`
|
||||
|
||||
Set the mode as development (this will assume the proxy is running on localhost:4000)
|
||||
```bash
|
||||
npm install # install dependencies
|
||||
```
|
||||
## 3. UI Development
|
||||
|
||||
There are two options for UI development:
|
||||
|
||||
### Option A: Development Mode (Hot Reload)
|
||||
|
||||
This runs the UI on port 3000 with hot reload. The proxy runs on port 4000.
|
||||
|
||||
```bash
|
||||
cd litellm/ui/litellm-dashboard
|
||||
|
||||
cd ui/litellm-dashboard
|
||||
npm install
|
||||
npm run dev
|
||||
|
||||
# starts on http://0.0.0.0:3000
|
||||
```
|
||||
|
||||
## 3. Go to local UI
|
||||
**Login flow:**
|
||||
1. Go to `http://localhost:3000`
|
||||
2. You'll be redirected to `http://localhost:4000/ui` for login
|
||||
3. After logging in, manually navigate back to `http://localhost:3000/`
|
||||
4. You're now authenticated and can develop with hot reload
|
||||
|
||||
:::note
|
||||
If you experience redirect loops or authentication issues, clear your browser cookies for localhost or use Build Mode instead.
|
||||
:::
|
||||
|
||||
### Option B: Build Mode
|
||||
|
||||
This builds the UI and copies it to the proxy. Changes require rebuilding.
|
||||
|
||||
1. Make your code changes in `ui/litellm-dashboard/src/`
|
||||
|
||||
2. Build the UI
|
||||
```bash
|
||||
cd ui/litellm-dashboard
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
After building, copy the output to the proxy:
|
||||
|
||||
```bash
|
||||
http://0.0.0.0:3000
|
||||
```
|
||||
cp -r out/* ../../litellm/proxy/_experimental/out/
|
||||
```
|
||||
|
||||
Then restart the proxy and access the UI at `http://localhost:4000/ui`
|
||||
|
||||
## 4. Submitting a PR
|
||||
|
||||
1. Create a new branch for your changes:
|
||||
```bash
|
||||
git checkout -b feat/your-feature-name
|
||||
```
|
||||
|
||||
2. Stage and commit your changes:
|
||||
```bash
|
||||
git add .
|
||||
git commit -m "feat: description of your changes"
|
||||
```
|
||||
|
||||
3. Push to your fork:
|
||||
```bash
|
||||
git push origin feat/your-feature-name
|
||||
```
|
||||
|
||||
4. Create a Pull Request on GitHub following the [PR template](https://github.com/BerriAI/litellm/blob/main/.github/pull_request_template.md)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ This policy outlines the requirements and controls/procedures LiteLLM Cloud has
|
|||
For Customers
|
||||
1. Active Accounts
|
||||
|
||||
- Customer data is retained for as long as the customer’s account is in active status. This includes data such as prompts, generated content, logs, and usage metrics.
|
||||
- Customer data is retained for as long as the customer’s account is in active status. This includes data such as prompts, generated content, logs, and usage metrics. By default, we do not store the message / response content of your API requests or responses. Cloud users need to explicitly opt in to store the message / response content of your API requests or responses.
|
||||
|
||||
2. Voluntary Account Closure
|
||||
|
||||
|
|
|
|||
|
|
@ -187,4 +187,37 @@ export AIOHTTP_TRUST_ENV='True'
|
|||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
## 7. Per-Service SSL Verification
|
||||
|
||||
LiteLLM allows you to override SSL verification settings for specific services or provider calls. This is useful when different services (e.g., an internal guardrail vs. a public LLM provider) require different CA certificates.
|
||||
|
||||
### Bedrock (SDK)
|
||||
You can pass `ssl_verify` directly in the `completion` call.
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
ssl_verify="path/to/bedrock_cert.pem" # Or False to disable
|
||||
)
|
||||
```
|
||||
|
||||
### AIM Guardrail (Proxy)
|
||||
You can configure `ssl_verify` per guardrail in your `config.yaml`.
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: aim-protected-app
|
||||
litellm_params:
|
||||
guardrail: aim
|
||||
ssl_verify: "/path/to/aim_cert.pem" # Use specific cert for AIM
|
||||
```
|
||||
|
||||
### Priority Logic
|
||||
LiteLLM resolves `ssl_verify` using the following priority:
|
||||
1. **Explicit Parameter**: Passed in `completion()` or guardrail config.
|
||||
2. **Environment Variable**: `SSL_VERIFY` environment variable.
|
||||
3. **Global Setting**: `litellm.ssl_verify` setting.
|
||||
4. **System Standard**: `SSL_CERT_FILE` environment variable.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
|
|||
| Supported operations | Create image edits | Single and multiple images supported |
|
||||
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
|
||||
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
|
||||
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. |
|
||||
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
|
||||
|
||||
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
|
|||
| Fallbacks | ✅ | Works between supported models |
|
||||
| Loadbalancing | ✅ | Works between supported models |
|
||||
| Guardrails | ✅ | Applies to input prompts (non-streaming only) |
|
||||
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | |
|
||||
| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
|
@ -238,6 +238,27 @@ print(response)
|
|||
|
||||
See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
|
||||
|
||||
## OpenRouter Image Generation Models
|
||||
|
||||
Use this for image generation models available through OpenRouter (e.g., Google Gemini image generation models)
|
||||
|
||||
#### Usage
|
||||
|
||||
```python showLineNumbers
|
||||
from litellm import image_generation
|
||||
import os
|
||||
|
||||
os.environ['OPENROUTER_API_KEY'] = "your-api-key"
|
||||
|
||||
response = image_generation(
|
||||
model="openrouter/google/gemini-2.5-flash-image",
|
||||
prompt="A beautiful sunset over a calm ocean",
|
||||
size="1024x1024",
|
||||
quality="high",
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## OpenAI Compatible Image Generation Models
|
||||
Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
|
||||
|
||||
|
|
@ -301,5 +322,6 @@ print(f"response: {response}")
|
|||
| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
|
||||
| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
|
||||
| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
|
||||
| OpenRouter | [OpenRouter Image Generation →](./providers/openrouter#image-generation) |
|
||||
| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
|
||||
| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) |
|
||||
|
|
@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
|
|||
| Logging | ✅ | Works across all integrations |
|
||||
| Streaming | ✅ | |
|
||||
| Loadbalancing | ✅ | Between supported models |
|
||||
| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) |
|
||||
| Supported LLM providers | **All LiteLLM supported CHAT COMPLETION providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
|
||||
|
||||
## **LiteLLM Python SDK Usage**
|
||||
|
||||
|
|
@ -207,8 +207,63 @@ for chunk in client.interactions.create_stream(
|
|||
}
|
||||
```
|
||||
|
||||
## **Calling non-Interactions API endpoints (`/interactions` to `/responses` Bridge)**
|
||||
|
||||
LiteLLM allows you to call non-Interactions API models via a bridge to LiteLLM's `/responses` endpoint. This is useful for calling OpenAI, Anthropic, and other providers that don't natively support the Interactions API.
|
||||
|
||||
#### Python SDK Usage
|
||||
|
||||
```python showLineNumbers title="SDK Usage"
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Set API key
|
||||
os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
|
||||
|
||||
# Non-streaming interaction
|
||||
response = litellm.interactions.create(
|
||||
model="gpt-4o",
|
||||
input="Tell me a short joke about programming."
|
||||
)
|
||||
|
||||
print(response.outputs[-1].text)
|
||||
```
|
||||
|
||||
#### LiteLLM Proxy Usage
|
||||
|
||||
**Setup Config:**
|
||||
|
||||
```yaml showLineNumbers title="Example Configuration"
|
||||
model_list:
|
||||
- model_name: openai-model
|
||||
litellm_params:
|
||||
model: gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
|
||||
**Start Proxy:**
|
||||
|
||||
```bash showLineNumbers title="Start LiteLLM Proxy"
|
||||
litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
**Make Request:**
|
||||
|
||||
```bash showLineNumbers title="non-Interactions API Model Request"
|
||||
curl http://localhost:4000/v1beta/interactions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "openai-model",
|
||||
"input": "Tell me a short joke about programming."
|
||||
}'
|
||||
```
|
||||
|
||||
## **Supported Providers**
|
||||
|
||||
| Provider | Link to Usage |
|
||||
|----------|---------------|
|
||||
| Google AI Studio | [Usage](#quick-start) |
|
||||
| All other LiteLLM providers | [Bridge Usage](#calling-non-interactions-api-endpoints-interactions-to-responses-bridge) |
|
||||
|
|
|
|||
|
|
@ -17,10 +17,15 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
|
|||
## Overview
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| MCP Operations | • List Tools<br/>• Call Tools |
|
||||
| MCP Operations | • List Tools<br/>• Call Tools <br/>• Prompts <br/>• Resources |
|
||||
| Supported MCP Transports | • Streamable HTTP<br/>• SSE<br/>• Standard Input/Output (stdio) |
|
||||
| LiteLLM Permission Management | • By Key<br/>• By Team<br/>• By Organization |
|
||||
|
||||
:::caution MCP protocol update
|
||||
Starting in LiteLLM v1.80.18, the LiteLLM MCP protocol version is `2025-11-25`.<br/>
|
||||
LiteLLM namespaces multiple MCP servers by prefixing each tool name with its MCP server name, so newly created servers now must use names that comply with SEP-986—noncompliant names cannot be added anymore. Existing servers that still violate SEP-986 only emit warnings today, but future MCP-side rollouts may block those names entirely, so we recommend updating any legacy server names proactively before MCP enforcement makes them unusable.
|
||||
:::
|
||||
|
||||
## Adding your MCP
|
||||
|
||||
### Prerequisites
|
||||
|
|
@ -60,6 +65,8 @@ model_list:
|
|||
|
||||
If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
|
||||
|
||||
For diagnosing connectivity problems after setup, see the [MCP Troubleshooting Guide](./mcp_troubleshoot.md).
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="ui" label="LiteLLM UI">
|
||||
|
||||
|
|
@ -110,6 +117,22 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t
|
|||
<br/>
|
||||
<br/>
|
||||
|
||||
### OAuth Configuration & Overrides
|
||||
|
||||
LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.ietf.org/doc/html/rfc8414) by default. When you create an MCP server in the UI and set `Authentication: OAuth`, LiteLLM will locate the provider metadata, dynamically register a client, and perform PKCE-based authorization without you providing any additional details.
|
||||
|
||||
**Customize the OAuth flow when needed:**
|
||||
|
||||
<Image
|
||||
img={require('../img/mcp_oauth.png')}
|
||||
style={{width: '80%', display: 'block', margin: '0'}}
|
||||
/>
|
||||
|
||||
- **Provide explicit client credentials** – If the MCP provider does not offer dynamic client registration or you prefer to manage the client yourself, fill in `client_id`, `client_secret`, and the desired `scopes`.
|
||||
- **Override discovery URLs** – In some environments, LiteLLM might not be able to reach the provider's metadata endpoints. Use the optional `authorization_url`, `token_url`, and `registration_url` fields to point LiteLLM directly to the correct endpoints.
|
||||
|
||||
<br/>
|
||||
|
||||
### Static Headers
|
||||
|
||||
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
|
||||
|
|
@ -182,6 +205,7 @@ mcp_servers:
|
|||
- `http` - Streamable HTTP transport
|
||||
- `stdio` - Standard Input/Output transport
|
||||
- **Command**: The command to execute for stdio transport (required for stdio)
|
||||
- **allow_all_keys**: Set to `true` to make the server available to every LiteLLM API key, even if the key/team doesn't list the server in its MCP permissions.
|
||||
- **Args**: Array of arguments to pass to the command (optional for stdio)
|
||||
- **Env**: Environment variables to set for the stdio process (optional for stdio)
|
||||
- **Description**: Optional description for the server
|
||||
|
|
@ -309,6 +333,7 @@ litellm_settings:
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Converting OpenAPI Specs to MCP Servers
|
||||
|
||||
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
|
||||
|
|
@ -485,7 +510,7 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
|
|||
|
||||
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
|
||||
|
||||
This configuration is currently available on the config.yaml, with UI support coming soon.
|
||||
You can configure this either in `config.yaml` or directly from the LiteLLM UI (MCP Servers → Authentication → OAuth).
|
||||
|
||||
```yaml
|
||||
mcp_servers:
|
||||
|
|
@ -746,8 +771,33 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \
|
|||
3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server
|
||||
4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers
|
||||
|
||||
---
|
||||
|
||||
### Passing Request Headers to STDIO env Vars
|
||||
|
||||
If your stdio MCP server needs per-request credentials, you can map HTTP headers from the client request directly into the environment for the launched stdio process. Reference the header name in the env value using the `${X-HEADER_NAME}` syntax. LiteLLM will read that header from the incoming request and set the env var before starting the command.
|
||||
|
||||
```json title="Forward X-GITHUB_PERSONAL_ACCESS_TOKEN header to stdio env" showLineNumbers
|
||||
{
|
||||
"mcpServers": {
|
||||
"github": {
|
||||
"command": "docker",
|
||||
"args": [
|
||||
"run",
|
||||
"-i",
|
||||
"--rm",
|
||||
"-e",
|
||||
"GITHUB_PERSONAL_ACCESS_TOKEN",
|
||||
"ghcr.io/github/github-mcp-server"
|
||||
],
|
||||
"env": {
|
||||
"GITHUB_PERSONAL_ACCESS_TOKEN": "${X-GITHUB_PERSONAL_ACCESS_TOKEN}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
|
||||
|
||||
## Using your MCP with client side credentials
|
||||
|
||||
|
|
@ -1431,3 +1481,17 @@ async with stdio_client(server_params) as (read, write):
|
|||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## FAQ
|
||||
|
||||
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
|
||||
|
||||
At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
|
||||
|
||||
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**
|
||||
|
||||
The UI keeps only transient state in `sessionStorage` so the OAuth redirect flow can finish; the token is not persisted in the server or database.
|
||||
|
||||
**Q: I'm seeing MCP connection errors—what should I check?**
|
||||
|
||||
Walk through the [MCP Troubleshooting Guide](./mcp_troubleshoot.md) for step-by-step isolation (Client → LiteLLM vs. LiteLLM → MCP), log examples, and verification methods like MCP Inspector and `curl`.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ LiteLLM provides fine-grained permission management for MCP servers, allowing yo
|
|||
- **Restrict MCP access by entity**: Control which keys, teams, or organizations can access specific MCP servers
|
||||
- **Tool-level filtering**: Automatically filter available tools based on entity permissions
|
||||
- **Centralized control**: Manage all MCP permissions from the LiteLLM Admin UI or API
|
||||
- **One-click public MCPs**: Mark specific servers as available to every LiteLLM API key when you don't need per-key restrictions
|
||||
|
||||
This ensures that only authorized entities can discover and use MCP tools, providing an additional security layer for your MCP infrastructure.
|
||||
|
||||
|
|
@ -95,6 +96,48 @@ mcp_servers:
|
|||
- If you specify both `allowed_tools` and `disallowed_tools`, the allowed list takes priority
|
||||
- Tool names are case-sensitive
|
||||
|
||||
## Public MCP Servers (allow_all_keys)
|
||||
|
||||
Some MCP servers are meant to be shared broadly—think internal knowledge bases, calendar integrations, or other low-risk utilities where every team should be able to connect without requesting access. Instead of adding those servers to every key, team, or organization, enable the new `allow_all_keys` toggle.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="ui" label="UI">
|
||||
|
||||
1. Open **MCP Servers → Add / Edit** in the Admin UI.
|
||||
2. Expand **Permission Management / Access Control**.
|
||||
3. Toggle **Allow All LiteLLM Keys** on.
|
||||
|
||||
<Image
|
||||
img={require('../img/mcp_allow_all_ui.png')}
|
||||
style={{width: '80%', display: 'block', margin: '1rem auto'}}
|
||||
alt="MCP server configuration in Admin UI"
|
||||
/>
|
||||
|
||||
The toggle makes the server “public” without touching existing access groups.
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="config" label="config.yaml">
|
||||
|
||||
Set `allow_all_keys: true` to mark the server as public:
|
||||
|
||||
```yaml title="Make an MCP server public" showLineNumbers
|
||||
mcp_servers:
|
||||
deepwiki:
|
||||
url: https://mcp.deepwiki.com/mcp
|
||||
allow_all_keys: true
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### When to use it
|
||||
|
||||
- You have shared MCP utilities where fine-grained ACLs would only add busywork.
|
||||
- You want a “default enabled” experience for internal users, while still being able to layer tool-level restrictions.
|
||||
- You’re onboarding new teams and want the safest MCPs available out of the box.
|
||||
|
||||
Once enabled, LiteLLM automatically includes the server for every key during tool discovery/calls—no extra virtual-key or team configuration is required.
|
||||
|
||||
---
|
||||
|
||||
## Allow/Disallow MCP Tool Parameters
|
||||
|
|
@ -591,3 +634,31 @@ Control which tools different teams can access from the same MCP server. For exa
|
|||
This video shows how to set allowed tools for a Key, Team, or Organization.
|
||||
|
||||
<iframe width="840" height="500" src="https://www.loom.com/embed/7464d444c3324078892367272fe50745" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
|
||||
|
||||
|
||||
## Dashboard View Modes
|
||||
|
||||
Proxy admins can also control what non-admins see inside the MCP dashboard via `general_settings.user_mcp_management_mode`:
|
||||
|
||||
- `restricted` *(default)* – users only see servers that their team explicitly has access to.
|
||||
- `view_all` – every dashboard user can see the full MCP server list.
|
||||
|
||||
```yaml title="Config example"
|
||||
general_settings:
|
||||
user_mcp_management_mode: view_all
|
||||
```
|
||||
|
||||
This is useful when you want discoverability for MCP offerings without granting additional execution privileges.
|
||||
|
||||
|
||||
## Publish MCP Registry
|
||||
|
||||
If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry).
|
||||
|
||||
1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy.
|
||||
2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`.
|
||||
3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL.
|
||||
|
||||
:::note Permissions still apply
|
||||
The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions.
|
||||
:::
|
||||
|
|
|
|||
|
|
@ -85,4 +85,5 @@ MCP guardrails work with all LiteLLM-supported guardrail providers:
|
|||
- **Bedrock**: AWS Bedrock guardrails
|
||||
- **Lakera**: Content moderation
|
||||
- **Aporia**: Custom guardrails
|
||||
- **Noma**: Noma Security
|
||||
- **Custom**: Your own guardrail implementations
|
||||
99
docs/my-website/docs/mcp_troubleshoot.md
Normal file
99
docs/my-website/docs/mcp_troubleshoot.md
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# MCP Troubleshooting Guide
|
||||
|
||||
When LiteLLM acts as an MCP proxy, traffic normally flows `Client → LiteLLM Proxy → MCP Server`, while OAuth-enabled setups add an authorization server for metadata discovery.
|
||||
|
||||
For provisioning steps, transport options, and configuration fields, refer to [mcp.md](./mcp.md).
|
||||
|
||||
## Locate the Error Source
|
||||
|
||||
Pin down where the failure occurs before adjusting settings so you do not mix symptoms from separate hops.
|
||||
|
||||
### LiteLLM UI / Playground Errors (LiteLLM → MCP)
|
||||
Failures shown on the MCP creation form or within the MCP Tool Testing Playground mean the LiteLLM proxy cannot reach the MCP server. Typical causes are misconfiguration (transport, headers, credentials), MCP/server outages, network/firewall blocks, or inaccessible OAuth metadata.
|
||||
|
||||
<Image
|
||||
img={require('../img/mcp_tool_testing_playground.png')}
|
||||
style={{width: '80%', display: 'block', margin: '0'}}
|
||||
/>
|
||||
|
||||
<br/>
|
||||
|
||||
**Actions**
|
||||
- Capture LiteLLM proxy logs alongside MCP-server logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) to inspect the request/response pair and stack traces.
|
||||
- From the LiteLLM server, run Method 2 ([`curl` smoke test](./mcp_troubleshoot#curl-smoke-test)) against the MCP endpoint to confirm basic connectivity.
|
||||
|
||||
### Client Traffic Issues (Client → LiteLLM)
|
||||
If only real client requests fail, determine whether LiteLLM ever reaches the MCP hop.
|
||||
|
||||
#### MCP Protocol Sessions
|
||||
Clients such as IDEs or agent runtimes speak the MCP protocol directly with LiteLLM.
|
||||
|
||||
**Actions**
|
||||
- Inspect LiteLLM access logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to verify the client request reached the proxy and which MCP server it targeted.
|
||||
- Review LiteLLM error logs (see [Error Log Example](./mcp_troubleshoot#error-log-example-failed-mcp-call)) for TLS, authentication, or routing errors that block the request before the MCP call starts.
|
||||
- Use the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to confirm the MCP server is reachable outside of the failing client.
|
||||
|
||||
#### Responses/Completions with Embedded MCP Calls
|
||||
During `/responses` or `/chat/completions`, LiteLLM may trigger MCP tool calls mid-request. An error could occur before the MCP call begins or after the MCP responds.
|
||||
|
||||
**Actions**
|
||||
- Check LiteLLM request logs (see [Access Log Example](./mcp_troubleshoot#access-log-example-successful-mcp-call)) to see whether an MCP attempt was recorded; if not, the problem lies in `Client → LiteLLM`.
|
||||
- Validate MCP connectivity with the [MCP Inspector](./mcp_troubleshoot#mcp-inspector) to ensure the server responds.
|
||||
- Reproduce the same MCP call via the LiteLLM Playground to confirm LiteLLM can complete the MCP hop independently.
|
||||
|
||||
<Image
|
||||
img={require('../img/mcp_playground.png')}
|
||||
style={{width: '80%', display: 'block', margin: '0'}}
|
||||
/>
|
||||
|
||||
### OAuth Metadata Discovery
|
||||
LiteLLM performs metadata discovery per the MCP spec ([section 2.3](https://modelcontextprotocol.info/specification/draft/basic/authorization/#23-server-metadata-discovery)). When OAuth is enabled, confirm the authorization server exposes the metadata URL and that LiteLLM can fetch it.
|
||||
|
||||
**Actions**
|
||||
- Use `curl <metadata_url>` (or similar) from the LiteLLM host to ensure the discovery document is reachable and contains the expected authorization/token endpoints.
|
||||
- Record the exact metadata URL, requested scopes, and any static client credentials so support can replay the discovery step if needed.
|
||||
|
||||
## Verify Connectivity
|
||||
|
||||
Run lightweight validations before impacting production traffic.
|
||||
|
||||
### MCP Inspector
|
||||
Use the MCP Inspector when you need to test both `Client → LiteLLM` and `Client → MCP` communications in one place; it makes isolating the failing hop straightforward.
|
||||
|
||||
1. Execute `npx @modelcontextprotocol/inspector` on your workstation.
|
||||
2. Configure and connect:
|
||||
- **Transport Type:** choose the transport the client uses (Streamable HTTP for LiteLLM).
|
||||
- **URL:** the endpoint under test (LiteLLM MCP URL for `Client → LiteLLM`, or the MCP server URL for `Client → MCP`).
|
||||
- **Custom Headers:** e.g., `Authorization: Bearer <LiteLLM API Key>`.
|
||||
3. Open the **Tools** tab and click **List Tools** to verify the MCP alias responds.
|
||||
|
||||
### `curl` Smoke Test
|
||||
`curl` is ideal on servers where installing the Inspector is impractical. It replicates the MCP tool call LiteLLM would make—swap in the domain of the system under test (LiteLLM or the MCP server).
|
||||
|
||||
```bash
|
||||
curl -X POST https://your-target-domain.example.com/mcp \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Accept: application/json, text/event-stream" \
|
||||
-d '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}'
|
||||
```
|
||||
|
||||
Add `-H "Authorization: Bearer <LiteLLM API Key>"` when the target is a LiteLLM endpoint that requires authentication. Adjust the headers, or payload to target other MCP methods. Matching failures between `curl` and LiteLLM confirm that the MCP server or network/OAuth layer is the culprit.
|
||||
|
||||
## Review Logs
|
||||
|
||||
Well-scoped logs make it clear whether LiteLLM reached the MCP server and what happened next.
|
||||
|
||||
### Access Log Example (successful MCP call)
|
||||
```text
|
||||
INFO: 127.0.0.1:57230 - "POST /everything/mcp HTTP/1.1" 200 OK
|
||||
```
|
||||
|
||||
### Error Log Example (failed MCP call)
|
||||
```text
|
||||
07:22:00 - LiteLLM:ERROR: client.py:224 - MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception), Server: http://localhost:3001/mcp, Transport: MCPTransport.http
|
||||
httpcore.ConnectError: All connection attempts failed
|
||||
ERROR:LiteLLM:MCP client list_tools failed - Error Type: ExceptionGroup, Error: unhandled errors in a TaskGroup (1 sub-exception)...
|
||||
httpx.ConnectError: All connection attempts failed
|
||||
```
|
||||
|
|
@ -68,6 +68,7 @@ environment_variables:
|
|||
ARIZE_API_KEY: "141a****"
|
||||
ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint
|
||||
ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT or Neither (defaults to https://otlp.arize.com/v1 on grpc)
|
||||
ARIZE_PROJECT_NAME: "my-litellm-project" # OPTIONAL - sets the arize project name
|
||||
```
|
||||
|
||||
2. Start the proxy
|
||||
|
|
|
|||
|
|
@ -65,6 +65,52 @@ Start your LiteLLM proxy with the configuration:
|
|||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
## Setup on UI
|
||||
|
||||
1\. Click "Settings"
|
||||
|
||||
|
||||
|
||||
|
||||
2\. Click "Logging & Alerts"
|
||||
|
||||
|
||||
|
||||
|
||||
3\. Click "CloudZero Cost Tracking"
|
||||
|
||||
|
||||
|
||||
|
||||
4\. Click "Add CloudZero Integration"
|
||||
|
||||
|
||||
|
||||
|
||||
5\. Enter your CloudZero API Key.
|
||||
|
||||
|
||||
|
||||
|
||||
6\. Enter your CloudZero Connection ID.
|
||||
|
||||
|
||||
|
||||
|
||||
7\. Click "Create"
|
||||
|
||||
|
||||
|
||||
|
||||
8\. Test your payload with "Run Dry Run Simulation"
|
||||
|
||||
|
||||
|
||||
|
||||
10\. Click "Export Data Now" to export to CLoudZero
|
||||
|
||||
|
||||
|
||||
## Testing Your Setup
|
||||
|
||||
### Dry Run Export
|
||||
|
|
|
|||
93
docs/my-website/docs/observability/focus.md
Normal file
93
docs/my-website/docs/observability/focus.md
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Focus Export (Experimental)
|
||||
|
||||
:::caution Experimental feature
|
||||
Focus Format export is under active development and currently considered experimental.
|
||||
Interfaces, schema mappings, and configuration options may change as we iterate based on user feedback.
|
||||
Please treat this integration as a preview and report any issues or suggestions to help us stabilize and improve the workflow.
|
||||
:::
|
||||
|
||||
LiteLLM can emit usage data in the [FinOps FOCUS format](https://focus.finops.org/focus-specification/v1-2/) and push artifacts (for example Parquet files) to destinations such as Amazon S3. This enables downstream cost-analysis tooling to ingest a standardised dataset directly from LiteLLM.
|
||||
|
||||
LiteLLM currently conforms to the FinOps FOCUS v1.2 specification when emitting this dataset.
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Destination | Export LiteLLM usage data in FOCUS format to managed storage (currently S3) |
|
||||
| Callback name | `focus` |
|
||||
| Supported operations | Automatic scheduled export |
|
||||
| Data format | FOCUS Normalised Dataset (Parquet) |
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Common settings
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `FOCUS_PROVIDER` | No | Destination provider (defaults to `s3`). |
|
||||
| `FOCUS_FORMAT` | No | Output format (currently only `parquet`). |
|
||||
| `FOCUS_FREQUENCY` | No | Export cadence. Prefer `hourly` or `daily` for production; `interval` is intended for short test loops. Defaults to `hourly`. |
|
||||
| `FOCUS_CRON_OFFSET` | No | Minute offset used for hourly/daily cron triggers. Defaults to `5`. |
|
||||
| `FOCUS_INTERVAL_SECONDS` | No | Interval (seconds) when `FOCUS_FREQUENCY="interval"`. |
|
||||
| `FOCUS_PREFIX` | No | Object key prefix/folder. Defaults to `focus_exports`. |
|
||||
|
||||
### S3 destination
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `FOCUS_S3_BUCKET_NAME` | Yes | Destination bucket for exported files. |
|
||||
| `FOCUS_S3_REGION_NAME` | No | AWS region for the bucket. |
|
||||
| `FOCUS_S3_ENDPOINT_URL` | No | Custom endpoint (useful for S3-compatible storage). |
|
||||
| `FOCUS_S3_ACCESS_KEY` | Yes | AWS access key for uploads. |
|
||||
| `FOCUS_S3_SECRET_KEY` | Yes | AWS secret key for uploads. |
|
||||
| `FOCUS_S3_SESSION_TOKEN` | No | AWS session token if using temporary credentials. |
|
||||
|
||||
## Setup via Config
|
||||
|
||||
### Configure environment variables
|
||||
|
||||
```bash
|
||||
export FOCUS_PROVIDER="s3"
|
||||
export FOCUS_PREFIX="focus_exports"
|
||||
|
||||
# S3 example
|
||||
export FOCUS_S3_BUCKET_NAME="my-litellm-focus-bucket"
|
||||
export FOCUS_S3_REGION_NAME="us-east-1"
|
||||
export FOCUS_S3_ACCESS_KEY="AKIA..."
|
||||
export FOCUS_S3_SECRET_KEY="..."
|
||||
```
|
||||
|
||||
### Update LiteLLM config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: sk-your-key
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["focus"]
|
||||
```
|
||||
|
||||
### Start the proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
During boot LiteLLM registers the Focus logger and a background job that runs according to the configured frequency.
|
||||
|
||||
## Planned Enhancements
|
||||
- Add "Setup on UI" flow alongside the current configuration-based setup.
|
||||
- Add GCS / Azure Blob to the Destination options.
|
||||
- Support CSV output alongside Parquet.
|
||||
|
||||
## Related Links
|
||||
|
||||
- [Focus](https://focus.finops.org/)
|
||||
|
||||
|
|
@ -47,6 +47,7 @@ callback_settings:
|
|||
| `endpoint` | string | Yes | HTTP endpoint to send logs to |
|
||||
| `headers` | dict | No | Custom headers for the request |
|
||||
| `event_types` | list | No | Filter events: `llm_api_success`, `llm_api_failure`. Defaults to all events. |
|
||||
| `log_format` | string | No | Output format: `json_array` (default), `ndjson`, or `single`. Controls how logs are batched and sent. |
|
||||
|
||||
## Pre-configured Callbacks
|
||||
|
||||
|
|
@ -107,4 +108,62 @@ callback_settings:
|
|||
flush_interval: 60 # seconds, default: 60
|
||||
```
|
||||
|
||||
## Log Format Options
|
||||
|
||||
Control how logs are formatted and sent to your endpoint.
|
||||
|
||||
### JSON Array (Default)
|
||||
|
||||
```yaml
|
||||
callback_settings:
|
||||
my_api:
|
||||
callback_type: generic_api
|
||||
endpoint: https://your-endpoint.com
|
||||
log_format: json_array # default if not specified
|
||||
```
|
||||
|
||||
Sends all logs in a batch as a single JSON array `[{log1}, {log2}, ...]`. This is the default behavior and maintains backward compatibility.
|
||||
|
||||
**When to use**: Most HTTP endpoints expecting batched JSON data.
|
||||
|
||||
### NDJSON (Newline-Delimited JSON)
|
||||
|
||||
```yaml
|
||||
callback_settings:
|
||||
my_api:
|
||||
callback_type: generic_api
|
||||
endpoint: https://your-endpoint.com
|
||||
log_format: ndjson
|
||||
```
|
||||
|
||||
Sends logs as newline-delimited JSON (one record per line):
|
||||
```
|
||||
{log1}
|
||||
{log2}
|
||||
{log3}
|
||||
```
|
||||
|
||||
**When to use**: Log aggregation services like Sumo Logic, Splunk, or Datadog that support field extraction on individual records.
|
||||
|
||||
**Benefits**:
|
||||
- Each log is ingested as a separate message
|
||||
- Field Extraction Rules work at ingest time
|
||||
- Better parsing and querying performance
|
||||
|
||||
### Single
|
||||
|
||||
```yaml
|
||||
callback_settings:
|
||||
my_api:
|
||||
callback_type: generic_api
|
||||
endpoint: https://your-endpoint.com
|
||||
log_format: single
|
||||
```
|
||||
|
||||
Sends each log as an individual HTTP request in parallel when the batch is flushed.
|
||||
|
||||
**When to use**: Endpoints that expect individual records, or when you need maximum compatibility.
|
||||
|
||||
**Note**: This mode sends N HTTP requests per batch (more overhead). Consider using `ndjson` instead if your endpoint supports it.
|
||||
|
||||
|
||||
|
|
|
|||
162
docs/my-website/docs/observability/levo_integration.md
Normal file
162
docs/my-website/docs/observability/levo_integration.md
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
---
|
||||
sidebar_label: Levo AI
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Levo AI
|
||||
|
||||
<div className="levo-logo-container" style={{ marginTop: '0.5rem', marginBottom: '1rem' }}>
|
||||
<div className="levo-logo-light">
|
||||
<Image img={require('../../img/levo_logo.png')} />
|
||||
</div>
|
||||
<div className="levo-logo-dark">
|
||||
<Image img={require('../../img/levo_logo_dark.png')} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
[Levo](https://levo.ai/) is an AI observability and compliance platform that provides comprehensive monitoring, analysis, and compliance tracking for LLM applications.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Send all your LLM requests and responses to Levo for monitoring and analysis using LiteLLM's built-in Levo integration.
|
||||
|
||||
### What You'll Get
|
||||
|
||||
- **Complete visibility** into all LLM API calls across all providers
|
||||
- **Request and response data** including prompts, completions, and metadata
|
||||
- **Usage and cost tracking** with token counts and cost breakdowns
|
||||
- **Error monitoring** and performance metrics
|
||||
- **Compliance tracking** for audit and governance
|
||||
|
||||
### Setup Steps
|
||||
|
||||
**1. Install OpenTelemetry dependencies:**
|
||||
|
||||
```bash
|
||||
pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc
|
||||
```
|
||||
|
||||
**2. Enable Levo callback in your LiteLLM config:**
|
||||
|
||||
Add to your `litellm_config.yaml`:
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
callbacks: ["levo"]
|
||||
```
|
||||
|
||||
**3. Configure environment variables:**
|
||||
|
||||
[Contact Levo support](mailto:support@levo.ai) to get your collector endpoint URL, API key, organization ID, and workspace ID.
|
||||
|
||||
Set these required environment variables:
|
||||
|
||||
```bash
|
||||
export LEVOAI_API_KEY="<your-levo-api-key>"
|
||||
export LEVOAI_ORG_ID="<your-levo-org-id>"
|
||||
export LEVOAI_WORKSPACE_ID="<your-workspace-id>"
|
||||
export LEVOAI_COLLECTOR_URL="<your-levo-collector-url>"
|
||||
```
|
||||
|
||||
**Note:** The collector URL should be the full endpoint URL provided by Levo support. It will be used exactly as provided.
|
||||
|
||||
**4. Start LiteLLM:**
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
**5. Make requests - they'll automatically be sent to Levo!**
|
||||
|
||||
```bash
|
||||
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, this is a test message"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
## What Data is Captured
|
||||
|
||||
| Feature | Details |
|
||||
|---------|---------|
|
||||
| **What is logged** | OpenTelemetry Trace Data (OTLP format) |
|
||||
| **Events** | Success + Failure |
|
||||
| **Format** | OTLP (OpenTelemetry Protocol) |
|
||||
| **Headers** | Automatically includes `Authorization: Bearer {LEVOAI_API_KEY}`, `x-levo-organization-id`, and `x-levo-workspace-id` |
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Required Environment Variables
|
||||
|
||||
| Variable | Description | Example |
|
||||
|----------|-------------|---------|
|
||||
| `LEVOAI_API_KEY` | Your Levo API key | `levo_abc123...` |
|
||||
| `LEVOAI_ORG_ID` | Your Levo organization ID | `org-123456` |
|
||||
| `LEVOAI_WORKSPACE_ID` | Your Levo workspace ID | `workspace-789` |
|
||||
| `LEVOAI_COLLECTOR_URL` | Full collector endpoint URL from Levo support | `https://collector.levo.ai/v1/traces` |
|
||||
|
||||
### Optional Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `LEVOAI_ENV_NAME` | Environment name for tagging traces | `None` |
|
||||
|
||||
**Note:** The collector URL is used exactly as provided by Levo support. No path manipulation is performed.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Not seeing traces in Levo?
|
||||
|
||||
1. **Verify Levo callback is enabled**: Check LiteLLM startup logs for `initializing callbacks=['levo']`
|
||||
|
||||
2. **Check required environment variables**: Ensure all required variables are set:
|
||||
```bash
|
||||
echo $LEVOAI_API_KEY
|
||||
echo $LEVOAI_ORG_ID
|
||||
echo $LEVOAI_WORKSPACE_ID
|
||||
echo $LEVOAI_COLLECTOR_URL
|
||||
```
|
||||
|
||||
3. **Verify collector connectivity**: Test if your collector is reachable:
|
||||
```bash
|
||||
curl <your-collector-url>/health
|
||||
```
|
||||
|
||||
4. **Check for initialization errors**: Look for errors in LiteLLM startup logs. Common issues:
|
||||
- Missing OpenTelemetry packages: Install with `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
|
||||
- Missing required environment variables: All four required variables must be set
|
||||
- Invalid collector URL: Ensure the URL is correct and reachable
|
||||
|
||||
5. **Enable debug logging**:
|
||||
```bash
|
||||
export LITELLM_LOG="DEBUG"
|
||||
```
|
||||
|
||||
6. **Wait for async export**: OTLP sends traces asynchronously. Wait 10-15 seconds after making requests before checking Levo.
|
||||
|
||||
### Common Errors
|
||||
|
||||
**Error: "LEVOAI_COLLECTOR_URL environment variable is required"**
|
||||
- Solution: Set the `LEVOAI_COLLECTOR_URL` environment variable with your collector endpoint URL from Levo support.
|
||||
|
||||
**Error: "No module named 'opentelemetry'"**
|
||||
- Solution: Install OpenTelemetry packages: `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Levo Documentation](https://docs.levo.ai)
|
||||
- [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/)
|
||||
|
||||
## Need Help?
|
||||
|
||||
For issues or questions about the Levo integration with LiteLLM, please [contact Levo support](mailto:support@levo.ai) or open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm/issues).
|
||||
|
|
@ -40,6 +40,10 @@ import os
|
|||
# from https://logfire.pydantic.dev/
|
||||
os.environ["LOGFIRE_TOKEN"] = ""
|
||||
|
||||
# Optionally customize the base url
|
||||
# from https://logfire.pydantic.dev/
|
||||
os.environ["LOGFIRE_BASE_URL"] = ""
|
||||
|
||||
# LLM API Keys
|
||||
os.environ['OPENAI_API_KEY']=""
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
|
|||
|
||||
# OpenTelemetry - Tracing LLMs with any observability tool
|
||||
|
||||
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop and others.
|
||||
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop, Levo AI and others.
|
||||
|
||||
<Image img={require('../../img/traceloop_dash.png')} />
|
||||
|
||||
|
|
@ -12,7 +12,9 @@ OpenTelemetry is a CNCF standard for observability. It connects to any observabi
|
|||
|
||||
From v1.81.0, the request/response will be set as attributes on the parent "Received Proxy Server Request" span by default. This allows you to see the request/response in the parent span in your observability tool.
|
||||
|
||||
To use the older behavior with nested "litellm_request" spans, set the following environment variable:
|
||||
**Note:** When making multiple LLM calls within an external OTEL span context, the last call's attributes will overwrite previous calls' attributes on the parent span.
|
||||
|
||||
To use the older behavior with nested "litellm_request" spans (which creates separate spans for each call), set the following environment variable:
|
||||
|
||||
```shell
|
||||
USE_OTEL_LITELLM_REQUEST_SPAN=true
|
||||
|
|
@ -61,6 +63,8 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc
|
|||
OTEL_EXPORTER_OTLP_HEADERS="api-key=key,other-config-value=value"
|
||||
```
|
||||
|
||||
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="laminar" label="Log to Laminar">
|
||||
|
|
@ -71,6 +75,8 @@ OTEL_ENDPOINT="https://api.lmnr.ai:8443"
|
|||
OTEL_HEADERS="authorization=Bearer <project-api-key>"
|
||||
```
|
||||
|
||||
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
|
@ -126,4 +132,4 @@ If you don't see traces landing on your integration, set `OTEL_DEBUG="True"` in
|
|||
export OTEL_DEBUG="True"
|
||||
```
|
||||
|
||||
This will emit any logging issues to the console.
|
||||
This will emit any logging issues to the console.
|
||||
|
|
|
|||
|
|
@ -73,6 +73,8 @@ environment_variables:
|
|||
PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the HTTP endpoint
|
||||
```
|
||||
|
||||
> Note: If you set the gRPC endpoint, install `grpcio` via `pip install "litellm[grpc]"` (or `grpcio`).
|
||||
|
||||
2. Start the proxy
|
||||
|
||||
```bash
|
||||
|
|
|
|||
122
docs/my-website/docs/observability/qualifire_integration.md
Normal file
122
docs/my-website/docs/observability/qualifire_integration.md
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# Qualifire - LLM Evaluation, Guardrails & Observability
|
||||
|
||||
[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications.
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities
|
||||
- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches
|
||||
- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents
|
||||
- **Prompt Management** - Centralized prompt management with versioning and no-code studio
|
||||
|
||||
:::tip
|
||||
|
||||
Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more.
|
||||
|
||||
:::
|
||||
|
||||
## Pre-Requisites
|
||||
|
||||
1. Create an account on [Qualifire](https://app.qualifire.ai/)
|
||||
2. Get your API key and webhook URL from the Qualifire dashboard
|
||||
|
||||
```bash
|
||||
pip install litellm
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire.
|
||||
|
||||
```python
|
||||
litellm.callbacks = ["qualifire_eval"]
|
||||
```
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Set Qualifire credentials
|
||||
os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key"
|
||||
os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url"
|
||||
|
||||
# LLM API Keys
|
||||
os.environ['OPENAI_API_KEY'] = "your-openai-api-key"
|
||||
|
||||
# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire
|
||||
litellm.callbacks = ["qualifire_eval"]
|
||||
|
||||
# OpenAI call
|
||||
response = litellm.completion(
|
||||
model="gpt-5",
|
||||
messages=[
|
||||
{"role": "user", "content": "Hi 👋 - i'm openai"}
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Using with LiteLLM Proxy
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["qualifire_eval"]
|
||||
|
||||
general_settings:
|
||||
master_key: "sk-1234"
|
||||
|
||||
environment_variables:
|
||||
QUALIFIRE_API_KEY: "your-qualifire-api-key"
|
||||
QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations"
|
||||
```
|
||||
|
||||
2. Start the proxy
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Description |
|
||||
| ----------------------- | ------------------------------------------------------ |
|
||||
| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication |
|
||||
| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard |
|
||||
|
||||
## What Gets Logged?
|
||||
|
||||
The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call.
|
||||
|
||||
This includes:
|
||||
|
||||
- Request messages and parameters
|
||||
- Response content and metadata
|
||||
- Token usage statistics
|
||||
- Latency metrics
|
||||
- Model information
|
||||
- Cost data
|
||||
|
||||
Once data is in Qualifire, you can:
|
||||
|
||||
- Run evaluations to detect hallucinations, toxicity, and policy violations
|
||||
- Set up guardrails to block or modify responses in real-time
|
||||
- View traces across your entire AI pipeline
|
||||
- Track performance and quality metrics over time
|
||||
398
docs/my-website/docs/observability/signoz.md
Normal file
398
docs/my-website/docs/observability/signoz.md
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# SigNoz LiteLLM Integration
|
||||
|
||||
For more details on setting up observability for LiteLLM, check out the [SigNoz LiteLLM observability docs](https://signoz.io/docs/litellm-observability/).
|
||||
|
||||
|
||||
## Overview
|
||||
|
||||
This guide walks you through setting up observability and monitoring for LiteLLM SDK and Proxy Server using [OpenTelemetry](https://opentelemetry.io/) and exporting logs, traces, and metrics to SigNoz. With this integration, you can observe various models performance, capture request/response details, and track system-level metrics in SigNoz, giving you real-time visibility into latency, error rates, and usage trends for your LiteLLM applications.
|
||||
|
||||
Instrumenting LiteLLM in your AI applications with telemetry ensures full observability across your AI workflows, making it easier to debug issues, optimize performance, and understand user interactions. By leveraging SigNoz, you can analyze correlated traces, logs, and metrics in unified dashboards, configure alerts, and gain actionable insights to continuously improve reliability, responsiveness, and user experience.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A [SigNoz Cloud account](https://signoz.io/teams/) with an active ingestion key
|
||||
- Internet access to send telemetry data to SigNoz Cloud
|
||||
- [LiteLLM](https://www.litellm.ai/) SDK or Proxy integration
|
||||
- For Python: `pip` installed for managing Python packages and _(optional but recommended)_ a Python virtual environment to isolate dependencies
|
||||
|
||||
## Monitoring LiteLLM
|
||||
|
||||
LiteLLM can be monitored in two ways: using the **LiteLLM SDK** (directly embedded in your Python application code for programmatic LLM calls) or the **LiteLLM Proxy Server** (a standalone server that acts as a centralized gateway for managing and routing LLM requests across your infrastructure).
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="LiteLLM SDK" label="LiteLLM SDK" default>
|
||||
|
||||
For more detailed info on instrumenting your LiteLLM SDK applications click [here](https://docs.litellm.ai/docs/observability/opentelemetry_integration).
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="No Code" label="No Code(Recommended)" default>
|
||||
|
||||
No-code auto-instrumentation is recommended for quick setup with minimal code changes. It's ideal when you want to get observability up and running without modifying your application code and are leveraging standard instrumentor libraries.
|
||||
|
||||
**Step 1:** Install the necessary packages in your Python environment.
|
||||
|
||||
```bash
|
||||
pip install \
|
||||
opentelemetry-api \
|
||||
opentelemetry-distro \
|
||||
opentelemetry-exporter-otlp \
|
||||
httpx \
|
||||
opentelemetry-instrumentation-httpx \
|
||||
litellm
|
||||
```
|
||||
|
||||
**Step 2:** Add Automatic Instrumentation
|
||||
|
||||
```bash
|
||||
opentelemetry-bootstrap --action=install
|
||||
```
|
||||
|
||||
**Step 3:** Instrument your LiteLLM SDK application
|
||||
|
||||
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
|
||||
|
||||
```python
|
||||
from litellm import litellm
|
||||
|
||||
litellm.callbacks = ["otel"]
|
||||
```
|
||||
|
||||
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
|
||||
|
||||
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
|
||||
|
||||
**Step 4:** Run an example
|
||||
|
||||
```python
|
||||
from litellm import completion, litellm
|
||||
|
||||
litellm.callbacks = ["otel"]
|
||||
|
||||
response = completion(
|
||||
model="openai/gpt-4o",
|
||||
messages=[{ "content": "What is SigNoz","role": "user"}]
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
|
||||
|
||||
**Step 5:** Run your application with auto-instrumentation
|
||||
|
||||
```bash
|
||||
OTEL_RESOURCE_ATTRIBUTES="service.name=<service_name>" \
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443" \
|
||||
OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>" \
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=grpc \
|
||||
OTEL_TRACES_EXPORTER=otlp \
|
||||
OTEL_METRICS_EXPORTER=otlp \
|
||||
OTEL_LOGS_EXPORTER=otlp \
|
||||
OTEL_PYTHON_LOG_CORRELATION=true \
|
||||
OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true \
|
||||
OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai \
|
||||
opentelemetry-instrument <your_run_command>
|
||||
```
|
||||
|
||||
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
|
||||
|
||||
> 📌 Note: We're using `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS=openai` in the run command to disable the OpenAI instrumentor for tracing. This avoids conflicts with LiteLLM's native telemetry/instrumentation, ensuring that telemetry is captured exclusively through LiteLLM's built-in instrumentation.
|
||||
|
||||
- **`<service_name>`** is the name of your service
|
||||
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
|
||||
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
|
||||
- Replace `<your_run_command>` with the actual command you would use to run your application. For example: `python main.py`
|
||||
|
||||
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
|
||||
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="Code" label="Code" default>
|
||||
|
||||
Code-based instrumentation gives you fine-grained control over your telemetry configuration. Use this approach when you need to customize resource attributes, sampling strategies, or integrate with existing observability infrastructure.
|
||||
|
||||
**Step 1:** Install the necessary packages in your Python environment.
|
||||
|
||||
```bash
|
||||
pip install \
|
||||
opentelemetry-api \
|
||||
opentelemetry-sdk \
|
||||
opentelemetry-exporter-otlp \
|
||||
opentelemetry-instrumentation-httpx \
|
||||
opentelemetry-instrumentation-system-metrics \
|
||||
litellm
|
||||
```
|
||||
|
||||
**Step 2:** Import the necessary modules in your Python application
|
||||
|
||||
**Traces:**
|
||||
|
||||
```python
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
```
|
||||
|
||||
**Logs:**
|
||||
|
||||
```python
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
import logging
|
||||
```
|
||||
|
||||
**Metrics:**
|
||||
|
||||
```python
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
|
||||
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
||||
```
|
||||
|
||||
**Step 3:** Set up the OpenTelemetry Tracer Provider to send traces directly to SigNoz Cloud
|
||||
|
||||
```python
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry import trace
|
||||
import os
|
||||
|
||||
resource = Resource.create({"service.name": "<service_name>"})
|
||||
provider = TracerProvider(resource=resource)
|
||||
span_exporter = OTLPSpanExporter(
|
||||
endpoint= os.getenv("OTEL_EXPORTER_TRACES_ENDPOINT"),
|
||||
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
|
||||
)
|
||||
processor = BatchSpanProcessor(span_exporter)
|
||||
provider.add_span_processor(processor)
|
||||
trace.set_tracer_provider(provider)
|
||||
```
|
||||
|
||||
- **`<service_name>`** is the name of your service
|
||||
- **`OTEL_EXPORTER_TRACES_ENDPOINT`** → SigNoz Cloud trace endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/traces`
|
||||
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
|
||||
|
||||
|
||||
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
|
||||
|
||||
|
||||
**Step 4**: Setup Logs
|
||||
|
||||
```python
|
||||
import logging
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
import os
|
||||
|
||||
resource = Resource.create({"service.name": "<service_name>"})
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
otlp_log_exporter = OTLPLogExporter(
|
||||
endpoint= os.getenv("OTEL_EXPORTER_LOGS_ENDPOINT"),
|
||||
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
|
||||
)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(otlp_log_exporter)
|
||||
)
|
||||
# Attach OTel logging handler to root logger
|
||||
handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
logging.basicConfig(level=logging.INFO, handlers=[handler])
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
```
|
||||
|
||||
- **`<service_name>`** is the name of your service
|
||||
- **`OTEL_EXPORTER_LOGS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/logs`
|
||||
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
|
||||
|
||||
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
|
||||
|
||||
|
||||
**Step 5**: Setup Metrics
|
||||
|
||||
```python
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
|
||||
import os
|
||||
|
||||
resource = Resource.create({"service.name": "<service-name>"})
|
||||
metric_exporter = OTLPMetricExporter(
|
||||
endpoint= os.getenv("OTEL_EXPORTER_METRICS_ENDPOINT"),
|
||||
headers={"signoz-ingestion-key": os.getenv("SIGNOZ_INGESTION_KEY")},
|
||||
)
|
||||
reader = PeriodicExportingMetricReader(metric_exporter)
|
||||
metric_provider = MeterProvider(metric_readers=[reader], resource=resource)
|
||||
metrics.set_meter_provider(metric_provider)
|
||||
|
||||
meter = metrics.get_meter(__name__)
|
||||
|
||||
# turn on out-of-the-box metrics
|
||||
SystemMetricsInstrumentor().instrument()
|
||||
HTTPXClientInstrumentor().instrument()
|
||||
```
|
||||
|
||||
- **`<service_name>`** is the name of your service
|
||||
- **`OTEL_EXPORTER_METRICS_ENDPOINT`** → SigNoz Cloud endpoint with appropriate [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint):`https://ingest.<region>.signoz.cloud:443/v1/metrics`
|
||||
- **`SIGNOZ_INGESTION_KEY`** → Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
|
||||
|
||||
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
|
||||
|
||||
|
||||
> 📌 Note: SystemMetricsInstrumentor provides system metrics (CPU, memory, etc.), and HTTPXClientInstrumentor provides outbound HTTP request metrics such as request duration. If you want to add custom metrics to your LiteLLM application, see [Python Custom Metrics](https://signoz.io/opentelemetry/python-custom-metrics/).
|
||||
|
||||
**Step 6:** Instrument your LiteLLM application
|
||||
|
||||
Initialize LiteLLM SDK instrumentation by calling `litellm.callbacks = ["otel"]`:
|
||||
|
||||
```python
|
||||
from litellm import litellm
|
||||
|
||||
litellm.callbacks = ["otel"]
|
||||
```
|
||||
|
||||
This call enables automatic tracing, logs, and metrics collection for all LiteLLM SDK calls in your application.
|
||||
|
||||
> 📌 Note: Ensure this is called before any LiteLLM related calls to properly configure instrumentation of your application
|
||||
|
||||
**Step 7:** Run an example
|
||||
|
||||
```python
|
||||
from litellm import completion, litellm
|
||||
|
||||
litellm.callbacks = ["otel"]
|
||||
|
||||
response = completion(
|
||||
model="openai/gpt-4o",
|
||||
messages=[{ "content": "What is SigNoz","role": "user"}]
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
> 📌 Note: LiteLLM supports a [variety of model providers](https://docs.litellm.ai/docs/providers) for LLMs. In this example, we're using OpenAI. Before running this code, ensure that you have set the environment variable `OPENAI_API_KEY` with your generated API key.
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## View Traces, Logs, and Metrics in SigNoz
|
||||
|
||||
Your LiteLLM commands should now automatically emit traces, logs, and metrics.
|
||||
|
||||
You should be able to view traces in Signoz Cloud under the traces tab:
|
||||
|
||||
|
||||
|
||||
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
|
||||
|
||||
|
||||
|
||||
You should be able to view logs in Signoz Cloud under the logs tab. You can also view logs by clicking on the “Related Logs” button in the trace view to see correlated logs:
|
||||
|
||||
|
||||
|
||||
When you click on any of these logs in SigNoz, you'll see a detailed view of the log, including attributes:
|
||||
|
||||
|
||||
|
||||
You should be able to see LiteLLM related metrics in Signoz Cloud under the metrics tab:
|
||||
|
||||
|
||||
|
||||
When you click on any of these metrics in SigNoz, you'll see a detailed view of the metric, including attributes:
|
||||
|
||||
|
||||
|
||||
## Dashboard
|
||||
|
||||
You can also check out our custom LiteLLM SDK dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-sdk-dashboard/) which provides specialized visualizations for monitoring your LiteLLM usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
|
||||
|
||||
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="LiteLLM Proxy Server" label="LiteLLM Proxy Server" default>
|
||||
|
||||
**Step 1:** Install the necessary packages in your Python environment.
|
||||
|
||||
```bash
|
||||
pip install opentelemetry-api \
|
||||
opentelemetry-sdk \
|
||||
opentelemetry-exporter-otlp \
|
||||
'litellm[proxy]'
|
||||
```
|
||||
|
||||
**Step 2:** Configure otel for the LiteLLM Proxy Server
|
||||
|
||||
Add the following to `config.yaml`:
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
callbacks: ['otel']
|
||||
```
|
||||
|
||||
**Step 3:** Set the following environment variables:
|
||||
|
||||
```bash
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT="https://ingest.<region>.signoz.cloud:443"
|
||||
export OTEL_EXPORTER_OTLP_HEADERS="signoz-ingestion-key=<your_ingestion_key>"
|
||||
export OTEL_EXPORTER_OTLP_PROTOCOL="grpc"
|
||||
export OTEL_TRACES_EXPORTER="otlp"
|
||||
export OTEL_METRICS_EXPORTER="otlp"
|
||||
export OTEL_LOGS_EXPORTER="otlp"
|
||||
```
|
||||
|
||||
> Note: OTLP gRPC requires `grpcio`. Install via `pip install "litellm[grpc]"` (or `grpcio`).
|
||||
|
||||
- Set the `<region>` to match your SigNoz Cloud [region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint)
|
||||
- Replace `<your_ingestion_key>` with your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/)
|
||||
|
||||
> 📌 Note: Using self-hosted SigNoz? Most steps are identical. To adapt this guide, update the endpoint and remove the ingestion key header as shown in [Cloud → Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted).
|
||||
|
||||
|
||||
**Step 4:** Run the proxy server using the config file:
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
Now any calls made through your LiteLLM proxy server will be traced and sent to SigNoz.
|
||||
|
||||
You should be able to view traces in Signoz Cloud under the traces tab:
|
||||
|
||||
|
||||
|
||||
When you click on a trace in SigNoz, you'll see a detailed view of the trace, including all associated spans, along with their events and attributes.
|
||||
|
||||
|
||||
|
||||
## Dashboard
|
||||
|
||||
You can also check out our custom LiteLLM Proxy dashboard [here](https://signoz.io/docs/dashboards/dashboard-templates/litellm-proxy-dashboard/) which provides specialized visualizations for monitoring your LiteLLM Proxy usage in applications. The dashboard includes pre-built charts specifically tailored for LLM usage, along with import instructions to get started quickly.
|
||||
|
||||
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -148,6 +148,51 @@ Example payload:
|
|||
|
||||
## Advanced Configuration
|
||||
|
||||
### Log Format
|
||||
|
||||
The Sumo Logic integration uses **NDJSON (newline-delimited JSON)** format by default. This format is optimal for Sumo Logic's parsing capabilities and allows Field Extraction Rules to work at ingest time.
|
||||
|
||||
#### NDJSON Format
|
||||
|
||||
Each log entry is sent as a separate line in the HTTP request:
|
||||
```
|
||||
{"id":"chatcmpl-1","model":"gpt-3.5-turbo","response_cost":0.0001,...}
|
||||
{"id":"chatcmpl-2","model":"gpt-4","response_cost":0.0003,...}
|
||||
{"id":"chatcmpl-3","model":"gpt-3.5-turbo","response_cost":0.0001,...}
|
||||
```
|
||||
|
||||
#### Benefits for Field Extraction Rules (FERs)
|
||||
|
||||
With NDJSON format, you can create Field Extraction Rules directly:
|
||||
|
||||
```
|
||||
_sourceCategory=litellm/logs
|
||||
| json field=_raw "model", "response_cost", "user" as model, cost, user
|
||||
```
|
||||
|
||||
**Before NDJSON** (with JSON array format):
|
||||
- Required `parse regex ... multi` workaround
|
||||
- FERs couldn't parse at ingest time
|
||||
- Query-time parsing impacted dashboard performance
|
||||
|
||||
**After NDJSON**:
|
||||
- ✅ FERs parse fields at ingest time
|
||||
- ✅ No query-time workarounds needed
|
||||
- ✅ Better dashboard performance
|
||||
- ✅ Simpler query syntax
|
||||
|
||||
#### Changing the Log Format (Advanced)
|
||||
|
||||
If you need to change the log format (not recommended for Sumo Logic):
|
||||
|
||||
```yaml
|
||||
callback_settings:
|
||||
sumologic:
|
||||
callback_type: generic_api
|
||||
callback_name: sumologic
|
||||
log_format: json_array # Override to use JSON array instead
|
||||
```
|
||||
|
||||
### Batching Settings
|
||||
|
||||
Control how LiteLLM batches logs before sending to Sumo Logic:
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ model_list:
|
|||
aws_region_name: us-west-2
|
||||
aws_session_name: "my-test-session"
|
||||
aws_role_name: "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
|
||||
aws_web_identity_token: "oidc/circleci_v2/"
|
||||
aws_web_identity_token: "oidc/example-provider/"
|
||||
```
|
||||
|
||||
#### Amazon IAM Role Configuration for CircleCI v2 -> Bedrock
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ model_list:
|
|||
litellm_params:
|
||||
model: vertex_ai/gemini-1.0-pro
|
||||
vertex_project: adroit-crow-413218
|
||||
vertex_region: us-central1
|
||||
vertex_location: us-central1
|
||||
vertex_credentials: /path/to/credentials.json
|
||||
use_in_pass_through: true # 👈 KEY CHANGE
|
||||
```
|
||||
|
|
@ -57,9 +57,9 @@ model_list:
|
|||
<TabItem value="yaml" label="Set in config.yaml">
|
||||
|
||||
```yaml
|
||||
default_vertex_config:
|
||||
default_vertex_config:
|
||||
vertex_project: adroit-crow-413218
|
||||
vertex_region: us-central1
|
||||
vertex_location: us-central1
|
||||
vertex_credentials: /path/to/credentials.json
|
||||
```
|
||||
</TabItem>
|
||||
|
|
@ -461,3 +461,48 @@ generateContent();
|
|||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Using Anthropic Beta Features on Vertex AI
|
||||
|
||||
When using Anthropic models via Vertex AI passthrough (e.g., Claude on Vertex), you can enable Anthropic beta features like extended context windows.
|
||||
|
||||
The `anthropic-beta` header is automatically forwarded to Vertex AI when calling Anthropic models.
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-east5/publishers/anthropic/models/claude-3-5-sonnet:rawPredict \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "anthropic-beta: context-1m-2025-08-07" \
|
||||
-d '{
|
||||
"anthropic_version": "vertex-2023-10-16",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 500
|
||||
}'
|
||||
```
|
||||
|
||||
### Forwarding Custom Headers with `x-pass-` Prefix
|
||||
|
||||
You can forward any custom header to the provider by prefixing it with `x-pass-`. The prefix is stripped before the header is sent to the provider.
|
||||
|
||||
For example:
|
||||
- `x-pass-anthropic-beta: value` becomes `anthropic-beta: value`
|
||||
- `x-pass-custom-header: value` becomes `custom-header: value`
|
||||
|
||||
This is useful when you need to send provider-specific headers that aren't in the default allowlist.
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-east5/publishers/anthropic/models/claude-3-5-sonnet:rawPredict \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "x-pass-anthropic-beta: context-1m-2025-08-07" \
|
||||
-H "x-pass-custom-feature: enabled" \
|
||||
-d '{
|
||||
"anthropic_version": "vertex-2023-10-16",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"max_tokens": 500
|
||||
}'
|
||||
```
|
||||
|
||||
:::info
|
||||
The `x-pass-` prefix works for all LLM pass-through endpoints, not just Vertex AI.
|
||||
:::
|
||||
|
|
|
|||
109
docs/my-website/docs/providers/abliteration.md
Normal file
109
docs/my-website/docs/providers/abliteration.md
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
# Abliteration
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. |
|
||||
| Provider Route on LiteLLM | `abliteration/` |
|
||||
| Link to Provider Doc | [Abliteration](https://abliteration.ai) |
|
||||
| Base URL | `https://api.abliteration.ai/v1` |
|
||||
| Supported Operations | [`/chat/completions`](#sample-usage) |
|
||||
|
||||
<br />
|
||||
|
||||
## Required Variables
|
||||
|
||||
```python showLineNumbers title="Environment Variables"
|
||||
os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
|
||||
```python showLineNumbers title="Abliteration Completion"
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
os.environ["ABLITERATION_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="abliteration/abliterated-model",
|
||||
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Sample Usage - Streaming
|
||||
|
||||
```python showLineNumbers title="Abliteration Streaming Completion"
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
os.environ["ABLITERATION_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="abliteration/abliterated-model",
|
||||
messages=[{"role": "user", "content": "Stream a short reply"}],
|
||||
stream=True,
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Usage with LiteLLM Proxy Server
|
||||
|
||||
1. Add the model to your proxy config:
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: abliteration-chat
|
||||
litellm_params:
|
||||
model: abliteration/abliterated-model
|
||||
api_key: os.environ/ABLITERATION_API_KEY
|
||||
```
|
||||
|
||||
2. Start the proxy:
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
## Direct API Usage (Bearer Token)
|
||||
|
||||
Use the environment variable as a Bearer token against the OpenAI-compatible endpoint:
|
||||
`https://api.abliteration.ai/v1/chat/completions`.
|
||||
|
||||
```bash showLineNumbers title="cURL"
|
||||
export ABLITERATION_API_KEY=""
|
||||
curl https://api.abliteration.ai/v1/chat/completions \
|
||||
-H "Authorization: Bearer ${ABLITERATION_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "abliterated-model",
|
||||
"messages": [{"role": "user", "content": "Hello from Abliteration"}]
|
||||
}'
|
||||
```
|
||||
|
||||
```python showLineNumbers title="Python (requests)"
|
||||
import os
|
||||
import requests
|
||||
|
||||
api_key = os.environ["ABLITERATION_API_KEY"]
|
||||
|
||||
response = requests.post(
|
||||
"https://api.abliteration.ai/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": "abliterated-model",
|
||||
"messages": [{"role": "user", "content": "Hello from Abliteration"}],
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
print(response.json())
|
||||
```
|
||||
|
|
@ -444,7 +444,7 @@ Here's what a sample Raw Request from LiteLLM for Anthropic Context Caching look
|
|||
POST Request Sent from LiteLLM:
|
||||
curl -X POST \
|
||||
https://api.anthropic.com/v1/messages \
|
||||
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' -H 'anthropic-beta: prompt-caching-2024-07-31' \
|
||||
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' \
|
||||
-d '{'model': 'claude-3-5-sonnet-20240620', [
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -472,6 +472,8 @@ https://api.anthropic.com/v1/messages \
|
|||
"max_tokens": 10
|
||||
}'
|
||||
```
|
||||
|
||||
**Note:** Anthropic no longer requires the `anthropic-beta: prompt-caching-2024-07-31` header. Prompt caching now works automatically when you use `cache_control` in your messages.
|
||||
:::
|
||||
|
||||
### Caching - Large Context Caching
|
||||
|
|
@ -1690,9 +1692,9 @@ Assistant:
|
|||
```
|
||||
|
||||
|
||||
## Usage - PDF
|
||||
## Usage - PDF
|
||||
|
||||
Pass base64 encoded PDF files to Anthropic models using the `image_url` field.
|
||||
Pass base64 encoded PDF files to Anthropic models using the `file` content type with a `file_data` field.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
|
|
|||
|
|
@ -1,43 +1,46 @@
|
|||
# Anthropic Tool Search
|
||||
# Tool Search
|
||||
|
||||
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
|
||||
|
||||
## Supported Providers
|
||||
|
||||
| Provider | Chat Completions API | Messages API |
|
||||
|----------|---------------------|--------------|
|
||||
| **Anthropic API** | ✅ | ✅ |
|
||||
| **Azure Anthropic** (Microsoft Foundry) | ✅ | ✅ |
|
||||
| **Google Cloud Vertex AI** | ✅ | ✅ |
|
||||
| **Amazon Bedrock** | ✅ (Invoke API only, Opus 4.5 only) | ✅ (Invoke API only, Opus 4.5 only) |
|
||||
|
||||
|
||||
## Benefits
|
||||
|
||||
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
|
||||
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
|
||||
- **On-demand loading**: Tools are only loaded when Claude needs them
|
||||
|
||||
## Supported Models
|
||||
|
||||
Tool search is available on:
|
||||
- Claude Opus 4.5
|
||||
- Claude Sonnet 4.5
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
- Anthropic API (direct)
|
||||
- Azure Anthropic (Microsoft Foundry)
|
||||
- Google Cloud Vertex AI
|
||||
- Amazon Bedrock (invoke API only, not converse API)
|
||||
|
||||
## Tool Search Variants
|
||||
|
||||
LiteLLM supports both tool search variants:
|
||||
|
||||
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
|
||||
|
||||
Claude constructs regex patterns to search for tools.
|
||||
Claude constructs regex patterns to search for tools. Best for exact pattern matching (faster).
|
||||
|
||||
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
|
||||
|
||||
Claude uses natural language queries to search for tools using the BM25 algorithm.
|
||||
Claude uses natural language queries to search for tools using the BM25 algorithm. Best for natural language semantic search.
|
||||
|
||||
## Quick Start
|
||||
**Note**: BM25 variant is not supported on Bedrock.
|
||||
|
||||
### Basic Example with Regex Tool Search
|
||||
---
|
||||
|
||||
```python
|
||||
## Chat Completions API
|
||||
|
||||
### SDK Usage
|
||||
|
||||
#### Basic Example with Regex Tool Search
|
||||
|
||||
```python showLineNumbers title="Basic Tool Search Example"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -70,26 +73,6 @@ response = litellm.completion(
|
|||
}
|
||||
},
|
||||
"defer_loading": True # Mark for deferred loading
|
||||
},
|
||||
# Another deferred tool
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_files",
|
||||
"description": "Search through files in the workspace",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"file_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
|
|
@ -97,9 +80,9 @@ response = litellm.completion(
|
|||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### BM25 Tool Search Example
|
||||
#### BM25 Tool Search Example
|
||||
|
||||
```python
|
||||
```python showLineNumbers title="BM25 Tool Search"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -134,9 +117,9 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
## Using with Azure Anthropic
|
||||
#### Azure Anthropic Example
|
||||
|
||||
```python
|
||||
```python showLineNumbers title="Azure Anthropic Tool Search"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -170,9 +153,9 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
## Using with Vertex AI
|
||||
#### Vertex AI Example
|
||||
|
||||
```python
|
||||
```python showLineNumbers title="Vertex AI Tool Search"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -192,11 +175,9 @@ response = litellm.completion(
|
|||
)
|
||||
```
|
||||
|
||||
## Streaming Support
|
||||
#### Streaming Support
|
||||
|
||||
Tool search works with streaming:
|
||||
|
||||
```python
|
||||
```python showLineNumbers title="Streaming with Tool Search"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
|
|
@ -233,13 +214,13 @@ for chunk in response:
|
|||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
## LiteLLM Proxy
|
||||
### AI Gateway Usage
|
||||
|
||||
Tool search works automatically through the LiteLLM proxy:
|
||||
Tool search works automatically through the LiteLLM proxy.
|
||||
|
||||
### Proxy Config
|
||||
#### Proxy Configuration
|
||||
|
||||
```yaml
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
|
|
@ -247,18 +228,19 @@ model_list:
|
|||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
### Client Request
|
||||
#### Client Request
|
||||
|
||||
```python
|
||||
import openai
|
||||
```python showLineNumbers title="Client Request via Proxy"
|
||||
from anthropic import Anthropic
|
||||
|
||||
client = openai.OpenAI(
|
||||
client = Anthropic(
|
||||
api_key="your-litellm-proxy-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
response = client.messages.create(
|
||||
model="claude-sonnet",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather?"}
|
||||
],
|
||||
|
|
@ -268,17 +250,14 @@ response = client.chat.completions.create(
|
|||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
|
|
@ -286,127 +265,278 @@ response = client.chat.completions.create(
|
|||
)
|
||||
```
|
||||
|
||||
## Important Notes
|
||||
---
|
||||
|
||||
### Beta Header
|
||||
## Messages API
|
||||
|
||||
LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
|
||||
The Messages API provides native Anthropic-style tool search support via the `litellm.anthropic.messages` interface.
|
||||
|
||||
- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
|
||||
- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
|
||||
- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
|
||||
### SDK Usage
|
||||
|
||||
You don't need to manually specify beta headers—LiteLLM handles this automatically.
|
||||
#### Basic Example
|
||||
|
||||
### Deferred Loading
|
||||
```python showLineNumbers title="Messages API - Basic Tool Search"
|
||||
import litellm
|
||||
|
||||
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
|
||||
- At least one tool must be non-deferred (the tool search tool itself)
|
||||
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
|
||||
|
||||
### Tool Descriptions
|
||||
|
||||
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
|
||||
- Tool names
|
||||
- Tool descriptions
|
||||
- Argument names
|
||||
- Argument descriptions
|
||||
|
||||
### Usage Tracking
|
||||
|
||||
Tool search requests are tracked in the usage object:
|
||||
|
||||
```python
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Search for tools"}],
|
||||
tools=[...]
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in San Francisco?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a location",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
|
||||
# Check tool search usage
|
||||
if response.usage.server_tool_use:
|
||||
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
#### Azure Anthropic Messages Example
|
||||
|
||||
### All Tools Deferred
|
||||
```python showLineNumbers title="Azure Anthropic Messages API"
|
||||
import litellm
|
||||
|
||||
```python
|
||||
# ❌ This will fail - at least one tool must be non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
|
||||
# ✅ Correct - tool search tool is non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="azure_anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the stock price of Apple?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_stock_price",
|
||||
"description": "Get the current stock price for a ticker symbol",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ticker": {
|
||||
"type": "string",
|
||||
"description": "The stock ticker symbol, e.g. AAPL"
|
||||
}
|
||||
},
|
||||
"required": ["ticker"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
```
|
||||
|
||||
### Missing Tool Definition
|
||||
#### Vertex AI Messages Example
|
||||
|
||||
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
|
||||
```python showLineNumbers title="Vertex AI Messages API"
|
||||
import litellm
|
||||
|
||||
## Best Practices
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="vertex_ai/claude-sonnet-4@20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Search the web for information about AI"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
},
|
||||
{
|
||||
"name": "search_web",
|
||||
"description": "Search the web for information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The search query"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
|
||||
)
|
||||
```
|
||||
|
||||
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
|
||||
#### Bedrock Messages Example
|
||||
|
||||
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
|
||||
```python showLineNumbers title="Bedrock Messages API (Invoke)"
|
||||
import litellm
|
||||
|
||||
3. **Choose the right variant**:
|
||||
- Use **regex** for exact pattern matching (faster)
|
||||
- Use **BM25** for natural language semantic search
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="bedrock/invoke/anthropic.claude-opus-4-20250514-v1:0",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
extra_headers={"anthropic-beta": "tool-search-tool-2025-10-19"}
|
||||
)
|
||||
```
|
||||
|
||||
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
|
||||
#### Streaming Support
|
||||
|
||||
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
|
||||
```python showLineNumbers title="Messages API - Streaming"
|
||||
import litellm
|
||||
import json
|
||||
|
||||
## When to Use Tool Search
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather in Tokyo?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
max_tokens=1024,
|
||||
stream=True,
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
|
||||
**Good use cases:**
|
||||
- 10+ tools available in your system
|
||||
- Tool definitions consuming >10K tokens
|
||||
- Experiencing tool selection accuracy issues
|
||||
- Building systems with multiple tool categories
|
||||
- Tool library growing over time
|
||||
async for chunk in response:
|
||||
if isinstance(chunk, bytes):
|
||||
chunk_str = chunk.decode("utf-8")
|
||||
for line in chunk_str.split("\n"):
|
||||
if line.startswith("data: "):
|
||||
try:
|
||||
json_data = json.loads(line[6:])
|
||||
print(json_data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
```
|
||||
|
||||
**When traditional tool calling is better:**
|
||||
- Less than 10 tools total
|
||||
- All tools are frequently used
|
||||
- Very small tool definitions (\<100 tokens total)
|
||||
### AI Gateway Usage
|
||||
|
||||
## Limitations
|
||||
Configure the proxy to use Messages API endpoints.
|
||||
|
||||
- Not compatible with tool use examples
|
||||
- Requires Claude Opus 4.5 or Sonnet 4.5
|
||||
- On Bedrock, only available via invoke API (not converse API)
|
||||
- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
|
||||
- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
|
||||
- Maximum 10,000 tools in catalog
|
||||
- Returns 3-5 most relevant tools per search
|
||||
#### Proxy Configuration
|
||||
|
||||
### Bedrock-Specific Notes
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: claude-sonnet-messages
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-20250514
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
When using Bedrock's Invoke API:
|
||||
- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
|
||||
- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
|
||||
- Tool search is only available for Claude Opus 4.5 models
|
||||
#### Client Request
|
||||
|
||||
```python showLineNumbers title="Client Request via Proxy (Messages API)"
|
||||
from anthropic import Anthropic
|
||||
|
||||
client = Anthropic(
|
||||
api_key="your-litellm-proxy-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.messages.create(
|
||||
model="claude-sonnet-messages",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What's the weather?"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
|
||||
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)
|
||||
|
||||
|
|
|
|||
129
docs/my-website/docs/providers/apertis.md
Normal file
129
docs/my-website/docs/providers/apertis.md
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
# Apertis AI (Stima API)
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Apertis AI (formerly Stima API) is a unified API platform providing access to 430+ AI models through a single interface, with cost savings of up to 50%. |
|
||||
| Provider Route on LiteLLM | `apertis/` |
|
||||
| Link to Provider Doc | [Apertis AI Website ↗](https://api.stima.tech) |
|
||||
| Base URL | `https://api.stima.tech/v1` |
|
||||
| Supported Operations | [`/chat/completions`](#sample-usage) |
|
||||
|
||||
<br />
|
||||
|
||||
## What is Apertis AI?
|
||||
|
||||
Apertis AI is a unified API platform that lets developers:
|
||||
- **Access 430+ AI Models**: All models through a single API
|
||||
- **Save 50% on Costs**: Competitive pricing with significant discounts
|
||||
- **Unified Billing**: Single bill for all model usage
|
||||
- **Quick Setup**: Start with just $2 registration
|
||||
- **GitHub Integration**: Link with your GitHub account
|
||||
|
||||
## Required Variables
|
||||
|
||||
```python showLineNumbers title="Environment Variables"
|
||||
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
|
||||
```
|
||||
|
||||
Get your Apertis AI API key from [api.stima.tech](https://api.stima.tech).
|
||||
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
||||
```python showLineNumbers title="Apertis AI Non-streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
|
||||
|
||||
messages = [{"content": "What is the capital of France?", "role": "user"}]
|
||||
|
||||
# Apertis AI call
|
||||
response = completion(
|
||||
model="apertis/model-name", # Replace with actual model name
|
||||
messages=messages
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```python showLineNumbers title="Apertis AI Streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
|
||||
|
||||
messages = [{"content": "Write a short poem about AI", "role": "user"}]
|
||||
|
||||
# Apertis AI call with streaming
|
||||
response = completion(
|
||||
model="apertis/model-name", # Replace with actual model name
|
||||
messages=messages,
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Proxy Server
|
||||
|
||||
### 1. Save key in your environment
|
||||
|
||||
```bash
|
||||
export STIMA_API_KEY=""
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: apertis-model
|
||||
litellm_params:
|
||||
model: apertis/model-name # Replace with actual model name
|
||||
api_key: os.environ/STIMA_API_KEY
|
||||
```
|
||||
|
||||
## Supported OpenAI Parameters
|
||||
|
||||
Apertis AI supports all standard OpenAI-compatible parameters:
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
|
||||
| `model` | string | **Required**. Model ID from 430+ available models |
|
||||
| `stream` | boolean | Optional. Enable streaming responses |
|
||||
| `temperature` | float | Optional. Sampling temperature |
|
||||
| `top_p` | float | Optional. Nucleus sampling parameter |
|
||||
| `max_tokens` | integer | Optional. Maximum tokens to generate |
|
||||
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
|
||||
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
|
||||
| `stop` | string/array | Optional. Stop sequences |
|
||||
| `tools` | array | Optional. List of available tools/functions |
|
||||
| `tool_choice` | string/object | Optional. Control tool/function calling |
|
||||
|
||||
## Cost Benefits
|
||||
|
||||
Apertis AI offers significant cost advantages:
|
||||
- **50% Cost Savings**: Save money compared to direct provider costs
|
||||
- **Unified Billing**: Single invoice for all your AI model usage
|
||||
- **Low Entry**: Start with just $2 registration
|
||||
|
||||
## Model Availability
|
||||
|
||||
With access to 430+ AI models, Apertis AI provides:
|
||||
- Multiple providers through one API
|
||||
- Latest model releases
|
||||
- Various model types (text, image, video)
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Apertis AI Website](https://api.stima.tech)
|
||||
- [Apertis AI Enterprise](https://api.stima.tech/enterprise)
|
||||
364
docs/my-website/docs/providers/aws_polly.md
Normal file
364
docs/my-website/docs/providers/aws_polly.md
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
# AWS Polly Text to Speech (tts)
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Convert text to natural-sounding speech using AWS Polly's neural and standard TTS engines |
|
||||
| Provider Route on LiteLLM | `aws_polly/` |
|
||||
| Supported Operations | `/audio/speech` |
|
||||
| Link to Provider Doc | [AWS Polly SynthesizeSpeech ↗](https://docs.aws.amazon.com/polly/latest/dg/API_SynthesizeSpeech.html) |
|
||||
|
||||
## Quick Start
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="SDK Usage"
|
||||
import litellm
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
# Set environment variables
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = ""
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
|
||||
os.environ["AWS_REGION_NAME"] = "us-east-1"
|
||||
|
||||
# AWS Polly call
|
||||
speech_file_path = Path(__file__).parent / "speech.mp3"
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="the quick brown fox jumped over the lazy dogs",
|
||||
)
|
||||
response.stream_to_file(speech_file_path)
|
||||
```
|
||||
|
||||
### **LiteLLM PROXY**
|
||||
|
||||
```yaml showLineNumbers title="proxy_config.yaml"
|
||||
model_list:
|
||||
- model_name: polly-neural
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
aws_access_key_id: "os.environ/AWS_ACCESS_KEY_ID"
|
||||
aws_secret_access_key: "os.environ/AWS_SECRET_ACCESS_KEY"
|
||||
aws_region_name: "us-east-1"
|
||||
```
|
||||
|
||||
## Polly Engines
|
||||
|
||||
AWS Polly supports different speech synthesis engines. Specify the engine in the model name:
|
||||
|
||||
| Model | Engine | Cost (per 1M chars) | Description |
|
||||
|-------|--------|---------------------|-------------|
|
||||
| `aws_polly/standard` | Standard | $4.00 | Original Polly voices, faster and lowest cost |
|
||||
| `aws_polly/neural` | Neural | $16.00 | More natural, human-like speech (recommended) |
|
||||
| `aws_polly/generative` | Generative | $30.00 | Most expressive, highest quality (limited voices) |
|
||||
| `aws_polly/long-form` | Long-form | $100.00 | Optimized for long content like articles |
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="Using Different Engines"
|
||||
import litellm
|
||||
|
||||
# Neural engine (recommended)
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello world",
|
||||
)
|
||||
|
||||
# Standard engine (lower cost)
|
||||
response = litellm.speech(
|
||||
model="aws_polly/standard",
|
||||
voice="Joanna",
|
||||
input="Hello world",
|
||||
)
|
||||
|
||||
# Generative engine (highest quality)
|
||||
response = litellm.speech(
|
||||
model="aws_polly/generative",
|
||||
voice="Matthew",
|
||||
input="Hello world",
|
||||
)
|
||||
```
|
||||
|
||||
### **LiteLLM PROXY**
|
||||
|
||||
```yaml showLineNumbers title="proxy_config.yaml"
|
||||
model_list:
|
||||
- model_name: polly-neural
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
aws_region_name: "us-east-1"
|
||||
- model_name: polly-standard
|
||||
litellm_params:
|
||||
model: aws_polly/standard
|
||||
aws_region_name: "us-east-1"
|
||||
- model_name: polly-generative
|
||||
litellm_params:
|
||||
model: aws_polly/generative
|
||||
aws_region_name: "us-east-1"
|
||||
```
|
||||
|
||||
## Available Voices
|
||||
|
||||
### Native Polly Voices
|
||||
|
||||
AWS Polly has many voices across different languages. Here are popular US English voices:
|
||||
|
||||
| Voice | Gender | Engine Support |
|
||||
|-------|--------|----------------|
|
||||
| `Joanna` | Female | Neural, Standard |
|
||||
| `Matthew` | Male | Neural, Standard, Generative |
|
||||
| `Ivy` | Female (child) | Neural, Standard |
|
||||
| `Kendra` | Female | Neural, Standard |
|
||||
| `Amy` | Female (British) | Neural, Standard |
|
||||
| `Brian` | Male (British) | Neural, Standard |
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="Using Native Polly Voices"
|
||||
import litellm
|
||||
|
||||
# US English female
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello from Joanna",
|
||||
)
|
||||
|
||||
# US English male
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Matthew",
|
||||
input="Hello from Matthew",
|
||||
)
|
||||
|
||||
# British English female
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Amy",
|
||||
input="Hello from Amy",
|
||||
)
|
||||
```
|
||||
|
||||
### **LiteLLM PROXY**
|
||||
|
||||
```yaml showLineNumbers title="proxy_config.yaml"
|
||||
model_list:
|
||||
- model_name: polly-joanna
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
voice: "Joanna"
|
||||
aws_region_name: "us-east-1"
|
||||
- model_name: polly-matthew
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
voice: "Matthew"
|
||||
aws_region_name: "us-east-1"
|
||||
```
|
||||
|
||||
### OpenAI Voice Mappings
|
||||
|
||||
LiteLLM also supports OpenAI voice names, which are automatically mapped to Polly voices:
|
||||
|
||||
| OpenAI Voice | Maps to Polly Voice |
|
||||
|--------------|---------------------|
|
||||
| `alloy` | Joanna |
|
||||
| `echo` | Matthew |
|
||||
| `fable` | Amy |
|
||||
| `onyx` | Brian |
|
||||
| `nova` | Ivy |
|
||||
| `shimmer` | Kendra |
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="Using OpenAI Voice Names"
|
||||
import litellm
|
||||
|
||||
# These are equivalent
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="alloy", # Maps to Joanna
|
||||
input="Hello world",
|
||||
)
|
||||
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna", # Native Polly voice
|
||||
input="Hello world",
|
||||
)
|
||||
```
|
||||
|
||||
## SSML Support
|
||||
|
||||
AWS Polly supports SSML (Speech Synthesis Markup Language) for advanced control over speech output. LiteLLM automatically detects SSML input.
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="SSML Example"
|
||||
import litellm
|
||||
|
||||
ssml_input = """
|
||||
<speak>
|
||||
Hello, <break time="500ms"/>
|
||||
this is a test with <emphasis level="strong">emphasis</emphasis>
|
||||
and <prosody rate="slow">slower speech</prosody>.
|
||||
</speak>
|
||||
"""
|
||||
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input=ssml_input,
|
||||
)
|
||||
```
|
||||
|
||||
### **LiteLLM PROXY**
|
||||
|
||||
```bash showLineNumbers title="cURL Request with SSML"
|
||||
curl -X POST http://localhost:4000/v1/audio/speech \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "polly-neural",
|
||||
"voice": "Joanna",
|
||||
"input": "<speak>Hello <break time=\"500ms\"/> world</speak>"
|
||||
}' \
|
||||
--output speech.mp3
|
||||
```
|
||||
|
||||
## Supported Parameters
|
||||
|
||||
```python showLineNumbers title="All Parameters"
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna", # Required: Voice selection
|
||||
input="text to convert", # Required: Input text (or SSML)
|
||||
response_format="mp3", # Optional: mp3, ogg_vorbis, pcm
|
||||
|
||||
# AWS-specific parameters
|
||||
language_code="en-US", # Optional: Language code
|
||||
sample_rate="22050", # Optional: Sample rate in Hz
|
||||
)
|
||||
```
|
||||
|
||||
## Response Formats
|
||||
|
||||
| Format | Description |
|
||||
|--------|-------------|
|
||||
| `mp3` | MP3 audio (default) |
|
||||
| `ogg_vorbis` | Ogg Vorbis audio |
|
||||
| `pcm` | Raw PCM audio |
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="Different Response Formats"
|
||||
import litellm
|
||||
|
||||
# MP3 (default)
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
response_format="mp3",
|
||||
)
|
||||
|
||||
# Ogg Vorbis
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
response_format="ogg_vorbis",
|
||||
)
|
||||
```
|
||||
|
||||
## AWS Authentication
|
||||
|
||||
LiteLLM supports multiple AWS authentication methods.
|
||||
|
||||
### **LiteLLM SDK**
|
||||
|
||||
```python showLineNumbers title="Authentication Options"
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Option 1: Environment variables (recommended)
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key"
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key"
|
||||
os.environ["AWS_REGION_NAME"] = "us-east-1"
|
||||
|
||||
response = litellm.speech(model="aws_polly/neural", voice="Joanna", input="Hello")
|
||||
|
||||
# Option 2: Pass credentials directly
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
aws_access_key_id="your-access-key",
|
||||
aws_secret_access_key="your-secret-key",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
# Option 3: IAM Role (when running on AWS)
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
# Option 4: AWS Profile
|
||||
response = litellm.speech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
aws_profile_name="my-profile",
|
||||
)
|
||||
```
|
||||
|
||||
### **LiteLLM PROXY**
|
||||
|
||||
```yaml showLineNumbers title="proxy_config.yaml"
|
||||
model_list:
|
||||
# Using environment variables
|
||||
- model_name: polly-neural
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
aws_access_key_id: "os.environ/AWS_ACCESS_KEY_ID"
|
||||
aws_secret_access_key: "os.environ/AWS_SECRET_ACCESS_KEY"
|
||||
aws_region_name: "us-east-1"
|
||||
|
||||
# Using IAM Role (when proxy runs on AWS)
|
||||
- model_name: polly-neural-iam
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
aws_region_name: "us-east-1"
|
||||
|
||||
# Using AWS Profile
|
||||
- model_name: polly-neural-profile
|
||||
litellm_params:
|
||||
model: aws_polly/neural
|
||||
aws_profile_name: "my-profile"
|
||||
```
|
||||
|
||||
## Async Support
|
||||
|
||||
```python showLineNumbers title="Async Usage"
|
||||
import litellm
|
||||
import asyncio
|
||||
|
||||
async def main():
|
||||
response = await litellm.aspeech(
|
||||
model="aws_polly/neural",
|
||||
voice="Joanna",
|
||||
input="Hello from async AWS Polly",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
with open("output.mp3", "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
232
docs/my-website/docs/providers/azure_ai/azure_model_router.md
Normal file
232
docs/my-website/docs/providers/azure_ai/azure_model_router.md
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
# Azure Model Router
|
||||
|
||||
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
|
||||
|
||||
## Key Features
|
||||
|
||||
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
|
||||
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
|
||||
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
|
||||
|
||||
## LiteLLM Python SDK
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import os
|
||||
|
||||
response = litellm.completion(
|
||||
model="azure_ai/azure-model-router",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Streaming with Usage Tracking
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import os
|
||||
|
||||
response = await litellm.acompletion(
|
||||
model="azure_ai/azure-model-router",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
async for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## LiteLLM Proxy (AI Gateway)
|
||||
|
||||
### config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: azure-model-router
|
||||
litellm_params:
|
||||
model: azure_ai/azure-model-router
|
||||
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
|
||||
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
|
||||
```
|
||||
|
||||
### Start Proxy
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
### Test Request
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:4000/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "azure-model-router",
|
||||
"messages": [{"role": "user", "content": "Hello!"}]
|
||||
}'
|
||||
```
|
||||
|
||||
## Add Azure Model Router via LiteLLM UI
|
||||
|
||||
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
|
||||
|
||||
### Select Provider
|
||||
|
||||
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
|
||||
|
||||
#### Navigate to Models Page
|
||||
|
||||
|
||||
|
||||
#### Click Provider Dropdown
|
||||
|
||||
|
||||
|
||||
#### Choose Azure AI Foundry
|
||||
|
||||
|
||||
|
||||
### Configure Model Name
|
||||
|
||||
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
|
||||
|
||||
#### Click Model Name Field
|
||||
|
||||
|
||||
|
||||
#### Select Custom Model Name
|
||||
|
||||
|
||||
|
||||
#### Enter LiteLLM Model Name
|
||||
|
||||
|
||||
|
||||
#### Click Custom Model Name Field
|
||||
|
||||
|
||||
|
||||
#### Type Model Prefix
|
||||
|
||||
Type `azure_ai/` as the prefix.
|
||||
|
||||
|
||||
|
||||
#### Copy Model Name from Azure Portal
|
||||
|
||||
Switch to Azure AI Foundry and copy your model router deployment name.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#### Paste Model Name
|
||||
|
||||
Paste to get `azure_ai/azure-model-router`.
|
||||
|
||||
|
||||
|
||||
### Configure API Base and Key
|
||||
|
||||
Copy the endpoint URL and API key from Azure portal.
|
||||
|
||||
#### Copy API Base URL from Azure
|
||||
|
||||
|
||||
|
||||
#### Enter API Base in LiteLLM
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#### Copy API Key from Azure
|
||||
|
||||
|
||||
|
||||
#### Enter API Key in LiteLLM
|
||||
|
||||
|
||||
|
||||
### Test and Add Model
|
||||
|
||||
Verify your configuration works and save the model.
|
||||
|
||||
#### Test Connection
|
||||
|
||||
|
||||
|
||||
#### Close Test Dialog
|
||||
|
||||
|
||||
|
||||
#### Add Model
|
||||
|
||||
|
||||
|
||||
### Verify in Playground
|
||||
|
||||
Test your model and verify cost tracking is working.
|
||||
|
||||
#### Open Playground
|
||||
|
||||
|
||||
|
||||
#### Select Model
|
||||
|
||||
|
||||
|
||||
#### Send Test Message
|
||||
|
||||
|
||||
|
||||
#### View Logs
|
||||
|
||||
|
||||
|
||||
#### Verify Cost Tracking
|
||||
|
||||
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
|
||||
|
||||
|
||||
|
||||
## Cost Tracking
|
||||
|
||||
LiteLLM automatically handles cost tracking for Azure Model Router by:
|
||||
|
||||
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
|
||||
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
|
||||
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
|
||||
|
||||
### Example Response with Cost
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="azure_ai/azure-model-router",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key="your-api-key",
|
||||
)
|
||||
|
||||
# The response will show the actual model used
|
||||
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
|
||||
|
||||
# Get cost
|
||||
from litellm import completion_cost
|
||||
cost = completion_cost(completion_response=response)
|
||||
print(f"Cost: ${cost}")
|
||||
```
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 254 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 240 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 239 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 247 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 272 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 307 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 248 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 315 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 492 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 562 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 265 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 471 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 225 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 219 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 485 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 227 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 236 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 214 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 232 KiB |
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue