scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'upstream/main' into feat/uv-migration

Browse source 
# Conflicts:
#	.circleci/config.yml
#	.github/workflows/test-litellm-matrix.yml
#	.github/workflows/test-proxy-e2e-azure-batches.yml
#	AGENTS.md
#	ci_cd/security_scans.sh
#	docker/Dockerfile.non_root
#	poetry.lock
#	pyproject.toml
#	requirements.txt
#	tests/proxy_e2e_azure_batches_tests/validate_e2e_setup.py
#	tests/test_litellm/proxy/db/test_check_migration.py
This commit is contained in:
user 2026-04-08 17:33:24 +00:00
commit a733c74a68
No known key found for this signature in database
834 changed files with 20013 additions and 9550 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
.circleci/config.yml
View file

@ -2878,112 +2878,6 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_azure_batches_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.12
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.12 -y
conda activate myenv
python --version
- run:
name: Prepare uv
command: "true"
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=llmproxy \
-e POSTGRES_PASSWORD=dbpassword9090 \
-e POSTGRES_DB=litellm \
-p 5432:5432 \
postgres:15
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install system dependencies
command: |
sudo apt-get update -y
sudo apt-get install -y libpq-dev
- run:
name: Install Dependencies
command: |
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
export PATH="$HOME/miniconda/bin:$PATH"
source "$HOME/miniconda/etc/profile.d/conda.sh"
conda activate myenv
fi
uv sync --frozen --all-groups --all-extras --python "$(which python)"
- run:
name: Setup litellm-enterprise
command: "true"
- run:
name: Generate Prisma client
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run Prisma migrations
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
cd litellm/proxy
uv run --no-sync prisma migrate deploy --schema schema.prisma
cd ../..
- run:
name: Run Azure Batch E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
export USE_LOCAL_LITELLM=true
export USE_MOCK_MODELS=true
export USE_STATE_TRACKER=true
export LITELLM_LOG=DEBUG
uv run --no-sync pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10 \
--junitxml=test-results/junit.xml
no_output_timeout: 15m
upload-coverage:
docker:
- image: cimg/python:3.9
@ -3605,12 +3499,6 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_azure_batches_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:

48
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,6 +27,10 @@ on:
required: false
type: number
default: 10
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: true
type: string
permissions:
contents: read
@ -86,4 +90,46 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

54
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -37,6 +37,11 @@ on:
required: false
type: boolean
default: false
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: false
type: string
default: "run"
secrets:
REDIS_HOST:
required: false
@ -144,7 +149,10 @@ jobs:
--maxfail="${MAX_FAILURES}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
@ -153,5 +161,47 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist=loadscope \
--durations=20
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

16
.github/workflows/create-release.yml vendored
View file

@ -48,7 +48,21 @@ jobs:
const cosignSection = [
`## Verify Docker Image Signature`,
``,
`All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
`All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`,
``,
`**Verify using the pinned commit hash (recommended):**`,
``,
`A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`,
``,
'```bash',
`cosign verify \\`,
` --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`,
` ghcr.io/berriai/litellm:${tag}`,
'```',
``,
`**Verify using the release tag (convenience):**`,
``,
`Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`,
``,
'```bash',
`cosign verify \\`,

0
.github/workflows/run_llm_translation_tests.py vendored Executable file → Normal file
View file

2
.github/workflows/scorecard.yml vendored
View file

@ -42,6 +42,6 @@ jobs:
retention-days: 5
- name: Upload to code scanning
uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
sarif_file: results.sarif

206
.github/workflows/test-litellm-matrix.yml vendored
View file

@ -1,206 +0,0 @@
name: LiteLLM Unit Tests (Matrix)
on:
pull_request:
branches: [main]
permissions:
contents: read
# Cancel in-progress runs for the same PR
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 30 # Root shard now runs close to 20m on CI; leave headroom for reruns/coverage upload
strategy:
fail-fast: false
matrix:
test-group:
# tests/test_litellm split by subdirectory (~560 files total)
# Vertex AI tests separated for better isolation (prevent auth/env pollution)
- name: "llms-vertex"
path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
- name: "llms-other"
path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
# tests/test_litellm/proxy split by subdirectory (~180 files total)
- name: "proxy-guardrails"
path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
workers: 2
reruns: 2
- name: "proxy-core"
path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
workers: 2
reruns: 2
- name: "proxy-misc"
path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
workers: 2
reruns: 2
- name: "integrations"
path: "tests/test_litellm/integrations"
workers: 2
reruns: 3 # Integration tests tend to be flakier
- name: "core-utils"
path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
- name: "other-1"
# responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
- name: "other-2"
# enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
workers: 2
reruns: 2
- name: "other-3"
# remaining dirs ≈ 8.0k lines
path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
workers: 2
reruns: 2
- name: "root"
path: "tests/test_litellm/test_*.py"
workers: 2
reruns: 2
# tests/proxy_unit_tests split alphabetically (~48 files total)
- name: "proxy-unit-a1"
# test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
path: "tests/proxy_unit_tests/test_[a-j]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-a2"
# test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
path: "tests/proxy_unit_tests/test_[k-o]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b1"
# lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b2"
# proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
path: "tests/proxy_unit_tests/test_proxy_server.py"
workers: 2
reruns: 1
- name: "proxy-unit-b3"
# proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
workers: 2
reruns: 1
- name: "proxy-unit-b4"
# proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
path: "tests/proxy_unit_tests/test_proxy_utils.py"
workers: 2
reruns: 1
- name: "proxy-unit-b5"
# proxy_token_counter (1279) - runs independently from utils
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
workers: 2
reruns: 1
- name: "proxy-unit-b6"
# test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
path: "tests/proxy_unit_tests/test_[r-t]*.py"
workers: 2
reruns: 1
- name: "proxy-unit-b7"
# test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
path: "tests/proxy_unit_tests/test_[u-z]*.py"
workers: 2
reruns: 1
name: test (${{ matrix.test-group.name }})
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
run: |
uv run --no-sync pytest ${{ matrix.test-group.path }} \
--tb=short -vv \
--maxfail=10 \
-n ${{ matrix.test-group.workers }} \
--reruns ${{ matrix.test-group.reruns }} \
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=litellm \
--cov-report=xml:coverage-${{ matrix.test-group.name }}.xml \
--cov-config=pyproject.toml
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ matrix.test-group.name }}
path: coverage-${{ matrix.test-group.name }}.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: test
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # Required for OIDC tokenless upload
pull-requests: write # Required for Codecov PR comments
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Download all coverage reports
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-*
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@aa56896cf108bd10b5eb883cd1d24196da57f695 # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
fail_ci_if_error: false

91
.github/workflows/test-proxy-e2e-azure-batches.yml vendored
View file

@ -1,91 +0,0 @@
name: Proxy E2E Azure Batches Tests
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
proxy_e2e_azure_batches_tests:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-e2e-batches-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-e2e-batches-
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra proxy
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
run: |
cd litellm/proxy
uv run --no-sync prisma migrate deploy --schema schema.prisma
cd ../..
- name: Run Azure Batch E2E Tests
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
USE_LOCAL_LITELLM: "true"
USE_MOCK_MODELS: "true"
USE_STATE_TRACKER: "true"
LITELLM_LOG: DEBUG
run: |
uv run --no-sync pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10

3
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -8,6 +8,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -29,6 +31,7 @@ jobs:
timeout-minutes: 20
enable-redis: true
enable-postgres: false
artifact-name: caching-redis
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}

3
.github/workflows/test-unit-core-utils.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
artifact-name: core-utils

3
.github/workflows/test-unit-enterprise-routing.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -22,3 +24,4 @@ jobs:
tests/test_litellm/router_strategy
workers: 2
reruns: 2
artifact-name: enterprise-routing

3
.github/workflows/test-unit-integrations.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
artifact-name: integrations

10
.github/workflows/test-unit-llm-providers.yml vendored
View file

@ -14,16 +14,26 @@ concurrency:
jobs:
vertex-ai:
name: Vertex AI
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
artifact-name: llm-vertex-ai
other-providers:
name: All Other Providers
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
artifact-name: llm-other-providers

3
.github/workflows/test-unit-misc.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -29,3 +31,4 @@ jobs:
tests/test_litellm/test_*.py
workers: 2
reruns: 2
artifact-name: misc

3
.github/workflows/test-unit-proxy-auth.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
workers: 2
reruns: 2
artifact-name: proxy-auth

5
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -14,6 +14,10 @@ concurrency:
jobs:
proxy-db:
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
@ -39,6 +43,7 @@ jobs:
timeout-minutes: ${{ matrix.timeout }}
enable-redis: false
enable-postgres: true
artifact-name: proxy-db-${{ matrix.test-group }}
secrets:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}

3
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -33,3 +35,4 @@ jobs:
tests/test_litellm/proxy/ui_crud_endpoints
workers: 2
reruns: 2
artifact-name: proxy-endpoints

3
.github/workflows/test-unit-proxy-infra.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -26,3 +28,4 @@ jobs:
tests/test_litellm/proxy/test_*.py
workers: 2
reruns: 2
artifact-name: proxy-infra

3
.github/workflows/test-unit-responses-caching-types.yml vendored
View file

@ -6,6 +6,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@ -18,3 +20,4 @@ jobs:
test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
artifact-name: responses-caching-types

3
.github/workflows/test-unit-security.yml vendored
View file

@ -7,6 +7,8 @@ on:
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@ -22,6 +24,7 @@ jobs:
timeout-minutes: 20
enable-redis: false
enable-postgres: true
artifact-name: security
secrets:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}

12
.trivyignore
View file

@ -1,12 +0,0 @@
# LiteLLM Trivy Ignore File
# CVEs listed here are temporarily allowlisted pending fixes
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
# Allowlisted: 2026-01-31, 7-day fix timeline
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
# HIGH: DoS via request deserialization
GHSA-h25m-26qc-wcjf
# MEDIUM: Image Optimizer DoS
CVE-2025-59471

2
AGENTS.md
View file

@ -253,7 +253,7 @@ See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry.
### Lint

1
CLAUDE.md
View file

@ -20,6 +20,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
### Single Test Files
- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file

13
CONTRIBUTING.md
View file

@ -157,6 +157,19 @@ Apply formatting (auto-fixes issues):
make format
```
> **Black formatting is enforced in CI.** All PRs must pass the Black formatting check.
>
> - **AI coding agents** (Claude Code, Copilot, Cursor, etc.): `AGENTS.md` and `CLAUDE.md` instruct agents to run `poetry run black .` before committing.
> - **VS Code users**: Install the [Black Formatter extension](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter) and enable format-on-save:
> ```json
> {
> "[python]": {
> "editor.defaultFormatter": "ms-python.black-formatter",
> "editor.formatOnSave": true
> }
> }
> ```
### CI Compatibility
To ensure your changes will pass CI, run the exact same checks locally:

39
README.md
View file

@ -2,20 +2,24 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
<p align="center">LiteLLM AI Gateway
</p>
<p align="center">Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.</p>
<p align="center">
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
<a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic">
<img src="https://railway.com/button.svg" alt="Deploy on Railway">
</a>
</p>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://litellm.ai/" target="_blank">Website</a></h4>
<h4 align="center">
<a href="https://pypi.org/project/litellm/" target="_blank">
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
</a>
<a href="https://github.com/BerriAI/litellm" target="_blank">
<img src="https://img.shields.io/github/stars/BerriAI/litellm.svg?style=social" alt="GitHub Stars">
</a>
<a href="https://www.ycombinator.com/companies/berriai">
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
</a>
@ -400,9 +404,36 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
2. Install dependencies `npm install`
3. Run `npm run dev` to start the dashboard
# Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
# Enterprise
For companies that need better security, user management and professional support
[Get an Enterprise License](https://litellm.ai/enterprise)
[Talk to founders](https://enterprise.litellm.ai/demo)
This covers:

36
ci_cd/.grype.yaml
View file

@ -1,36 +0,0 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
# Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
- vulnerability: CVE-2025-55130
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59465
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55131
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59466
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2026-21637
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55132
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: GHSA-hx9q-6w63-j58v
reason: orjson dumps recursion; allowlisted
- vulnerability: GHSA-73rr-hh4g-fpgx
reason: diff npm transitive dep; override in package.json, allowlisted
- vulnerability: CVE-2026-0865
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15282
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-0672
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15366
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15367
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-11468
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-12781
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-1299
reason: Python 3.13 in Wolfi base; no fixed apk build yet

265
ci_cd/security_scans.sh
View file

@ -1,265 +0,0 @@
#!/bin/bash
# Security Scans Script for LiteLLM
# This script runs comprehensive security scans including Trivy and Grype
set -e
echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
TRIVY_VERSION="0.35.0"
sudo apt-get update
sudo apt-get install -y wget jq curl bsdmainutils
wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb"
sudo dpkg -i trivy.deb
rm trivy.deb
echo "Trivy ${TRIVY_VERSION} installed successfully"
}
# Function to install Grype
install_grype() {
echo "Installing Grype..."
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
echo "Grype installed successfully"
}
# Function to install ggshield
install_ggshield() {
echo "Installing ggshield..."
if ! command -v uv >/dev/null 2>&1; then
curl -LsSf https://astral.sh/uv/0.10.9/install.sh | env UV_NO_MODIFY_PATH=1 sh
export PATH="$HOME/.local/bin:$PATH"
fi
uv tool install --force ggshield==1.48.0
export PATH="$(uv tool dir --bin):$PATH"
echo "ggshield installed successfully"
}
# # Function to run secret detection scans
# run_secret_detection() {
# echo "Running secret detection scans..."
# if ! command -v ggshield &> /dev/null; then
# install_ggshield
# fi
# # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
# if [ -z "$GITGUARDIAN_API_KEY" ]; then
# echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
# echo "ggshield requires a GitGuardian API key to scan for secrets."
# echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
# exit 1
# fi
# echo "Scanning codebase for secrets..."
# echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
# echo "ggshield will automatically handle rate limits and retry as needed."
# echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
# # Use --recursive for directory scanning and auto-confirm if prompted
# # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# # GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
# echo "Secret detection scans completed successfully"
# }
# Function to run Trivy scans
run_trivy_scans() {
echo "Running Trivy scans..."
echo "Scanning LiteLLM Docs..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
echo "Scanning LiteLLM UI..."
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
echo "Trivy scans completed successfully"
}
# Function to build and scan Docker images with Grype
run_grype_scans() {
echo "Running Grype scans..."
# Temporarily add wheel files to .dockerignore for security scans
echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
echo "/*.whl" >> .dockerignore
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build --no-cache -t litellm:latest .
grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
# Restore original .dockerignore
echo "Restoring original .dockerignore..."
mv .dockerignore.backup .dockerignore
# Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); legacy installer fallback unused; no OS-level fix
# - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
# - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
# and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
ALLOWED_CVES=(
"CVE-2025-8869"
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
"CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
"CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
"CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
"CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # Node only used for Admin UI build/prisma
"CVE-2025-55131" # Node only used for Admin UI build/prisma
"CVE-2025-59466" # Node only used for Admin UI build/prisma
"CVE-2025-55130" # Node only used for Admin UI build/prisma
"CVE-2025-59467" # Node only used for Admin UI build/prisma
"CVE-2026-21637" # Node only used for Admin UI build/prisma
"CVE-2025-55132" # Node only used for Admin UI build/prisma
"GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
"CVE-2026-0672" # No fix available yet
"CVE-2025-15366" # No fix available yet
"CVE-2025-15367" # No fix available yet
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
"CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
"CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
echo ""
# Show all high-severity vulnerabilities for transparency
TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| .vulnerability.id' | wc -l)
if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
echo ""
echo "All high-severity vulnerabilities (including allowlisted):"
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
| @tsv' | column -t -s $'\t'
echo ""
fi
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo ""
echo "=========================================="
echo "ERROR: Security Scan Failed"
echo "=========================================="
echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo ""
echo "These vulnerabilities are NOT in the allowlist and must be addressed."
echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
echo ""
echo "Detailed vulnerability report:"
echo ""
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
echo ""
echo "=========================================="
echo "Action Required:"
echo "=========================================="
echo "1. If a fix is available, update the package to the fixed version"
echo "2. If the vulnerability is not applicable or has no fix:"
echo " - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
echo " - Add a comment explaining why it's safe to ignore"
echo ""
echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
echo "Add all relevant IDs to the allowlist if they refer to the same issue."
echo "=========================================="
echo ""
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
fi
echo "Grype scans completed successfully"
}
# Main execution
main() {
echo "Installing security scanning tools..."
install_trivy
install_grype
# echo "Running secret detection scans..."
# run_secret_detection
echo "Running filesystem vulnerability scans..."
run_trivy_scans
echo "Running Docker image vulnerability scans..."
run_grype_scans
echo "All security scans completed successfully!"
}
# Execute main function
main "$@"

1
docker/Dockerfile.dev
View file

@ -17,6 +17,7 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
libssl-dev \
pkg-config \

16
docker/Dockerfile.non_root
View file

@ -45,9 +45,16 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
COPY . .
# Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true
# Build Admin UI once and stage the static output for the runtime image.
# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
# are temporarily renamed during npm install/ci so they don't block lifecycle
# scripts needed by the build. This is safe because npm ci installs from
# package-lock.json with pinned versions + integrity hashes.
RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
mv /app/.npmrc /app/.npmrc.bak && \
([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
NVM_VERSION="v0.40.4" && \
NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \
NODE_VERSION="v20.20.2" && \
@ -61,15 +68,16 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
nvm use "${NODE_VERSION}" && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \
ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
mv .npmrc .npmrc.bak && \
([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
npm ci --no-audit --no-fund && \
mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \
([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \

7
docs/my-website/.trivyignore
View file

@ -1,7 +0,0 @@
# js-yaml CVE-2025-64718
# This vulnerability is not applicable because we've forced js-yaml to version 4.1.1
# via npm overrides in package.json. Trivy incorrectly reports this based on
# dependency requirements in the lockfile, but the actual installed version is 4.1.1.
# Verified with: npm list js-yaml
CVE-2025-64718

35
docs/my-website/blog/ci_cd_v2_improvements/index.md
View file

@ -27,6 +27,41 @@ Building on the roadmap from our [security incident](https://docs.litellm.ai/blo
- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials.
- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases.
- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well.
- Docker image signing with [Cosign](https://github.com/sigstore/cosign) - all release images are signed so users can independently verify they came from us.
## Verify Docker image signatures
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
## What's next

35
docs/my-website/blog/security_townhall_updates/index.md
View file

@ -143,8 +143,41 @@ This will ensure, your releases are safe, even when:
- Tampered registry artifacts are published
- Tag mutations are made after the release is published
We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have already begun working on it [PR](https://github.com/BerriAI/litellm/pull/24683).
We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have shipped it in [PR #24683](https://github.com/BerriAI/litellm/pull/24683).
#### How to verify a Docker image with Cosign
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key that was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
### Avoid Compromised Packages

34
docs/my-website/blog/security_update_march_2026/index.md
View file

@ -708,6 +708,40 @@ The LiteLLM AI Gateway team has already taken the following steps:
- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain
## Verify Docker image signatures
Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
## Verified safe versions
We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by:

3
docs/my-website/docs/mcp.md
View file

@ -278,7 +278,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
aws_service_name: bedrock-agentcore

53
docs/my-website/docs/mcp_aws_sigv4.md
View file

@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r
| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
| **AWS Session Token** | No | Only needed for temporary STS credentials |
| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing |
| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted |
Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
@ -66,8 +68,8 @@ mcp_servers:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended)
aws_session_name: "litellm-prod" # optional — for CloudTrail auditing
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing |
| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` |
## How It Works
@ -157,6 +161,42 @@ mcp_servers:
aws_service_name: "bedrock-agentcore"
```
## Using IAM Role Assumption (AssumeRole)
For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests:
```yaml title="config.yaml with AssumeRole" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole"
aws_session_name: "litellm-prod" # optional
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials.
You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call:
```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_role_name: os.environ/AWS_ROLE_ARN
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: "us-east-1"
```
:::tip
For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically.
:::
## Troubleshooting
### 403 Forbidden from AWS
@ -166,6 +206,15 @@ mcp_servers:
- Ensure `aws_service_name` is set to `bedrock-agentcore`
- If using STS credentials, confirm `aws_session_token` is set and not expired
### AssumeRole AccessDenied
If you get `AccessDenied` when using `aws_role_name`:
- Verify the role ARN is correct
- Check that the trust policy on the target role allows your source identity to assume it
- If running on EKS, ensure the pod's service account is annotated with the correct IAM role
- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error
### Health check errors on startup
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.

231
docs/my-website/docs/mcp_toolsets.md Normal file
View file

@ -0,0 +1,231 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MCP Toolsets
A **Toolset** is a named collection of specific tools drawn from one or more MCP servers. Instead of giving an agent access to every tool on every server, you pick exactly which tools it needs — from whichever servers they live on — and bundle them under a single name.
## How it works
```
┌─────────────────────────────────┐
│ MCP Toolset │
│ "devtooling-prod" │
└────────────┬────────────────────┘
│
┌──────────────────┴──────────────────┐
│ │
┌────────▼────────┐ ┌────────▼────────┐
│ CircleCI MCP │ │ DeepWiki MCP │
│ (10+ tools) │ │ (3 tools) │
└────────┬────────┘ └────────┬────────┘
│ │
┌─────────┴──────────┐ ┌──────────┴──────────┐
│ ✓ get_build_logs │ │ ✓ read_wiki_structure│
│ ✓ find_flaky_tests │ │ ✓ read_wiki_contents │
│ ✓ get_pipeline_ │ │ ✗ ask_question │
│ status │ └─────────────────────┘
│ ✓ run_pipeline │
│ ✗ list_followed_ │
│ projects │
└────────────────────┘
Agent sees exactly 6 tools, nothing more.
```
Instead of 13+ tools across two servers, the agent gets 6 — the ones it actually needs.
**Why this matters:**
- Smaller tool lists → fewer tokens, faster responses, less hallucination
- Combine tools from GitHub + Linear + CircleCI into one named grant
- Assign to keys and teams the same way you assign MCP servers today
---
## Create a toolset
### 1. Go to the MCP page
Navigate to **MCP** in the left sidebar.
![Navigate to MCP](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/1a96c713-6a37-4f96-92f1-07bd58c1973c/ascreenshot_23515f386ccc4597b0633987667fe01f_text_export.jpeg)
### 2. Open the Toolsets tab
Click the **Toolsets** tab on the MCP page.
![Click Toolsets tab](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/65b6986b-595a-4b28-8fdc-a7b36bc76e59/ascreenshot_ca70c18fe7ec415486f96a6b405bf550_text_export.jpeg)
### 3. Click "New Toolset"
![New Toolset button](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/798c55c4-5d6b-4815-a642-70ac9f34f102/ascreenshot_3f144f54a1a944e28454239c837b4e6d_text_export.jpeg)
### 4. Enter a name
Type a name for the toolset. Pick something descriptive — this is what agents will reference.
![Enter toolset name](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/62b412e0-d38f-44c3-99e4-3693f1512f6a/ascreenshot_b678c7c988a04f8b887b0f54c4dd95a7_text_export.jpeg)
![Toolset name field](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ba5ebc95-cab7-470b-a7c9-21f12b9b01a3/ascreenshot_a602e982a2a44890a83dca64d61c38eb_text_export.jpeg)
### 5. Add the first tool
Select an MCP server from the dropdown, then choose the tool you want to include from that server.
![Select MCP server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/2aa5bcba-6414-42e3-9813-efb0a9078e32/ascreenshot_58fbff35ba654210a1b4dc5452aa6bd9_text_export.jpeg)
![Choose server from dropdown](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/4fd9cffb-d3ba-461a-8679-89f278bf67ad/ascreenshot_b61e9e85a51b494a8d09fe61198d63e1_text_export.jpeg)
![Select tool from server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/60718e72-2062-494b-9a23-456992c88cbd/ascreenshot_7a1f8eeab30a4a05ba39c450e5458b78_text_export.jpeg)
### 6. Add tools from a second server
Click **Add Tool**, pick a different MCP server, and select another tool. Repeat for as many tools as you need — they can come from any number of servers.
![Add tool from second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/f34e0600-cc74-4b18-8794-88d45f326144/ascreenshot_98834b14ab9343e39fb503e458d72b7c_text_export.jpeg)
![Select second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/75150368-2202-4da1-99f1-6f0620e9b133/ascreenshot_f94d0bc08ea147348a9cf021cce7d854_text_export.jpeg)
![Select tool from second server](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ed2cdf6e-025d-4d50-8b12-ed68745d5c51/ascreenshot_0c1c7f76524b46c5a056fda5e6956e2b_text_export.jpeg)
### 7. Create the toolset
Click **Create Toolset** to save.
![Create Toolset](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/021ca7b3-2d9a-49a0-8758-dae3dc3bcb4d/ascreenshot_14c6434e71114a6091e359a996f20e12_text_export.jpeg)
---
## Use a toolset in the Playground
Once created, your toolset appears alongside MCP servers in the **MCP Servers** dropdown in the Playground — it's selectable the same way.
### 1. Go to the Playground
![Navigate to Playground](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/f9d4aa4c-d98e-4767-b98e-aad2890e97ca/ascreenshot_d84239c441bb4e828f229d0c9e079e3f_text_export.jpeg)
![Click Playground](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/d8a07563-97fe-453a-b974-88da46c87294/ascreenshot_ea494300a536400abb2ea6bf3bdfd5ab_text_export.jpeg)
### 2. Select your toolset from MCP Servers
In the left panel under **MCP Servers**, open the dropdown and pick your toolset. The model will only see the tools you included in it.
![Select MCP servers dropdown](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/ee8cb38c-c4ff-4b4b-844c-22f2e40832ae/ascreenshot_e300fb39cea0434fb5e3986e912a2b8d_text_export.jpeg)
![Open MCP server picker](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/8672070c-5d07-4f63-878c-6fc7dcbc9b65/ascreenshot_326ddd0868224c99a6fa5dab2d144f1f_text_export.jpeg)
![Select toolset](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/955826ad-2bbb-403e-ab26-c1ac03ec2675/ascreenshot_13f837ad53574535986ca7ca5998d34a_text_export.jpeg)
![Toolset selected and active](https://colony-recorder.s3.amazonaws.com/files/2026-03-22/9a59c3b9-1563-4731-838f-1c35d636ddc9/ascreenshot_c05d8fa5f37a4b3093fc46e26f293b4d_text_export.jpeg)
The model now has access to exactly the tools in your toolset and nothing else.
---
## Use a toolset via API
Pass the toolset's route as the `server_url` in your tools list. LiteLLM resolves it server-side — no public URL needed.
<Tabs>
<TabItem value="responses" label="Responses API">
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://your-proxy/v1",
)
response = client.responses.create(
model="gpt-4o",
input="What CI/CD tools do you have?",
tools=[
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never",
}
],
)
print(response.output_text)
```
</TabItem>
<TabItem value="chat" label="Chat Completions API">
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://your-proxy/v1",
)
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": "What CI/CD tools do you have?"}],
tools=[
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never",
}
],
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="rest" label="REST">
```bash
curl http://your-proxy/v1/responses \
-H "Authorization: Bearer your-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"input": "What CI/CD tools do you have?",
"tools": [
{
"type": "mcp",
"server_label": "devtooling-prod",
"server_url": "litellm_proxy/mcp/devtooling-prod",
"require_approval": "never"
}
]
}'
```
</TabItem>
</Tabs>
---
## Manage toolsets via API
```bash
# List all toolsets
curl http://your-proxy/v1/mcp/toolset \
-H "Authorization: Bearer your-litellm-key"
# Create a toolset
curl -X POST http://your-proxy/v1/mcp/toolset \
-H "Authorization: Bearer your-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"toolset_name": "devtooling-prod",
"description": "CircleCI + DeepWiki tools for the dev team",
"tools": [
{"server_id": "<circleci-server-id>", "tool_name": "get_build_failure_logs"},
{"server_id": "<circleci-server-id>", "tool_name": "run_pipeline"},
{"server_id": "<deepwiki-server-id>", "tool_name": "read_wiki_structure"}
]
}'
# Delete a toolset
curl -X DELETE http://your-proxy/v1/mcp/toolset/<toolset_id> \
-H "Authorization: Bearer your-litellm-key"
```

24
docs/my-website/docs/providers/bedrock_image_gen.md
View file

@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Amazon Nova Canvas - Image Edit
Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html):
| Scenario | `taskType` sent to Bedrock |
|----------|----------------------------|
| Image + prompt (no mask) | `IMAGE_VARIATION` |
| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) |
| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) |
| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` |
```python
from litellm import image_edit
response = image_edit(
image=open("photo.png", "rb"),
prompt="Add soft sunset lighting",
model="bedrock/amazon.nova-canvas-v1:0",
)
```
For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`).
## Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
@ -147,4 +170,3 @@ model_list:
## Authentication
All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.

109
docs/my-website/docs/providers/oci.md
View file

@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ
## Supported Models
### Meta Llama Models
### Chat / Text Generation
#### Meta Llama Models
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
- `meta.llama-4-scout-17b-16e-instruct`
- `meta.llama-3.3-70b-instruct`
- `meta.llama-3.3-70b-instruct-fp8-dynamic`
- `meta.llama-3.2-90b-vision-instruct`
- `meta.llama-3.2-11b-vision-instruct`
- `meta.llama-3.1-405b-instruct`
- `meta.llama-3.1-70b-instruct`
### xAI Grok Models
#### xAI Grok Models
- `xai.grok-4.20`
- `xai.grok-4.20-multi-agent`
- `xai.grok-4`
- `xai.grok-4-fast`
- `xai.grok-4.1-fast`
- `xai.grok-3`
- `xai.grok-3-fast`
- `xai.grok-3-mini`
- `xai.grok-3-mini-fast`
- `xai.grok-code-fast-1`
### Cohere Models
#### Cohere Models
- `cohere.command-latest`
- `cohere.command-a-03-2025`
- `cohere.command-a-reasoning-08-2025`
- `cohere.command-a-vision-07-2025`
- `cohere.command-a-translate-08-2025`
- `cohere.command-plus-latest`
- `cohere.command-r-08-2024`
- `cohere.command-r-plus-08-2024`
#### Google Gemini Models (via OCI)
- `google.gemini-2.5-pro`
- `google.gemini-2.5-flash`
- `google.gemini-2.5-flash-lite`
### Embedding Models
- `cohere.embed-english-v3.0` (1024 dimensions)
- `cohere.embed-english-light-v3.0` (384 dimensions)
- `cohere.embed-multilingual-v3.0` (1024 dimensions)
- `cohere.embed-multilingual-light-v3.0` (384 dimensions)
- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal)
- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal)
- `cohere.embed-v4.0` (1536 dimensions, multimodal)
## Authentication
@ -394,4 +424,75 @@ response = completion(
| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy |
| `oci_key` | string | - | (Manual auth) The private key content as a string |
| `oci_key_file` | string | - | (Manual auth) Path to the private key file |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
## Embeddings
LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above.
<Tabs>
<TabItem value="embed-manual" label="Manual Credentials" default>
```python
from litellm import embedding
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_region="us-ashburn-1",
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
oci_key=<string_with_content_of_oci_key>,
oci_compartment_id=<oci_compartment_id>,
)
print(response)
```
</TabItem>
<TabItem value="embed-sdk" label="OCI SDK Signer">
```python
from litellm import embedding
from oci.signer import Signer
signer = Signer(
tenancy="ocid1.tenancy.oc1..",
user="ocid1.user.oc1..",
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
private_key_file_location="~/.oci/key.pem",
)
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_signer=signer,
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
)
print(response)
```
</TabItem>
</Tabs>
### Embedding Optional Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` |
| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` |
### Using Dedicated Embedding Endpoints
```python
response = embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world"],
oci_serving_mode="DEDICATED",
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...",
oci_region="us-ashburn-1",
oci_compartment_id="<oci_compartment_id>",
# ... auth params
)
```

7
docs/my-website/docs/proxy/config_settings.md
View file

@ -201,6 +201,7 @@ router_settings:
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
| default_team_params | object | Default parameters applied to every new team created via `/team/new` (including SSO auto-created teams). Only fills in fields not explicitly set in the request. Sub-fields: `max_budget` (float), `budget_duration` (string, e.g. `"30d"`), `tpm_limit` (integer), `rpm_limit` (integer), `team_member_permissions` (array of strings, e.g. `["/team/daily/activity", "/key/generate"]`), `models` (array of strings — only applied to SSO auto-created teams). |
### general_settings - Reference
@ -237,7 +238,7 @@ router_settings:
| public_routes | List[str] | (Enterprise Feature) Control list of public routes |
| alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] |
| enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy |
| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication on LLM + info routes |
| use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address |
| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] |
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
@ -288,6 +289,7 @@ router_settings:
| database_connection_pool_timeout | integer | Database connection pool timeout in seconds |
| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database |
| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry |
| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations |
| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls |
@ -396,6 +398,7 @@ router_settings:
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments |
| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale |
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
### environment variables - Reference
@ -820,6 +823,7 @@ router_settings:
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
| LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS | TTL in seconds for the distributed lock used by the key rotation job. Default is 600 (10 minutes).
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
@ -1028,6 +1032,7 @@ router_settings:
| SENDGRID_SENDER_EMAIL | Email address used as the sender in SendGrid email transactions
| SPEND_LOGS_URL | URL for retrieving spend logs
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| STALE_OBJECT_CLEANUP_BATCH_SIZE | Max number of stale managed objects updated per cleanup cycle. Default is 1000
| SSL_CERTIFICATE | Path to the SSL certificate file
| SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC).
| SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1`

38
docs/my-website/docs/proxy/deploy.md
View file

@ -65,7 +65,43 @@ docker compose up
</TabItem>
</Tabs>
### Docker Run
### Verify Docker image signatures
All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
**Verify using a release tag (convenience):**
Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```
Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
Expected output:
```
The following checks were performed on each of these signatures:
- The cosign claims were validated
- The signatures were verified against the specified public key
```
Learn more about LiteLLM's release signing in the [CI/CD v2 announcement](https://docs.litellm.ai/blog/ci-cd-v2-improvements#verify-docker-image-signatures).
### Docker Run
#### Step 1. CREATE config.yaml

4
docs/my-website/docs/proxy/guardrails/guardrail_policies.md
View file

@ -311,7 +311,7 @@ Response:
## Policy Flow Builder
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step pass/fail actions.
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step **pass**, **fail**, and optional **error** actions (`on_pass`, `on_fail`, `on_error`).
## Config Reference
@ -337,7 +337,7 @@ policies:
| `guardrails.add` | `list[string]` | Guardrails to enable. |
| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions. See [Policy Flow Builder](./policy_flow_builder). |
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions (`on_pass`, `on_fail`, optional `on_error`). See [Policy Flow Builder](./policy_flow_builder). |
### `policy_attachments`

65
docs/my-website/docs/proxy/guardrails/policy_flow_builder.md
View file

@ -1,8 +1,8 @@
# Policy Flow Builder
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail passes or fails.
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail **passes**, **fails a policy check** (content intervention), or hits a **technical error** (e.g. timeout, unreachable provider, missing guardrail).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors).
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors). With **`on_error`**, you can treat **technical** failures differently from **policy** failures—for example, fall back to another provider when the primary API errors, while still blocking on flagged content.
## When to use the Flow Builder
@ -19,6 +19,7 @@ Use the Flow Builder when you need:
- **Custom responses** — return a specific message when a guardrail fails instead of a generic block
- **Data chaining** — pass modified data (e.g., PII-masked content) from one step to the next
- **Fine-grained control** — different actions on pass vs. fail per step
- **Technical-error routing** — set `on_error` separately from `on_fail` so outages or timeouts can **allow**, **block**, **go to the next step**, or return a **custom response** without conflating them with content violations
## Concepts
@ -29,24 +30,37 @@ A pipeline has:
- **Mode**: `pre_call` (before the LLM) or `post_call` (after the LLM)
- **Steps**: Ordered list of guardrail steps
### Outcomes: pass, fail, and error
Each step run produces one of three outcomes:
| Outcome | Meaning | Typical cause |
|--------|---------|----------------|
| **pass** | Guardrail completed without blocking | Content allowed, or data was modified and returned |
| **fail** | Policy intervention | Guardrail raised an intervention (e.g. flagged content, blocked request) |
| **error** | Technical failure | Timeouts, network errors, guardrail not registered, or other non-intervention exceptions |
`on_pass` and `on_fail` apply to **pass** and **fail** respectively. **`on_error`** applies only to **error**. If `on_error` is omitted, the pipeline uses **`on_fail`** for error outcomes (backward compatible).
### Step actions
Each step defines what happens when the guardrail **passes** and when it **fails**:
For each step you choose an action for **pass**, **fail**, and optionally **error**. Allowed values are: `next`, `allow`, `block`, `modify_response`.
| Action | Description |
|--------|-------------|
| **Next Step** | Continue to the next guardrail in the pipeline |
| **Allow** | Stop the pipeline and allow the request to proceed |
| **Block** | Stop the pipeline and block the request |
| **Custom Response** | Return a custom message instead of the default block |
| **Next Step** (`next`) | Continue to the next guardrail in the pipeline |
| **Allow** (`allow`) | Stop the pipeline and allow the request to proceed |
| **Block** (`block`) | Stop the pipeline and block the request |
| **Custom Response** (`modify_response`) | Return a custom message instead of the default block |
### Step options
| Field | Type | Description |
|-------|------|--------------|
| `guardrail` | `string` | Name of the guardrail to run |
| `on_pass` | `string` | Action when guardrail passes: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when guardrail fails: `next`, `allow`, `block`, `modify_response` |
| `on_pass` | `string` | Action when outcome is **pass**: `next`, `allow`, `block`, `modify_response` |
| `on_fail` | `string` | Action when outcome is **fail** (policy intervention): `next`, `allow`, `block`, `modify_response` |
| `on_error` | `string` (optional) | Action when outcome is **error** (technical). If omitted, **error** uses `on_fail`. |
| `pass_data` | `boolean` | Forward modified request data (e.g., PII-masked) to the next step |
| `modify_response_message` | `string` | Custom message when using `modify_response` action |
@ -57,7 +71,7 @@ Each step defines what happens when the guardrail **passes** and when it **fails
3. Select **Flow Builder** (instead of the simple form)
4. Design your flow:
- **Trigger** — Incoming LLM request (runs when the policy matches)
- **Steps** — Add guardrails, set ON PASS and ON FAIL actions per step
- **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL)
- **End** — Request proceeds to the LLM
5. Use the **+** between steps to insert new steps
6. Use the **Test** panel to run sample messages through the pipeline before saving
@ -151,6 +165,37 @@ policies:
First attempt passes → allow. First attempt fails → retry the same guardrail; second pass → allow, second fail → block.
## Technical errors vs policy failures (`on_error`)
Use **`on_error`** when you want different behavior for **API/infra problems** than for **content policy** violations.
- **`on_fail`** — Runs when the guardrail **intervenes** (e.g. toxic content, PII detected).
- **`on_error`** — Runs when the step ends in **error** (timeout, connection failure, guardrail not loaded, etc.). If you omit `on_error`, **error** outcomes use **`on_fail`**.
Example: block on bad content, but if the primary scanner is down, fall back to a second guardrail instead of blocking every request:
```yaml
policies:
error-fallback-policy:
guardrails:
add:
- primary_scanner
- backup_scanner
pipeline:
mode: pre_call
steps:
- guardrail: primary_scanner
on_pass: allow
on_fail: block
on_error: next
- guardrail: backup_scanner
on_pass: allow
on_fail: block
on_error: allow
```
If `primary_scanner` errors → run `backup_scanner`. If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed).
## Example: Custom response on fail
Return a branded message instead of a generic block:

81
docs/my-website/docs/proxy/health.md
View file

@ -316,86 +316,9 @@ general_settings:
## Health Check Driven Routing
By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown.
Route traffic away from unhealthy deployments proactively — before user requests hit them. Supports per-error-type failure thresholds, transient error suppression, and automatic safety nets.
With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown.
### How it works
1. Background health checks run on their configured interval
2. After each cycle, every deployment is marked healthy or unhealthy
3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing
4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage)
5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored
### Quick start
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true
```
### Configuration
| Setting | Where | Default | Description |
|---------|-------|---------|-------------|
| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing |
| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles |
### Interaction with cooldown
Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism:
- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check
- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL
This means request failures still provide fast detection between health check intervals.
### Staleness
If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down.
The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s.
### Example: custom staleness
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_staleness_threshold: 90 # ignore health state older than 90s
```
### Debugging
Run the proxy with `--detailed_debug` and look for:
```
health_check_routing_state_updated healthy=3 unhealthy=1
```
This is logged after each health check cycle when routing state is written.
If the safety net triggers (all deployments unhealthy), you'll see:
```
All deployments marked unhealthy by health checks, bypassing health filter
```
See the full guide: [Health Check Driven Routing](./health_check_routing.md)
## Health Check Timeout

340
docs/my-website/docs/proxy/health_check_routing.md Normal file
View file

@ -0,0 +1,340 @@
# Health Check Driven Routing
Route traffic away from unhealthy deployments before users hit errors. Background health checks run on a configurable interval, and any deployment that fails gets removed from the routing pool proactively, not after a user request already failed.
## Architecture
<svg viewBox="0 0 860 600" xmlns="http://www.w3.org/2000/svg" style={{maxWidth: '100%', fontFamily: 'system-ui, sans-serif'}}>
{/* Background */}
<rect width="860" height="600" fill="#f8fafc" rx="12"/>
{/* LEFT PANEL: Background health check loop */}
<rect x="20" y="20" width="240" height="560" fill="#eff6ff" rx="10" stroke="#bfdbfe" strokeWidth="1.5"/>
<text x="140" y="48" textAnchor="middle" fill="#1d4ed8" fontSize="13" fontWeight="600">Background Loop</text>
<text x="140" y="64" textAnchor="middle" fill="#3b82f6" fontSize="11">every health_check_interval seconds</text>
{/* Deployment A */}
<rect x="40" y="82" width="200" height="50" fill="white" rx="8" stroke="#93c5fd" strokeWidth="1.5"/>
<text x="140" y="102" textAnchor="middle" fill="#1e40af" fontSize="12" fontWeight="500">Deployment A</text>
<text x="140" y="120" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 200 ✓</text>
{/* Deployment B */}
<rect x="40" y="148" width="200" height="50" fill="white" rx="8" stroke="#fca5a5" strokeWidth="1.5"/>
<text x="140" y="168" textAnchor="middle" fill="#991b1b" fontSize="12" fontWeight="500">Deployment B</text>
<text x="140" y="186" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 401 ✗</text>
{/* Deployment C */}
<rect x="40" y="214" width="200" height="50" fill="white" rx="8" stroke="#fde68a" strokeWidth="1.5"/>
<text x="140" y="234" textAnchor="middle" fill="#92400e" fontSize="12" fontWeight="500">Deployment C</text>
<text x="140" y="252" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 429 ⚡</text>
{/* ignore_transient box */}
<rect x="40" y="282" width="200" height="68" fill="#fefce8" rx="8" stroke="#fde047" strokeWidth="1.5"/>
<text x="140" y="302" textAnchor="middle" fill="#713f12" fontSize="11" fontWeight="600">ignore_transient_errors: true</text>
<text x="140" y="320" textAnchor="middle" fill="#92400e" fontSize="11">429 / 408 → ignored</text>
<text x="140" y="338" textAnchor="middle" fill="#92400e" fontSize="11">not written to cache</text>
{/* allowed_fails_policy box */}
<rect x="40" y="368" width="200" height="84" fill="#f0fdf4" rx="8" stroke="#86efac" strokeWidth="1.5"/>
<text x="140" y="388" textAnchor="middle" fill="#166534" fontSize="11" fontWeight="600">allowed_fails_policy</text>
<text x="140" y="406" textAnchor="middle" fill="#15803d" fontSize="11">401 → increment counter</text>
<text x="140" y="424" textAnchor="middle" fill="#15803d" fontSize="11">counter &gt; threshold</text>
<text x="140" y="442" textAnchor="middle" fill="#15803d" fontSize="11">→ cooldown triggered</text>
{/* CENTER PANEL: Shared State */}
<rect x="300" y="20" width="220" height="560" fill="#f5f3ff" rx="10" stroke="#c4b5fd" strokeWidth="1.5"/>
<text x="410" y="48" textAnchor="middle" fill="#6d28d9" fontSize="13" fontWeight="600">Shared State</text>
{/* Health State Cache */}
<rect x="320" y="62" width="180" height="116" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="84" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">DeploymentHealthCache</text>
<text x="410" y="104" textAnchor="middle" fill="#64748b" fontSize="11">A → healthy ✓</text>
<text x="410" y="122" textAnchor="middle" fill="#64748b" fontSize="11">B → unhealthy ✗</text>
<text x="410" y="140" textAnchor="middle" fill="#64748b" fontSize="11">C → not written (ignored)</text>
<text x="410" y="164" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: staleness_threshold × 1.5</text>
{/* Cooldown Cache */}
<rect x="320" y="196" width="180" height="104" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="218" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">Cooldown Cache</text>
<text x="410" y="238" textAnchor="middle" fill="#64748b" fontSize="11">B → cooling down</text>
<text x="410" y="256" textAnchor="middle" fill="#64748b" fontSize="11">(after policy threshold)</text>
<text x="410" y="278" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time</text>
{/* failed_calls counter */}
<rect x="320" y="318" width="180" height="90" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
<text x="410" y="340" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">failed_calls counter</text>
<text x="410" y="360" textAnchor="middle" fill="#64748b" fontSize="11">B: 2 / AuthAllowedFails: 1</text>
<text x="410" y="378" textAnchor="middle" fill="#64748b" fontSize="11">→ threshold exceeded</text>
<text x="410" y="398" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time (must &gt; interval)</text>
{/* RIGHT PANEL: Request path */}
<rect x="560" y="20" width="280" height="560" fill="#fff7ed" rx="10" stroke="#fed7aa" strokeWidth="1.5"/>
<text x="700" y="48" textAnchor="middle" fill="#c2410c" fontSize="13" fontWeight="600">Request Path</text>
{/* Incoming request */}
<rect x="580" y="62" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="85" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="500">Incoming request</text>
{/* All deployments */}
<rect x="580" y="120" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="143" textAnchor="middle" fill="#9a3412" fontSize="12">All deployments [A, B, C]</text>
<line x1="700" y1="100" x2="700" y2="120" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Health check filter */}
<rect x="580" y="178" width="240" height="62" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="200" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">① Health Check Filter</text>
<text x="700" y="218" textAnchor="middle" fill="#64748b" fontSize="11">if policy set → bypass</text>
<text x="700" y="234" textAnchor="middle" fill="#64748b" fontSize="11">else → remove unhealthy</text>
<line x1="700" y1="158" x2="700" y2="178" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Cooldown filter */}
<rect x="580" y="262" width="240" height="50" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="284" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">② Cooldown Filter</text>
<text x="700" y="302" textAnchor="middle" fill="#64748b" fontSize="11">remove deployments in cooldown</text>
<line x1="700" y1="240" x2="700" y2="262" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Safety net */}
<rect x="580" y="334" width="240" height="52" fill="#fef9c3" rx="7" stroke="#fbbf24" strokeWidth="1.5"/>
<text x="700" y="356" textAnchor="middle" fill="#713f12" fontSize="12" fontWeight="600">Safety Net</text>
<text x="700" y="376" textAnchor="middle" fill="#713f12" fontSize="11">if all removed → return all</text>
<line x1="700" y1="312" x2="700" y2="334" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Load balancer */}
<rect x="580" y="408" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
<text x="700" y="431" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">③ Load Balancer</text>
<line x1="700" y1="386" x2="700" y2="408" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
{/* Selected deployment */}
<rect x="580" y="468" width="240" height="38" fill="#dcfce7" rx="7" stroke="#4ade80" strokeWidth="1.5"/>
<text x="700" y="491" textAnchor="middle" fill="#14532d" fontSize="12" fontWeight="600">Selected: Deployment A ✓</text>
<line x1="700" y1="446" x2="700" y2="468" stroke="#4ade80" strokeWidth="1.5" markerEnd="url(#arrow-green)"/>
{/* ARROWS: left → center */}
<line x1="240" y1="107" x2="320" y2="110" stroke="#3b82f6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-blue)"/>
<line x1="240" y1="173" x2="320" y2="240" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="173" x2="320" y2="348" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
<line x1="240" y1="316" x2="320" y2="130" stroke="#eab308" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-yellow)"/>
{/* ARROWS: center → right */}
<line x1="500" y1="120" x2="580" y2="190" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
<line x1="500" y1="248" x2="580" y2="274" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
{/* Arrow markers */}
<defs>
<marker id="arrow-orange" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#fb923c"/>
</marker>
<marker id="arrow-blue" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#3b82f6"/>
</marker>
<marker id="arrow-red" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#ef4444"/>
</marker>
<marker id="arrow-yellow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#eab308"/>
</marker>
<marker id="arrow-purple" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#8b5cf6"/>
</marker>
<marker id="arrow-green" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
<path d="M0,0 L0,6 L8,3 z" fill="#4ade80"/>
</marker>
</defs>
</svg>
## What problem does this solve?
By default, LiteLLM routes traffic to all deployments and only stops sending to a broken one after it has already failed a user request. The cooldown system is reactive.
Health check driven routing makes this **proactive**: a background loop pings every deployment on a configurable interval. If a deployment fails its health check, it gets removed from the routing pool immediately, before a user request lands on it.
When you also set `allowed_fails_policy`, you control exactly how many health check failures of each error type (auth errors, rate limits, timeouts) are needed before a deployment enters cooldown. This avoids false positives from transient noise.
## Setup
### Step 1: Enable background health checks
Background health checks are off by default. Turn them on in `general_settings`:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60 # seconds between each full check cycle
```
### Step 2: Enable health check routing
```yaml
general_settings:
background_health_checks: true
health_check_interval: 60
enable_health_check_routing: true # ← route away from unhealthy deployments
```
At this point, any deployment that fails its health check is immediately excluded from routing until the next check cycle clears it.
### Step 3: Add a policy to control how many failures trigger cooldown
Without a policy, the first health check failure marks a deployment as unhealthy. If you want more tolerance (e.g., only act after 2 consecutive auth failures), use `allowed_fails_policy`:
```yaml
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY_SECONDARY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
router_settings:
cooldown_time: 60 # how long a deployment stays in cooldown
allowed_fails_policy:
AuthenticationErrorAllowedFails: 1 # cooldown after 2nd auth failure
TimeoutErrorAllowedFails: 3 # cooldown after 4th timeout
```
When `allowed_fails_policy` is set, the binary health check filter is bypassed. Only the cooldown system controls routing exclusion, and it only fires after your configured threshold is crossed.
### Step 4 (optional): Ignore transient errors
429 (rate limit) and 408 (timeout) from a health check usually mean the deployment is temporarily overloaded, not broken. To prevent these from affecting routing at all:
```yaml
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true # 429 and 408 never affect routing
```
With this on, only hard failures (401, 404, 5xx) from health checks contribute to cooldown.
## Full example
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY_SECONDARY
- model_name: gpt-4o
litellm_params:
model: azure/gpt-4o
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
general_settings:
background_health_checks: true
health_check_interval: 30
enable_health_check_routing: true
health_check_ignore_transient_errors: true
router_settings:
cooldown_time: 60
allowed_fails_policy:
AuthenticationErrorAllowedFails: 0 # cooldown immediately on auth failure
TimeoutErrorAllowedFails: 2 # cooldown after 3 timeouts
RateLimitErrorAllowedFails: 5 # cooldown after 6 rate limits (if not ignoring transients)
```
## Configuration reference
| Setting | Where | Default | Description |
|---|---|---|---|
| `enable_health_check_routing` | `general_settings` | `false` | Route away from deployments that fail health checks |
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
| `health_check_interval` | `general_settings` | `300` | Seconds between full health check cycles |
| `health_check_staleness_threshold` | `general_settings` | `interval x 2` | Seconds before cached health state is ignored |
| `health_check_ignore_transient_errors` | `general_settings` | `false` | Ignore 429 and 408 from health checks; these never affect routing |
| `cooldown_time` | `router_settings` | `5` | Seconds a deployment stays in cooldown after threshold is crossed |
| `allowed_fails_policy` | `router_settings` | `null` | Per-error-type failure thresholds before cooldown (see below) |
### `allowed_fails_policy` fields
| Field | Error type | HTTP status |
|---|---|---|
| `AuthenticationErrorAllowedFails` | Bad API key | 401 |
| `TimeoutErrorAllowedFails` | Request timeout | 408 |
| `RateLimitErrorAllowedFails` | Rate limit exceeded | 429 |
| `BadRequestErrorAllowedFails` | Malformed request | 400 |
| `ContentPolicyViolationErrorAllowedFails` | Content filtered | 400 |
The value is the number of failures **tolerated** before cooldown. `0` means cooldown on the first failure. `2` means cooldown on the third.
## Things to keep in mind
- **Counter TTL must be longer than the health check interval.** `allowed_fails_policy` works by incrementing a `failed_calls` counter per deployment. That counter expires after `cooldown_time` seconds. If `cooldown_time` is shorter than `health_check_interval`, the counter resets between every check cycle and failures never accumulate. Set `cooldown_time` greater than `health_check_interval` when using `allowed_fails_policy`.
```yaml
router_settings:
cooldown_time: 60 # must be > health_check_interval (30s here)
general_settings:
health_check_interval: 30
```
- **`AllowedFails: N` means cooldown on the (N+1)th failure.** The counter check is `updated_fails > allowed_fails`, so `0` triggers on the 1st failure, `1` on the 2nd, `2` on the 3rd.
| `AllowedFails` | Cooldown triggers after |
|---|---|
| `0` | 1st failure |
| `1` | 2nd failure |
| `2` | 3rd failure |
- **Without `allowed_fails_policy`, the first failure is enough.** The first failed health check immediately excludes the deployment from routing. Use `allowed_fails_policy` when you want tolerance for flaky checks.
- **If all deployments are unhealthy, the filter is bypassed.** Traffic keeps flowing rather than returning no deployment at all. Requests will fail, but the router keeps trying.
- **Health check failures and request failures share the same counters.** When `allowed_fails_policy` is set, both sources increment the same `failed_calls` counter. A deployment at 1 health check failure that then receives 1 failing request will hit the threshold for `AllowedFails: 1` and enter cooldown.
## Debugging
Run the proxy with `--detailed_debug` and look for these log lines:
After each health check cycle (written at DEBUG level):
```
health_check_routing_state_updated healthy=2 unhealthy=1
```
When a health check failure increments the counter and triggers cooldown (DEBUG level):
```
checks 'should_run_cooldown_logic'
Attempting to add <deployment_id> to cooldown list
```
When safety net fires because all deployments are in cooldown:
```
All deployments in cooldown via health-check routing, bypassing cooldown filter
```
When safety net fires because all deployments are unhealthy (binary filter, no `allowed_fails_policy`):
```
All deployments marked unhealthy by health checks, bypassing health filter
```

13
docs/my-website/docs/proxy/oauth2.md
View file

@ -63,16 +63,19 @@ Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more ve
## Using OAuth2 + JWT Together
If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths:
- JWT validation for user tokens
- OAuth2 introspection for machine tokens
LiteLLM supports two OAuth2 + JWT modes:
For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`:
1. **Global OAuth2 mode** (`enable_oauth2_auth: true`)
OAuth2 auth is enabled on LLM + info routes.
2. **Selective JWT override mode** (`enable_oauth2_auth: false`)
Only JWT-shaped tokens that match `litellm_jwtauth.routing_overrides` are routed to OAuth2 on LLM + info routes.
For selective routing (OAuth2 only for specific JWTs), configure:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
routing_overrides:
- iss: "machine-issuer.example.com"

25
docs/my-website/docs/proxy/self_serve.md
View file

@ -358,10 +358,15 @@ When you connect litellm to your SSO provider, litellm can auto-create teams. Us
```yaml showLineNumbers title="Default Params for new teams"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
- "/key/generate" # Allow members to generate API keys
```
@ -390,10 +395,14 @@ litellm_settings:
max_budget_in_team: 100 # Optional[float], optional): $100 budget for the team. Defaults to None.
user_role: "user" # Optional[str], optional): "user" or "admin". Defaults to "user"
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
tpm_limit: 100000 # Optional[int]: tokens per minute limit
rpm_limit: 1000 # Optional[int]: requests per minute limit
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity"
upperbound_key_generate_params: # Upperbound for /key/generate requests when self-serve flow is on

12
docs/my-website/docs/proxy/token_auth.md
View file

@ -792,16 +792,18 @@ litellm_jwtauth:
## Route JWT-Shaped Machine Tokens to OAuth2
Use this when both are enabled:
Use this when:
- `enable_jwt_auth: true` for standard JWT validation
- `enable_oauth2_auth: true` for OAuth2 introspection
- machine tokens are JWT-shaped and should be routed to OAuth2 based on claims
If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2.
`routing_overrides` supports two operating modes:
- **Selective mode**: set `enable_oauth2_auth: false` to send only matching JWTs to OAuth2 on LLM + info routes
- **Global mode**: set `enable_oauth2_auth: true` to also enable OAuth2 on LLM + info routes
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
user_id_jwt_field: "sub"
routing_overrides:
@ -822,7 +824,7 @@ general_settings:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
enable_oauth2_auth: false
litellm_jwtauth:
routing_overrides:
- iss: ["machine-issuer.example.com", "backup-issuer.example.com"]

10
docs/my-website/docs/tutorials/msft_sso.md
View file

@ -123,10 +123,12 @@ Navigate to your litellm config file and set the following params
```yaml showLineNumbers title="litellm config with default_team_params"
litellm_settings:
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
max_budget: 100 # Optional[float], optional): $100 budget for the team
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
max_budget: 100 # Optional[float]: $100 budget for the team
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
- "/team/daily/activity" # Allow members to view team usage
```
### 3.2 Auto-create a new team on LiteLLM

BIN
docs/my-website/img/release_notes/mcp_toolsets.jpeg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 111 KiB

BIN
docs/my-website/img/release_notes/skills_marketplace.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1 MiB

236
docs/my-website/release_notes/v1.83.3/index.md Normal file
View file

@ -0,0 +1,236 @@
---
title: "[Preview] v1.83.3.rc.1 - Introducing MCP Skills Marketplace"
slug: "v1-83-3-rc-1"
date: 2026-04-04T00:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Ryan Crabbe
title: Full Stack Engineer, LiteLLM
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
- name: Yuneng Jiang
title: Senior Full Stack Engineer, LiteLLM
url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
image_url: https://avatars.githubusercontent.com/u/171294688?v=4
- name: Shivam Rawat
title: Forward Deployed Engineer, LiteLLM
url: https://linkedin.com/in/shivam-rawat-482937318
image_url: https://github.com/shivamrawat1.png
hide_table_of_contents: false
---
## Deploy this version
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
<Tabs>
<TabItem value="docker" label="Docker">
```bash
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:main-v1.83.3.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
```bash
pip install litellm==1.83.3rc1
```
</TabItem>
</Tabs>
## Key Highlights
- **MCP Toolsets** — [Create curated tool subsets from one or more MCP servers with scoped permissions, and manage them from the UI or API](../../docs/mcp)
- **Skills Marketplace** — [Browse, install, and publish Claude Code skills from a self-hosted marketplace — works across Anthropic, Vertex AI, Azure, and Bedrock](../../docs/proxy/skills)
- **Guardrail Fallbacks** — [Configure `on_error` behavior so guardrail failures degrade gracefully instead of blocking the request](../../docs/proxy/guardrails)
- **Team Bring Your Own Guardrails** — [Teams can now attach and manage their own guardrails directly from team settings in the UI](../../docs/proxy/guardrails)
---
### Skills Marketplace
The Skills Marketplace gives teams a self-hosted catalog for discovering, installing, and publishing Claude Code skills. Skills are portable across Anthropic, Vertex AI, Azure, and Bedrock — so a skill published once works everywhere your gateway routes to.
![Skills Marketplace](../../img/release_notes/skills_marketplace.png)
[Get Started](../../docs/proxy/skills)
### Guardrail Fallbacks
Guardrail pipelines now support an optional `on_error` behavior. When a guardrail check fails or errors out, you can configure the pipeline to fall back gracefully — logging the failure and continuing the request — instead of returning a hard 500 to the caller. This is especially useful for non-critical guardrails where availability matters more than enforcement.
### Team Bring Your Own Guardrails
Teams can now attach guardrails directly from the team management UI. Admins configure available guardrails at the project or proxy level, and individual teams select which ones apply to their traffic — no config file changes or proxy restarts needed. This also ships with project-level guardrail support in the project create/edit flows.
### MCP Toolsets
MCP Toolsets let AI platform admins create curated subsets of tools from one or more MCP servers and assign them to teams and keys with scoped permissions. Instead of granting access to an entire MCP server, you can now bundle specific tools into a named toolset — controlling exactly which tools each team or API key can invoke. Toolsets are fully managed through the UI (new Toolsets tab) and API, and work seamlessly with the Responses API and Playground.
![MCP Toolsets](../../img/release_notes/mcp_toolsets.jpeg)
[Get Started](../../docs/mcp)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Brave Search | `brave/search` | - | - | - | Search tool integration metadata in cost map ([PR #25042](https://github.com/BerriAI/litellm/pull/25042)) |
| AWS Bedrock | `nvidia.nemotron-super-3-120b` | 256K | Added | Added | Chat completions, function calling, system messages ([PR #24588](https://github.com/BerriAI/litellm/pull/24588)) |
| OCI GenAI | Multiple new chat + embedding entries | Varies | Updated | Updated | Expanded chat + embedding model catalog |
#### Features
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Add Nova Canvas image edit support - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24869](https://github.com/BerriAI/litellm/pull/24869)
- Improve cache usage exposure for Claude-compatible streaming paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24850](https://github.com/BerriAI/litellm/pull/24850)
- Bedrock model catalog updates - [PR #24645](https://github.com/BerriAI/litellm/pull/24645)
- **[OCI GenAI](../../docs/providers/oci)**
- Add native embeddings support + expanded model catalog - [PR #25151](https://github.com/BerriAI/litellm/pull/25151), [PR #24887](https://github.com/BerriAI/litellm/pull/24887)
- **[Google Vertex AI](../../docs/providers/vertex)**
- Add unversioned Claude Haiku pricing entry to ensure accurate spend accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
### Bug Fixes
- **General**
- Fix `gpt-5.4` pricing metadata - [PR #24748](https://github.com/BerriAI/litellm/pull/24748)
- Fix gov pricing tests and Bedrock model test follow-ups - [PR #25022](https://github.com/BerriAI/litellm/pull/25022), [PR #24947](https://github.com/BerriAI/litellm/pull/24947), [PR #24931](https://github.com/BerriAI/litellm/pull/24931)
## LLM API Endpoints
#### Features
- **[A2A / MCP Gateway API (/a2a, /mcp)](../../docs/mcp)**
- Preserve JSON-RPC envelope for AgentCore A2A-native agents - [PR #25092](https://github.com/BerriAI/litellm/pull/25092)
- Bedrock Anthropic file/document handling fix from internal staging - [PR #25050](https://github.com/BerriAI/litellm/pull/25050), [PR #25047](https://github.com/BerriAI/litellm/pull/25047)
#### Bugs
- **[Search API (/search)](../../docs/search)**
- Support self-hosted Firecrawl response format in search transforms - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24866](https://github.com/BerriAI/litellm/pull/24866)
## Management Endpoints / UI
#### Features
- **Virtual Keys**
- Add substring search for `user_id` and `key_alias` on `/key/list` - [PR #24751](https://github.com/BerriAI/litellm/pull/24751), [PR #24746](https://github.com/BerriAI/litellm/pull/24746)
- Wire `team_id` filter to key alias dropdown on Virtual Keys tab - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25114](https://github.com/BerriAI/litellm/pull/25114)
- Allow hashed `token_id` in `/key/update` endpoint - [PR #24969](https://github.com/BerriAI/litellm/pull/24969)
- **Teams + Organizations**
- Resolve access-group models/MCP servers/agents in team endpoints and UI - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25027](https://github.com/BerriAI/litellm/pull/25027)
- Allow changing team organization from team settings - [PR #25095](https://github.com/BerriAI/litellm/pull/25095)
- Add per-model rate limits to team edit/info views - [PR #25156](https://github.com/BerriAI/litellm/pull/25156), [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
- **Usage + Analytics**
- Add paginated team search on usage page filters - [PR #25107](https://github.com/BerriAI/litellm/pull/25107)
- Use entity key for usage export display correctness - [PR #25153](https://github.com/BerriAI/litellm/pull/25153)
- **Models + Providers**
- Include access-group models in UI model listing - [PR #24743](https://github.com/BerriAI/litellm/pull/24743)
- Expose Azure Entra ID credential fields in provider forms - [PR #25137](https://github.com/BerriAI/litellm/pull/25137)
- Do not inject `vector_store_ids: []` when editing a model - [PR #25133](https://github.com/BerriAI/litellm/pull/25133)
- **Guardrails UI**
- Add project-level guardrails support in project create/edit flows - [PR #25100](https://github.com/BerriAI/litellm/pull/25100)
- Allow adding team guardrails from the UI - [PR #25038](https://github.com/BerriAI/litellm/pull/25038)
- **UI Cleanup**
- Migrate Tremor Text/Badge to antd Tag and native spans - [PR #24750](https://github.com/BerriAI/litellm/pull/24750)
#### Bugs
- Fix logs page showing unfiltered results when backend filter returns zero rows - [PR #24745](https://github.com/BerriAI/litellm/pull/24745)
- Enforce upperbound key params on `/key/update` and bulk update hook paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #25103](https://github.com/BerriAI/litellm/pull/25103)
- Fix team model update 500 due to unsupported Prisma JSON path filter - [PR #25152](https://github.com/BerriAI/litellm/pull/25152)
## AI Integrations
### Logging
- **General**
- Eliminate race condition in streaming `guardrail_information` logging - [PR #24592](https://github.com/BerriAI/litellm/pull/24592)
- Use actual `start_time` in failed request spend logs - [PR #24906](https://github.com/BerriAI/litellm/pull/24906)
- Harden credential redaction + stop logging raw sensitive auth values - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
### Guardrails
- Add optional `on_error` for guardrail pipeline failures - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24831](https://github.com/BerriAI/litellm/pull/24831)
- Return HTTP 400 (vs 500) for Model Armor streaming blocks - [PR #24693](https://github.com/BerriAI/litellm/pull/24693)
### Prompt Management
- Add environment + user tracking for prompts (`development/staging/production`) in CRUD + UI flows - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24855](https://github.com/BerriAI/litellm/pull/24855)
### Secret Managers
- No major new secret manager provider additions in this RC.
## Spend Tracking, Budgets and Rate Limiting
- Enforce budget for models not directly present in the cost map - [PR #24949](https://github.com/BerriAI/litellm/pull/24949)
- Add per-model rate limits in team settings/info UI - [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
- Fix unversioned Vertex Claude Haiku pricing entry to avoid `$0.00` accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
## MCP Gateway
- Introduce **MCP Toolsets** with DB types, CRUD APIs, scoped permissions, and UI management tab - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Resolve toolset names and enforce toolset access correctly in Responses API and streamable MCP paths - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Switch toolset permission caching to shared cache path and improve cache invalidation behavior - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Allow JWT auth for `/v1/mcp/server/*` sub-paths - [PR #25113](https://github.com/BerriAI/litellm/pull/25113), [PR #24698](https://github.com/BerriAI/litellm/pull/24698)
- Add STS AssumeRole support for MCP SigV4 auth - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
- Add tag query fix + MCP metadata support cherry-pick - [PR #25145](https://github.com/BerriAI/litellm/pull/25145)
## Performance / Loadbalancing / Reliability improvements
- Integrate router health-check failures with cooldown behavior and transient 429/408 handling - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24988](https://github.com/BerriAI/litellm/pull/24988)
- Add distributed lock for key rotation job execution - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #23364](https://github.com/BerriAI/litellm/pull/23364), [PR #23834](https://github.com/BerriAI/litellm/pull/23834)
- Improve team routing reliability with deterministic grouping, isolation fixes, stale alias controls, and order-based fallback - [PR #25154](https://github.com/BerriAI/litellm/pull/25154), [PR #25148](https://github.com/BerriAI/litellm/pull/25148)
- Regenerate GCP IAM token per async Redis cluster connection (fix token TTL failures) - [PR #25155](https://github.com/BerriAI/litellm/pull/25155), [PR #24426](https://github.com/BerriAI/litellm/pull/24426)
- Restore MCP server fields dropped by schema sync migration - [PR #24078](https://github.com/BerriAI/litellm/pull/24078)
- Proxy server reliability hardening with bounded queue usage - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
## Documentation Updates
- Improve HA control plane diagram clarity + mobile rendering updates - [PR #24747](https://github.com/BerriAI/litellm/pull/24747)
- Document `default_team_params` in config reference and examples - [PR #25032](https://github.com/BerriAI/litellm/pull/25032)
- Add JWT to Virtual Key mapping guide - [PR #24882](https://github.com/BerriAI/litellm/pull/24882)
- Add MCP Toolsets docs and sidebar updates - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
- Security docs updates and April hardening blog - [PR #24867](https://github.com/BerriAI/litellm/pull/24867), [PR #24868](https://github.com/BerriAI/litellm/pull/24868), [PR #24871](https://github.com/BerriAI/litellm/pull/24871), [PR #25102](https://github.com/BerriAI/litellm/pull/25102)
- General docs cleanup + townhall announcement updates - [PR #24839](https://github.com/BerriAI/litellm/pull/24839), [PR #25026](https://github.com/BerriAI/litellm/pull/25026), [PR #25021](https://github.com/BerriAI/litellm/pull/25021)
## Infrastructure / Security Notes
- Harden npm and Docker supply chain workflows and release pipeline checks - [PR #24838](https://github.com/BerriAI/litellm/pull/24838), [PR #24877](https://github.com/BerriAI/litellm/pull/24877), [PR #24881](https://github.com/BerriAI/litellm/pull/24881), [PR #24905](https://github.com/BerriAI/litellm/pull/24905), [PR #24951](https://github.com/BerriAI/litellm/pull/24951), [PR #25023](https://github.com/BerriAI/litellm/pull/25023), [PR #25034](https://github.com/BerriAI/litellm/pull/25034), [PR #25036](https://github.com/BerriAI/litellm/pull/25036), [PR #25037](https://github.com/BerriAI/litellm/pull/25037), [PR #25136](https://github.com/BerriAI/litellm/pull/25136), [PR #25158](https://github.com/BerriAI/litellm/pull/25158)
- Resolve CodeQL/security workflow issues and fix broken action SHA references - [PR #24880](https://github.com/BerriAI/litellm/pull/24880), [PR #24815](https://github.com/BerriAI/litellm/pull/24815)
- Re-add Codecov reporting in GHA matrix workflows - [PR #24804](https://github.com/BerriAI/litellm/pull/24804)
- Fix(docker): load enterprise hooks in non-root runtime image - [PR #24917](https://github.com/BerriAI/litellm/pull/24917)
- Apply Black formatting to 14 files - [PR #24532](https://github.com/BerriAI/litellm/pull/24532)
- Fix lint issues - [PR #24932](https://github.com/BerriAI/litellm/pull/24932)
## New Contributors
* @vanhtuan0409 made their first contribution in https://github.com/BerriAI/litellm/pull/24078
* @clfhhc made their first contribution in https://github.com/BerriAI/litellm/pull/24932
**Full Changelog**: https://github.com/BerriAI/litellm/compare/v1.83.0-nightly...v1.83.3.rc.1

4
docs/my-website/sidebars.js
View file

@ -325,6 +325,7 @@ const sidebars = {
"mcp_control",
"mcp_cost",
"mcp_guardrail",
"mcp_toolsets",
{
type: "link",
label: "MCP Troubleshooting Guide",
@ -1051,7 +1052,8 @@ const sidebars = {
"proxy/fallback_management",
"proxy/tag_routing",
"proxy/timeout",
"wildcard_routing"
"wildcard_routing",
"proxy/health_check_routing"
],
},
{

45
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -11,6 +11,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.constants import (
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
if TYPE_CHECKING:
@ -32,21 +33,49 @@ class CheckResponsesCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
async def _expire_stale_rows(
self, cutoff: datetime, batch_size: int
) -> int:
"""Execute the bounded UPDATE that marks stale rows as 'stale_expired'.
Isolated so it can be swapped / mocked in tests without touching the
orchestration logic in ``_cleanup_stale_managed_objects``.
Uses PostgreSQL syntax (``$1::timestamptz``, ``LIMIT``, double-quoted
identifiers) which is the only dialect the proxy supports — every
``schema.prisma`` in the repo sets ``provider = "postgresql"``.
Same pattern as ``spend_log_cleanup.py``.
"""
return await self.prisma_client.db.execute_raw(
"""
UPDATE "LiteLLM_ManagedObjectTable"
SET "status" = 'stale_expired'
WHERE "id" IN (
SELECT "id" FROM "LiteLLM_ManagedObjectTable"
WHERE "file_purpose" = 'response'
AND "status" NOT IN ('completed', 'complete', 'failed', 'expired', 'cancelled', 'stale_expired')
AND "created_at" < $1::timestamptz
ORDER BY "created_at" ASC
LIMIT $2
)
""",
cutoff,
batch_size,
)
async def _cleanup_stale_managed_objects(self) -> None:
"""
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
in non-terminal states as 'stale_expired'. These will never complete and
should not be polled.
Runs as a single DB query with a subquery LIMIT so no rows are loaded
into Python memory. Processes at most STALE_OBJECT_CLEANUP_BATCH_SIZE
rows per invocation to avoid overwhelming the DB when there is a large
backlog.
"""
cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={
"file_purpose": "response",
"status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
"created_at": {"lt": cutoff},
},
data={"status": "stale_expired"},
)
result = await self._expire_stale_rows(cutoff, STALE_OBJECT_CLEANUP_BATCH_SIZE)
if result > 0:
verbose_proxy_logger.warning(
f"CheckResponsesCost: marked {result} stale managed objects "

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.35"
version = "0.1.36"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -25,7 +25,7 @@ required-version = "==0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.35"
version = "0.1.36"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

19
litellm-proxy-extras/litellm_proxy_extras/migrations/20260321000000_add_mcp_toolsets/migration.sql Normal file
View file

@ -0,0 +1,19 @@
-- CreateTable: LiteLLM_MCPToolsetTable
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPToolsetTable" (
"toolset_id" TEXT NOT NULL,
"toolset_name" TEXT NOT NULL,
"description" TEXT,
"tools" JSONB NOT NULL DEFAULT '[]',
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_MCPToolsetTable_pkey" PRIMARY KEY ("toolset_id")
);
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_MCPToolsetTable_toolset_name_key" ON "LiteLLM_MCPToolsetTable"("toolset_name");
-- AlterTable: add mcp_toolsets to ObjectPermissionTable
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "mcp_toolsets" TEXT[] DEFAULT ARRAY[]::TEXT[];

12
litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql Normal file
View file

@ -0,0 +1,12 @@
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development';
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT;
-- DropIndex (old unique constraint)
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key";
-- CreateIndex (new unique constraint)
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment");
-- CreateIndex (new composite index)
CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment");

29
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -273,6 +273,7 @@ model LiteLLM_ObjectPermissionTable {
agent_access_groups String[] @default([])
models String[] @default([])
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
teams LiteLLM_TeamTable[]
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
@ -321,15 +322,28 @@ model LiteLLM_MCPServerTable {
byok_description String[] @default([])
byok_api_key_help_url String?
source_url String?
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
@@index([approval_status])
}
// Named collection of {server_id, tool_name} pairs that can be granted to keys/teams
model LiteLLM_MCPToolsetTable {
toolset_id String @id @default(uuid())
toolset_name String @unique
description String?
tools Json @default("[]") // [{server_id: string, tool_name: string}]
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
@ -1001,12 +1015,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.63"
version = "0.4.65"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -25,7 +25,7 @@ required-version = "==0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.63"
version = "0.4.65"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

1
litellm/__init__.py
View file

@ -1838,6 +1838,7 @@ if TYPE_CHECKING:
)
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
from .llms.lambda_ai.chat.transformation import (

38
litellm/_logging.py
View file

@ -26,6 +26,12 @@ _REDACTED = "REDACTED"
def _build_secret_patterns() -> re.Pattern:
patterns: List[str] = [
# ── PEM private key / certificate blocks ──
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# ── GCP OAuth2 access tokens (ya29.*) ──
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# ── Credential %s formatting (space separator, no key= prefix) ──
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern:
# Google API keys
r"AIza[0-9A-Za-z\-_]{35}",
# Password / secret params (handles key=value and 'key': 'value')
r"\w*(?:password|passwd|client_secret|secret_key|_secret)"
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern:
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"private_key|signing_key|encryption_key|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# ── Raw JWTs (without Bearer prefix) ──
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# ── Azure SAS tokens in URLs ──
r"[?&]sig=[A-Za-z0-9%+/=]+",
# ── Full JSON service-account blobs (single-line and multi-line) ──
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
@ -74,6 +89,23 @@ def _redact_string(value: str) -> str:
return _SECRET_RE.sub(_REDACTED, value)
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
Use this for code paths that bypass the logging system — e.g. Slack/Teams
alerting, HTTP error response bodies, or any other string that may contain
secrets and will be sent to an external sink.
Not to be confused with redact_message_input_output_from_logging() in
litellm_core_utils/redact_messages.py, which redacts LLM prompt/response
content for privacy — this function redacts credential patterns (API keys,
PEM blocks, tokens, etc.) by shape.
"""
if not _ENABLE_SECRET_REDACTION:
return value
return _redact_string(value)
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -441,7 +473,7 @@ def _enable_debugging():
def print_verbose(print_statement):
try:
if set_verbose:
print(print_statement) # noqa
print(redact_secrets(str(print_statement))) # noqa
except Exception:
pass

71
litellm/_redis.py
View file

@ -18,6 +18,10 @@ import redis # type: ignore
import redis.asyncio as async_redis # type: ignore
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
GCPIAMCredentialProvider,
_generate_gcp_iam_access_token,
)
from litellm.constants import REDIS_CONNECTION_POOL_TIMEOUT, REDIS_SOCKET_TIMEOUT
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
@ -107,33 +111,6 @@ def _redis_kwargs_from_environment():
return return_dict
def _generate_gcp_iam_access_token(service_account: str) -> str:
"""
Generate GCP IAM access token for Redis authentication.
Args:
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
Returns:
Access token string for GCP IAM authentication
"""
try:
from google.cloud import iam_credentials_v1
except ImportError:
raise ImportError(
"google-cloud-iam is required for GCP IAM Redis authentication. "
"Install it with: pip install google-cloud-iam"
)
client = iam_credentials_v1.IAMCredentialsClient()
request = iam_credentials_v1.GenerateAccessTokenRequest(
name=service_account,
scope=["https://www.googleapis.com/auth/cloud-platform"],
)
response = client.generate_access_token(request=request)
return str(response.access_token)
def create_gcp_iam_redis_connect_func(
service_account: str,
ssl_ca_certs: Optional[str] = None,
@ -266,7 +243,7 @@ def _get_redis_client_logic(**env_overrides):
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
)
# Store GCP service account in redis_connect_func for async cluster access
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account # type: ignore[attr-defined]
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("gcp_service_account", None)
@ -413,41 +390,13 @@ def get_redis_async_client(
# Handle GCP IAM authentication for async clusters
redis_connect_func = cluster_kwargs.pop("redis_connect_func", None)
from litellm import get_secret_str
# Get GCP service account - first try from redis_connect_func, then from environment
gcp_service_account = None
# Use a CredentialProvider so the IAM token is regenerated on every new
# connection — mirrors the sync path where redis_connect_func is invoked
# per connection. Without this, the token would expire after ~1 hour.
if redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
gcp_service_account = redis_connect_func._gcp_service_account
else:
gcp_service_account = redis_kwargs.get(
"gcp_service_account"
) or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
verbose_logger.debug(
f"DEBUG: Redis cluster kwargs: redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
)
# If GCP IAM is configured (indicated by redis_connect_func), generate access token and use as password
if redis_connect_func and gcp_service_account:
verbose_logger.debug(
"DEBUG: Generating IAM token for service account (value not logged for security reasons)"
)
try:
# Generate IAM access token using the helper function
access_token = _generate_gcp_iam_access_token(gcp_service_account)
cluster_kwargs["password"] = access_token
verbose_logger.debug(
"DEBUG: Successfully generated GCP IAM access token for async Redis cluster"
)
except Exception as e:
verbose_logger.error(f"Failed to generate GCP IAM access token: {e}")
from redis.exceptions import AuthenticationError
raise AuthenticationError("Failed to generate GCP IAM access token")
else:
verbose_logger.debug(
f"DEBUG: Not using GCP IAM auth - redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
new_startup_nodes: List[ClusterNode] = []

53
litellm/_redis_credential_provider.py Normal file
View file

@ -0,0 +1,53 @@
import asyncio
from typing import Tuple
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
def _generate_gcp_iam_access_token(service_account: str) -> str:
"""
Generate GCP IAM access token for Redis authentication.
Args:
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
Returns:
Access token string for GCP IAM authentication
"""
try:
from google.cloud import iam_credentials_v1
except ImportError:
raise ImportError(
"google-cloud-iam is required for GCP IAM Redis authentication. "
"Install it with: pip install google-cloud-iam"
)
client = iam_credentials_v1.IAMCredentialsClient()
request = iam_credentials_v1.GenerateAccessTokenRequest(
name=service_account,
scope=["https://www.googleapis.com/auth/cloud-platform"],
)
response = client.generate_access_token(request=request)
return str(response.access_token)
class GCPIAMCredentialProvider(CredentialProvider):
"""
redis.credentials.CredentialProvider implementation that generates a fresh GCP IAM
token on every new connection. This fixes the 1-hour token expiry issue for async
Redis cluster clients, which previously generated the token once at startup and
cached it as a static password.
"""
def __init__(self, gcp_service_account: str) -> None:
self._gcp_service_account = gcp_service_account
def get_credentials(self) -> Tuple[str]:
token = _generate_gcp_iam_access_token(self._gcp_service_account)
return (token,)
async def get_credentials_async(self) -> Tuple[str]:
token = await asyncio.to_thread(
_generate_gcp_iam_access_token, self._gcp_service_account
)
return (token,)

14
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
)
return response_data
@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler:
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk

6
litellm/a2a_protocol/providers/base.py
View file

@ -3,7 +3,7 @@ Base configuration for A2A protocol providers.
"""
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
class BaseA2AProviderConfig(ABC):
@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""

22
litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py Normal file
View file

@ -0,0 +1,22 @@
"""
Bedrock AgentCore A2A provider.
Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively,
bypassing the completion bridge that would otherwise strip the envelope.
"""
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
__all__ = [
"BedrockAgentCoreA2AConfig",
"BedrockAgentCoreA2AHandler",
"BedrockAgentCoreA2ATransformation",
]

61
litellm/a2a_protocol/providers/bedrock_agentcore/config.py Normal file
View file

@ -0,0 +1,61 @@
"""
Bedrock AgentCore A2A provider configuration.
"""
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig):
"""
Provider configuration for Bedrock AgentCore A2A-native agents.
AgentCore agents that speak A2A natively expect the full JSON-RPC envelope.
This config bypasses the completion bridge and forwards requests directly,
deriving the endpoint URL from the model ARN and signing with SigV4/JWT.
"""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
return await BedrockAgentCoreA2AHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
async for chunk in BedrockAgentCoreA2AHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
):
yield chunk

134
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py Normal file
View file

@ -0,0 +1,134 @@
"""
Handler for Bedrock AgentCore A2A-native agents.
Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the
completion bridge that would otherwise strip the envelope.
"""
import json
from typing import Any, AsyncIterator, Dict, cast
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
class BedrockAgentCoreA2AHandler:
"""
Handler for Bedrock AgentCore A2A requests.
Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig,
and POSTs directly to the AgentCore endpoint.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Returns:
A2A JSON-RPC response dict from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending non-streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
)
response.raise_for_status()
response_data = response.json()
if "error" in response_data:
verbose_logger.warning(
f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}"
)
return response_data
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Yields:
A2A streaming response events from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
stream=True,
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
stream=True,
)
response.raise_for_status()
# Check content type — AgentCore may return JSON instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
# Single JSON response fallback (not SSE)
verbose_logger.debug(
"BedrockAgentCore A2A streaming: received JSON instead of SSE, "
"yielding as single event"
)
response_body = await response.aread()
response_data = json.loads(response_body)
yield response_data
else:
# SSE stream — parse data: lines
async for event in BedrockAgentCoreA2ATransformation.parse_sse_events(
response
):
yield event

134
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py Normal file
View file

@ -0,0 +1,134 @@
"""
Transformation layer for Bedrock AgentCore A2A provider.
Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs,
and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
"""
import json
from typing import Any, AsyncIterator, Dict, Tuple
from litellm._logging import verbose_logger
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
Reuses AmazonAgentCoreConfig for URL construction, ARN parsing,
and request signing. No logic is duplicated.
"""
@staticmethod
def get_url_and_signed_request(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
method: str = "message/send",
stream: bool = False,
) -> Tuple[str, dict, bytes]:
"""
Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams
litellm_params: Agent's litellm_params (model, api_key, etc.)
method: JSON-RPC method name (default: "message/send")
stream: Whether this is a streaming request
Returns:
Tuple of (url, signed_headers, signed_body_bytes)
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
agentcore_config = AmazonAgentCoreConfig()
# Derive URL from ARN
url = agentcore_config.get_complete_url(
api_base=optional_params.get("api_base"),
api_key=optional_params.get("api_key"),
model=agentcore_model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)
# Construct JSON-RPC 2.0 envelope
json_rpc_body = {
"jsonrpc": "2.0",
"method": method,
"id": request_id,
"params": params,
}
# Set required AgentCore session headers (normally set by transform_request,
# which we skip because it also builds {"prompt": "..."})
headers: dict = {}
session_id = agentcore_config._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id = agentcore_config._get_runtime_user_id(optional_params)
if runtime_user_id:
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
# Sign the request (SigV4 or JWT depending on api_key presence)
signed_headers, signed_body = agentcore_config.sign_request(
headers=headers,
optional_params=optional_params,
request_data=json_rpc_body,
api_base=url,
api_key=optional_params.get("api_key"),
model=agentcore_model,
stream=stream,
)
# sign_request returns Optional[bytes] — ensure we have bytes
if signed_body is None:
signed_body = json.dumps(json_rpc_body).encode()
return url, signed_headers, signed_body
@staticmethod
async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]:
"""
Parse SSE events from an httpx streaming response.
Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict.
Args:
response: httpx streaming response
Yields:
Parsed JSON dicts from SSE data lines
"""
async for line in response.aiter_lines():
line = line.strip()
if not line:
continue
if line.startswith("data:"):
data_str = line[len("data:") :].strip()
if not data_str:
continue
try:
event = json.loads(data_str)
yield event
except json.JSONDecodeError:
verbose_logger.debug(
f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}"
)
continue

12
litellm/a2a_protocol/providers/config_manager.py
View file

@ -19,12 +19,14 @@ class A2AProviderConfigManager:
@staticmethod
def get_provider_config(
custom_llm_provider: Optional[str],
model: Optional[str] = None,
) -> Optional[BaseA2AProviderConfig]:
"""
Get the provider configuration for a given custom_llm_provider.
Args:
custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents")
model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock)
Returns:
Provider configuration instance or None if not found
@ -39,9 +41,11 @@ class A2AProviderConfigManager:
return PydanticAIProviderConfig()
# Add more providers here as needed
# elif custom_llm_provider == "another_provider":
# from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig
# return AnotherProviderConfig()
if custom_llm_provider == "bedrock" and model and "agentcore" in model:
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
return BedrockAgentCoreA2AConfig()
return None

10
litellm/a2a_protocol/providers/pydantic_ai_agents/config.py
View file

@ -2,7 +2,7 @@
Pydantic AI provider configuration.
"""
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to Pydantic AI agent."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
return await PydanticAIHandler.handle_non_streaming(
request_id=request_id,
params=params,
@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request with fake streaming."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
async for chunk in PydanticAIHandler.handle_streaming(
request_id=request_id,
params=params,

10
litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py
View file

@ -5,7 +5,7 @@ Pydantic AI agents follow A2A protocol but don't support streaming natively.
This handler provides fake streaming by converting non-streaming responses into streaming chunks.
"""
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import (
@ -26,7 +26,7 @@ class PydanticAIHandler:
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
@ -41,6 +41,8 @@ class PydanticAIHandler:
Returns:
A2A SendMessageResponse dict
"""
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(f"Pydantic AI: Routing to Pydantic AI agent at {api_base}")
# Send request directly to Pydantic AI agent
@ -57,7 +59,7 @@ class PydanticAIHandler:
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
timeout: float = 60.0,
chunk_size: int = 50,
delay_ms: int = 10,
@ -80,6 +82,8 @@ class PydanticAIHandler:
Yields:
A2A streaming response events
"""
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
)

12
litellm/constants.py
View file

@ -1319,6 +1319,9 @@ LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
LITELLM_KEY_ROTATION_GRACE_PERIOD: str = os.getenv(
"LITELLM_KEY_ROTATION_GRACE_PERIOD", ""
) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default)
LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int(
os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600)
) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
@ -1341,12 +1344,14 @@ LITELLM_UI_SESSION_DURATION = os.getenv("LITELLM_UI_SESSION_DURATION", "24h")
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME = "db_daily_tag_spend_update_job"
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
)
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
@ -1362,6 +1367,9 @@ MAX_OBJECTS_PER_POLL_CYCLE = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE",
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS = max(
1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))
)
STALE_OBJECT_CLEANUP_BATCH_SIZE = max(
1, int(os.getenv("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))
)
# Set PROXY_BATCH_POLLING_ENABLED=false to disable the CheckBatchCost and
# CheckResponsesCost background polling jobs entirely (e.g. to avoid DB load on
# installations with large numbers of stale managed objects).
@ -1393,6 +1401,10 @@ APSCHEDULER_REPLACE_EXISTING = os.getenv(
"1",
] # always replace existing jobs
# The number of tag entries are higher than number of user, team entries. This leads to a higher QPS.
# This will run tag spcific tasks at a later time to smooth QPS
DAILY_TAG_SPEND_BATCH_MULTIPLIER = 2.3
DEFAULT_HEALTH_CHECK_INTERVAL = int(
os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)
) # 5 minutes

50
litellm/experimental_mcp_client/client.py
View file

@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth):
aws_session_token: Optional[str] = None,
aws_region_name: Optional[str] = None,
aws_service_name: Optional[str] = None,
aws_role_name: Optional[str] = None,
aws_session_name: Optional[str] = None,
):
try:
from botocore.credentials import Credentials
@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth):
# Note: os.environ/ prefixed values are already resolved by
# ProxyConfig._check_for_os_environ_vars() at config load time.
# Values arrive here as plain strings.
if aws_access_key_id and aws_secret_access_key:
if aws_role_name:
self.credentials = self._assume_role(
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_region_name=self.region_name,
)
elif aws_access_key_id and aws_secret_access_key:
self.credentials = Credentials(
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth):
"(env vars, ~/.aws/credentials, instance profile)."
)
@staticmethod
def _assume_role(
aws_role_name: str,
aws_session_name: Optional[str],
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
aws_region_name: str,
):
"""Call STS AssumeRole and return temporary credentials."""
import boto3
from botocore.credentials import Credentials
session_name = (
aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
)
sts_kwargs: dict = {"region_name": aws_region_name}
if aws_access_key_id and aws_secret_access_key:
sts_kwargs["aws_access_key_id"] = aws_access_key_id
sts_kwargs["aws_secret_access_key"] = aws_secret_access_key
if aws_session_token:
sts_kwargs["aws_session_token"] = aws_session_token
sts_client = boto3.client("sts", **sts_kwargs)
sts_response = sts_client.assume_role(
RoleArn=aws_role_name,
RoleSessionName=session_name,
)
sts_creds = sts_response["Credentials"]
return Credentials(
access_key=sts_creds["AccessKeyId"],
secret_key=sts_creds["SecretAccessKey"],
token=sts_creds["SessionToken"],
)
def auth_flow(
self, request: httpx.Request
) -> Generator[httpx.Request, httpx.Response, None]:

5
litellm/integrations/azure_storage/azure_storage.py
View file

@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
"""
Gets Azure AD token to use for Azure Storage API requests
"""
verbose_logger.debug("Getting Azure AD Token from Azure Storage")
verbose_logger.debug(
"tenant_id %s, client_id %s, client_secret %s",
"Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]",
tenant_id,
client_id,
client_secret,
client_secret is not None,
)
if tenant_id is None:
raise ValueError(

8
litellm/integrations/gcs_bucket/gcs_bucket_base.py
View file

@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",
@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger):
custom_llm_provider="vertex_ai",
api_base=None,
)
verbose_logger.debug("constructed auth_header %s", auth_header)
verbose_logger.debug(
"constructed auth_header [set=%s]", auth_header is not None
)
headers = {
"Authorization": f"Bearer {auth_header}", # auth_header
"Content-Type": "application/json",

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -202,8 +202,8 @@ def get_llm_provider( # noqa: PLR0915
)
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
raise Exception(
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
dynamic_api_key
"dynamic_api_key needs to be a string. Got type={}".format(
type(dynamic_api_key).__name__
)
)
return model, custom_llm_provider, dynamic_api_key, api_base

254
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
View file

@ -38,6 +38,102 @@ class FakeAnthropicMessagesStreamIterator:
self.chunks = self._create_streaming_chunks()
self.current_index = 0
def _create_content_block_chunks(
self, block_dict: Dict[str, Any], index: int
) -> List[bytes]:
"""Build SSE chunks for a single content block."""
chunks = []
block_type = block_dict.get("type")
if block_type == "text":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
elif block_type == "thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "thinking_delta", "thinking": thinking_text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "signature_delta", "signature": signature},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
elif block_type == "redacted_thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "input_json_delta", "partial_json": json.dumps(input_data)},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
return chunks
def _create_streaming_chunks(self) -> List[bytes]:
"""Convert the non-streaming response to streaming chunks"""
chunks = []
@ -69,152 +165,34 @@ class FakeAnthropicMessagesStreamIterator:
# 2-4. For each content block, send start/delta/stop events
content_blocks = response_dict.get("content", [])
if content_blocks:
for index, block in enumerate(content_blocks):
# Cast block to dict for easier access
block_dict = cast(Dict[str, Any], block)
block_type = block_dict.get("type")
if block_type == "text":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send full text as one delta for simplicity)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "thinking":
# content_block_start for thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "thinking",
"thinking": "",
"signature": "",
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta for thinking text
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "thinking_delta",
"thinking": thinking_text,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_delta for signature (if present)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "signature_delta",
"signature": signature,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "redacted_thinking":
# content_block_start for redacted_thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_stop (no delta for redacted thinking)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "tool_use":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send input as JSON delta)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
for index, block in enumerate(content_blocks):
block_dict = cast(Dict[str, Any], block)
chunks.extend(self._create_content_block_chunks(block_dict, index))
# 5. message_delta event (with final usage and stop_reason)
# Include cache usage fields so clients that only read message_delta
# (like Claude Code's SDK) see the full input token breakdown.
delta_usage: Dict[str, Any] = {
"output_tokens": usage.get("output_tokens", 0) if usage else 0,
}
if usage:
if usage.get("input_tokens") is not None:
delta_usage["input_tokens"] = usage["input_tokens"]
if usage.get("cache_creation_input_tokens") is not None:
delta_usage["cache_creation_input_tokens"] = usage[
"cache_creation_input_tokens"
]
if usage.get("cache_read_input_tokens") is not None:
delta_usage["cache_read_input_tokens"] = usage[
"cache_read_input_tokens"
]
message_delta = {
"type": "message_delta",
"delta": {
"stop_reason": response_dict.get("stop_reason"),
"stop_sequence": response_dict.get("stop_sequence"),
},
"usage": {"output_tokens": usage.get("output_tokens", 0) if usage else 0},
"usage": delta_usage,
}
chunks.append(
f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode()

14
litellm/llms/azure/common_utils.py
View file

@ -101,17 +101,15 @@ def get_azure_ad_token_from_entra_id(
_client_secret = client_secret
verbose_logger.debug(
"tenant_id %s, client_id %s, client_secret %s",
"tenant_id=%s, client_id=%s, client_secret=[set=%s]",
_tenant_id,
_client_id,
_client_secret,
_client_secret is not None,
)
if _tenant_id is None or _client_id is None or _client_secret is None:
raise ValueError("tenant_id, client_id, and client_secret must be provided")
credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
verbose_logger.debug("credential %s", credential)
token_provider = get_bearer_token_provider(credential, scope)
verbose_logger.debug("token_provider %s", token_provider)
@ -140,10 +138,10 @@ def get_azure_ad_token_from_username_password(
from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
verbose_logger.debug(
"client_id %s, azure_username %s, azure_password %s",
"client_id=%s, azure_username=[set=%s], azure_password=[set=%s]",
client_id,
azure_username,
azure_password,
azure_username is not None,
azure_password is not None,
)
credential = UsernamePasswordCredential(
client_id=client_id,
@ -151,8 +149,6 @@ def get_azure_ad_token_from_username_password(
password=azure_password,
)
verbose_logger.debug("credential %s", credential)
token_provider = get_bearer_token_provider(credential, scope)
verbose_logger.debug("token_provider %s", token_provider)

16
litellm/llms/bedrock/base_aws_llm.py
View file

@ -156,24 +156,24 @@ class BaseAWSLLM:
verbose_logger.debug(
"in get credentials\n"
"aws_access_key_id=%s\n"
"aws_secret_access_key=%s\n"
"aws_session_token=%s\n"
"aws_access_key_id=[set=%s]\n"
"aws_secret_access_key=[set=%s]\n"
"aws_session_token=[set=%s]\n"
"aws_region_name=%s\n"
"aws_session_name=%s\n"
"aws_profile_name=%s\n"
"aws_role_name=%s\n"
"aws_web_identity_token=%s\n"
"aws_web_identity_token=[set=%s]\n"
"aws_sts_endpoint=%s\n"
"aws_external_id=%s",
aws_access_key_id,
aws_secret_access_key,
aws_session_token,
aws_access_key_id is not None,
aws_secret_access_key is not None,
aws_session_token is not None,
aws_region_name,
aws_session_name,
aws_profile_name,
aws_role_name,
aws_web_identity_token,
aws_web_identity_token is not None,
aws_sts_endpoint,
aws_external_id,
)

515
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py Normal file
View file

@ -0,0 +1,515 @@
"""
Amazon Nova Canvas image edit on Bedrock (InvokeModel).
Maps OpenAI-style image edit (image + prompt, optional mask) to Nova Canvas task types:
- With mask: INPAINTING (inPaintingParams per AWS docs)
- Without mask: IMAGE_VARIATION (imageVariationParams)
Refs:
- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html
- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html
"""
from __future__ import annotations
import base64
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
from litellm.utils import (
_get_model_cost_key,
_get_potential_model_names,
get_model_info,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
def _nova_canvas_task_body(
*,
image_b64: str,
mask_b64: Optional[str],
text: str,
negative_text: Optional[str],
similarity_strength: Optional[float],
task_type: Optional[str],
mask_prompt: Optional[str],
out_painting_mode: Optional[str],
) -> Dict[str, Any]:
"""Build InvokeModel body task section (without imageGenerationConfig)."""
if task_type == "BACKGROUND_REMOVAL":
return {
"taskType": "BACKGROUND_REMOVAL",
"backgroundRemovalParams": {"image": image_b64},
}
if task_type == "OUTPAINTING":
if mask_prompt is None and mask_b64 is None:
raise ValueError(
"OUTPAINTING requires either a mask image or a mask prompt. "
"Pass mask=<file> or maskPrompt=<str> in the request."
)
out_params: Dict[str, Any] = {
"image": image_b64,
"text": text,
}
if mask_prompt is not None:
out_params["maskPrompt"] = mask_prompt
elif mask_b64 is not None:
out_params["maskImage"] = mask_b64
if negative_text is not None:
out_params["negativeText"] = negative_text
if out_painting_mode is not None:
out_params["outPaintingMode"] = out_painting_mode
return {
"taskType": "OUTPAINTING",
"outPaintingParams": out_params,
}
# Honour explicit IMAGE_VARIATION even when a mask is present (mask is ignored
# for this task type; callers use INPAINTING when they want mask semantics).
if task_type == "IMAGE_VARIATION":
var_params_explicit: Dict[str, Any] = {
"images": [image_b64],
"text": text,
}
if negative_text is not None:
var_params_explicit["negativeText"] = negative_text
if similarity_strength is not None:
var_params_explicit["similarityStrength"] = similarity_strength
return {
"taskType": "IMAGE_VARIATION",
"imageVariationParams": var_params_explicit,
}
# Explicit taskType must be INPAINTING or omitted from here on; anything else is invalid.
if task_type is not None and str(task_type).strip() != "":
if task_type != "INPAINTING":
raise ValueError(
f"Unsupported Amazon Nova Canvas taskType: {task_type!r}. "
"Use BACKGROUND_REMOVAL, OUTPAINTING, IMAGE_VARIATION, INPAINTING, "
"or omit taskType for automatic routing (mask → INPAINTING, else IMAGE_VARIATION)."
)
if mask_b64 is not None or mask_prompt is not None or task_type == "INPAINTING":
in_params: Dict[str, Any] = {"image": image_b64, "text": text}
if mask_prompt is not None:
in_params["maskPrompt"] = mask_prompt
elif mask_b64 is not None:
in_params["maskImage"] = mask_b64
if negative_text is not None:
in_params["negativeText"] = negative_text
if "maskPrompt" not in in_params and "maskImage" not in in_params:
raise ValueError(
"Amazon Nova Canvas INPAINTING requires either maskPrompt or maskImage "
"(use OpenAI mask= for maskImage, or pass maskPrompt in optional params). "
"See https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html"
)
return {"taskType": "INPAINTING", "inPaintingParams": in_params}
var_params: Dict[str, Any] = {
"images": [image_b64],
"text": text,
}
if negative_text is not None:
var_params["negativeText"] = negative_text
if similarity_strength is not None:
var_params["similarityStrength"] = similarity_strength
return {
"taskType": "IMAGE_VARIATION",
"imageVariationParams": var_params,
}
def _file_types_to_b64(image: Optional[FileTypes]) -> str:
"""Encode OpenAI image input to base64 string for Nova Canvas."""
if image is None:
raise ValueError("Nova Canvas image edit requires an image input")
if hasattr(image, "read") and callable(getattr(image, "read", None)):
if hasattr(image, "seek"):
image.seek(0) # type: ignore[union-attr]
image_bytes = image.read() # type: ignore[union-attr]
return base64.b64encode(image_bytes).decode("utf-8")
if isinstance(image, bytes):
return base64.b64encode(image).decode("utf-8")
if isinstance(image, str):
return image
if isinstance(image, os.PathLike):
with open(image, "rb") as f:
return base64.b64encode(f.read()).decode("utf-8")
if isinstance(image, tuple):
raise ValueError(
"Nova Canvas image edit does not support tuple FileTypes. "
"Pass a file-like object, bytes, or a base64-encoded string."
)
return base64.b64encode(bytes(image)).decode("utf-8") # type: ignore[arg-type]
def _supports_nova_canvas_image_edit_from_model_cost(model: str) -> bool:
"""
True when model_cost has supports_nova_canvas_image_edit for a resolved catalog key.
get_model_info / ModelInfoBase omit arbitrary JSON keys, so we read model_cost
directly (same idea as supports_* bare_entry fallback).
"""
import litellm as _litellm
if not model:
return False
seen: set[str] = set()
candidates: List[str] = []
def _add(name: Optional[str]) -> None:
if name and name not in seen:
seen.add(name)
candidates.append(name)
_add(model)
if "/" in model:
suffix = model.split("/")[-1]
_add(suffix)
_add(f"bedrock/{suffix}")
# Cross-region inference ids (e.g. us.amazon.nova-canvas-v1:0) share pricing with
# the base model id (amazon.nova-canvas-v1:0) in model_cost.
try:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
base_model = BedrockModelInfo.get_base_model(model)
if base_model and base_model != model:
_add(base_model)
_add(f"bedrock/{base_model}")
except Exception:
pass
try:
potential = _get_potential_model_names(model=model, custom_llm_provider=None)
for field in (
"combined_model_name",
"combined_stripped_model_name",
"stripped_model_name",
"split_model",
):
raw = potential.get(field)
if isinstance(raw, str):
_add(raw)
except Exception:
pass
for name in candidates:
key = _get_model_cost_key(name)
if key is None:
continue
entry = _litellm.model_cost.get(key) or {}
if entry.get("supports_nova_canvas_image_edit") is True:
return True
return False
class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
"""
Bedrock InvokeModel image edit for amazon.nova-canvas-v1:0 and regional variants.
"""
@classmethod
def _is_nova_canvas_image_edit_model(cls, model: Optional[str] = None) -> bool:
"""
Use model_cost.supports_nova_canvas_image_edit so new Nova Canvas inference IDs
are added via model_prices_and_context_window.json only (not get_model_info, which
drops keys not on ModelInfoBase).
"""
return _supports_nova_canvas_image_edit_from_model_cost(model or "")
def get_supported_openai_params(self, model: str) -> list:
return [
"n",
"size",
"response_format",
"mask",
"negativeText",
"similarityStrength",
"cfgScale",
"seed",
"quality",
"taskType",
"maskPrompt",
"outPaintingMode",
"imageGenerationConfig",
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict[str, Any]:
supported = set(self.get_supported_openai_params(model))
mapped: Dict[str, Any] = dict(image_edit_optional_params)
_size = mapped.pop("size", None)
if _size is not None and isinstance(_size, str) and "x" in _size:
w, h = _size.split("x", 1)
try:
mapped["width"], mapped["height"] = int(w), int(h)
except ValueError:
pass
_n = mapped.pop("n", None)
if _n is not None:
mapped["numberOfImages"] = _n
_quality = mapped.pop("quality", None)
if _quality is not None:
if _quality in ("hd", "premium"):
mapped["quality"] = "premium"
elif _quality == "standard":
mapped["quality"] = "standard"
else:
# Re-emit unknown values (e.g. OpenAI "auto") so transform_image_edit_request
# forwards them and the API can reject, or drop_params can still apply upstream.
mapped["quality"] = _quality
# Accepted for OpenAI compatibility but ignored for Nova Canvas image edit;
# Bedrock returns base64 images only (no URL mode).
response_format = mapped.pop("response_format", None)
if response_format not in (None, "b64_json"):
verbose_logger.debug(
"Nova Canvas image edit ignores response_format=%s and returns base64 images",
response_format,
)
# Drop unknown keys if drop_params
if drop_params:
for k in list(mapped.keys()):
if k.startswith("_"):
continue
if k not in supported and k not in (
"width",
"height",
"numberOfImages",
"mask",
):
mapped.pop(k, None)
return mapped
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],
image: Optional[FileTypes],
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, Any]:
op = dict(image_edit_optional_request_params)
image_b64 = _file_types_to_b64(image)
mask_raw = op.pop("mask", None)
mask_b64: Optional[str] = None
if mask_raw is not None:
mask_b64 = _file_types_to_b64(mask_raw) # type: ignore[arg-type]
_size = op.pop("size", None)
width = op.pop("width", None)
height = op.pop("height", None)
if (
width is None
and height is None
and _size is not None
and isinstance(_size, str)
and "x" in _size
):
w, h = _size.split("x", 1)
try:
width, height = int(w), int(h)
except ValueError:
pass
number_of_images = op.pop("numberOfImages", None)
quality = op.pop("quality", None)
cfg_scale = op.pop("cfgScale", None)
seed = op.pop("seed", None)
image_generation_config: Dict[str, Any] = {}
nested_igc = op.pop("imageGenerationConfig", None)
if isinstance(nested_igc, dict):
image_generation_config.update(nested_igc)
if width is not None:
image_generation_config["width"] = width
if height is not None:
image_generation_config["height"] = height
if number_of_images is not None:
image_generation_config["numberOfImages"] = number_of_images
if quality is not None:
image_generation_config["quality"] = quality
if cfg_scale is not None:
image_generation_config["cfgScale"] = cfg_scale
if seed is not None:
image_generation_config["seed"] = seed
task_type = op.pop("taskType", None)
if (prompt is None or prompt == "") and task_type in (
"INPAINTING",
"OUTPAINTING",
):
raise ValueError(
f"Amazon Nova Canvas {task_type} requires a text prompt. "
"Pass a non-empty `prompt` in your request."
)
text = prompt if prompt is not None and prompt != "" else " "
negative_text = op.pop("negativeText", None)
similarity_strength = op.pop("similarityStrength", None)
mask_prompt = op.pop("maskPrompt", None)
out_painting_mode = op.pop("outPaintingMode", None)
body = _nova_canvas_task_body(
image_b64=image_b64,
mask_b64=mask_b64,
text=text,
negative_text=negative_text,
similarity_strength=similarity_strength,
task_type=task_type,
mask_prompt=mask_prompt,
out_painting_mode=out_painting_mode,
)
# BACKGROUND_REMOVAL InvokeModel body must not include imageGenerationConfig (AWS rejects it).
if image_generation_config and body.get("taskType") != "BACKGROUND_REMOVAL":
body["imageGenerationConfig"] = image_generation_config
return body, {}
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Nova Canvas image edit response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if raw_response.status_code not in (200,):
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
images: List[str] = response_data.get("images") or []
if "errors" in response_data and not images:
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs);
# it does not use Stability-style "finish_reasons".
error_msg = response_data.get("message") or response_data.get("error")
if error_msg and not images:
if not isinstance(error_msg, str):
error_msg = str(error_msg)
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {error_msg}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
model_response = ImageResponse()
model_response.data = []
for image_b64 in images:
if image_b64:
model_response.data.append(
ImageObject(
b64_json=image_b64,
url=None,
revised_prompt=None,
)
)
if not model_response.data:
raise self.get_error_class(
error_message="Nova Canvas image edit returned no images",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not hasattr(model_response, "_hidden_params"):
model_response._hidden_params = {}
if "additional_headers" not in model_response._hidden_params:
model_response._hidden_params["additional_headers"] = {}
try:
model_info = get_model_info(model, custom_llm_provider="bedrock")
cost_per_image = model_info.get("output_cost_per_image", 0)
if cost_per_image is not None and model_response.data:
model_response._hidden_params["additional_headers"][
"llm_provider-x-litellm-response-cost"
] = float(cost_per_image) * len(model_response.data)
except Exception:
pass
return model_response
def use_multipart_form_data(self) -> bool:
return False
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
raise NotImplementedError(
"Nova Canvas image edit URLs are built in BedrockImageEdit._prepare_request "
"(AWS runtime endpoint + model invoke path). Do not use get_complete_url for "
"this config."
)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
if headers is None:
headers = {}
if "Content-Type" not in headers:
headers["Content-Type"] = "application/json"
return headers
def get_bedrock_image_edit_config_for_model(
model: str,
) -> BaseImageEditConfig:
"""
Return the correct Bedrock image-edit config for the model id.
Same routing as ``BedrockImageEdit.get_config_class``: Stability edit models,
Nova Canvas when marked in model_cost; otherwise raises ``ValueError``.
"""
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
return BedrockStabilityImageEditConfig()
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model):
return BedrockAmazonNovaCanvasImageEditConfig()
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "
"Use a stability.* image-edit model id or add supports_nova_canvas_image_edit "
"in model_prices for this id."
)

14
litellm/llms/bedrock/image_edit/handler.py
View file

@ -15,6 +15,9 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import (
BedrockAmazonNovaCanvasImageEditConfig,
)
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
@ -55,8 +58,15 @@ class BedrockImageEdit(BaseAWSLLM):
def get_config_class(cls, model: str | None):
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
return BedrockStabilityImageEditConfig
else:
raise ValueError(f"Unsupported model for bedrock image edit: {model}")
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(
model
):
return BedrockAmazonNovaCanvasImageEditConfig
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "
"Use a stability.* image-edit model id or add supports_nova_canvas_image_edit "
"in model_prices for this id."
)
def image_edit(
self,

80
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -502,6 +502,12 @@ class AmazonAnthropicClaudeMessagesConfig(
):
"""
Bedrock invoke does not return SSE formatted data. This function is a wrapper to ensure litellm chunks are SSE formatted.
Bedrock's Anthropic-compatible streaming puts cache usage fields
(cache_creation_input_tokens, cache_read_input_tokens) only on
message_stop, not on message_start or message_delta. Claude Code's
SDK only merges usage from message_delta, so we promote those fields
from message_stop onto message_delta before yielding.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
@ -512,9 +518,81 @@ class AmazonAnthropicClaudeMessagesConfig(
request_body=request_body,
)
async for chunk in handler.async_sse_wrapper(completion_stream):
patched_stream = self._promote_message_stop_usage(completion_stream)
async for chunk in handler.async_sse_wrapper(patched_stream):
yield chunk
@staticmethod
async def _promote_message_stop_usage(
completion_stream: AsyncIterator[
Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]
],
) -> AsyncIterator[Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]]:
"""
Promote cache usage fields from message_stop onto message_delta.
Bedrock reports input_tokens (uncached only) on message_start, and
the full breakdown (input_tokens, cache_creation_input_tokens,
cache_read_input_tokens) only on message_stop. Claude Code's SDK
merges usage from message_start and message_delta but ignores
message_stop. This method buffers message_delta and, when
message_stop arrives with cache usage, merges those fields into the
message_delta usage and also updates the input_tokens on
message_delta to include the full count (uncached + cache_creation +
cache_read).
"""
_CACHE_FIELDS = ("cache_creation_input_tokens", "cache_read_input_tokens")
pending_delta = None
async for chunk in completion_stream:
if not isinstance(chunk, dict):
if pending_delta is not None:
yield pending_delta
pending_delta = None
yield chunk
continue
chunk_type = chunk.get("type")
if chunk_type == "message_delta":
pending_delta = chunk
continue
if chunk_type == "message_stop" and pending_delta is not None:
stop_usage = dict(chunk.get("usage") or {})
delta_usage = dict(pending_delta.get("usage") or {})
for field in _CACHE_FIELDS:
if field in stop_usage:
delta_usage[field] = stop_usage[field]
raw_input = stop_usage.get("input_tokens")
if raw_input is not None:
uncached = raw_input if isinstance(raw_input, int) else 0
raw_cc = delta_usage.get("cache_creation_input_tokens", 0)
cache_creation = raw_cc if isinstance(raw_cc, int) else 0
raw_cr = delta_usage.get("cache_read_input_tokens", 0)
cache_read = raw_cr if isinstance(raw_cr, int) else 0
delta_usage["input_tokens"] = uncached + cache_creation + cache_read
if delta_usage:
pending_delta["usage"] = delta_usage # type: ignore[arg-type]
yield pending_delta
pending_delta = None
yield chunk
continue
if pending_delta is not None:
yield pending_delta
pending_delta = None
yield chunk
if pending_delta is not None:
yield pending_delta
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
def __init__(

80
litellm/llms/firecrawl/search/transformation.py
View file

@ -159,15 +159,13 @@ class FirecrawlSearchConfig(BaseSearchConfig):
"""
Transform Firecrawl API response to LiteLLM unified SearchResponse format.
Firecrawl → LiteLLM mappings:
- data.web[].title → SearchResult.title
- data.web[].url → SearchResult.url
- data.web[].description OR data.web[].markdown → SearchResult.snippet
- No date field in web results (set to None)
- No last_updated field in Firecrawl response (set to None)
Supports both response formats:
Note: Firecrawl v2 returns results organized by source type (web, images, news).
We primarily use web results for the unified format.
Firecrawl Cloud (v2):
{"data": {"web": [...], "news": [...]}}
Firecrawl Self-Hosted (v1):
{"success": true, "data": [{"url": "...", "title": "...", ...}, ...]}
Args:
raw_response: Raw httpx response from Firecrawl API
@ -181,36 +179,52 @@ class FirecrawlSearchConfig(BaseSearchConfig):
# Transform results to SearchResult objects
results = []
# Process web results (primary source)
data = response_json.get("data", {})
web_results = data.get("web", [])
for result in web_results:
# Use markdown if available, otherwise fall back to description
snippet = result.get("markdown") or result.get("description", "")
if isinstance(data, list):
# Self-hosted Firecrawl (v1) format: data is a flat list of results
for result in data:
snippet = (
result.get("markdown") or result.get("description", "")
)
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None,
last_updated=None,
)
results.append(search_result)
elif isinstance(data, dict):
# Firecrawl Cloud (v2) format: data is a dict with web/news keys
web_results = data.get("web", [])
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None, # Web results don't include date
last_updated=None, # Firecrawl doesn't provide last_updated in response
)
results.append(search_result)
for result in web_results:
# Use markdown if available, otherwise fall back to description
snippet = result.get("markdown") or result.get("description", "")
# Process news results if available (they have date field)
news_results = data.get("news", [])
for result in news_results:
snippet = result.get("markdown") or result.get("snippet", "")
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None,
last_updated=None,
)
results.append(search_result)
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=result.get("date"), # News results include date
last_updated=None,
)
results.append(search_result)
# Process news results if available (they have date field)
news_results = data.get("news", [])
for result in news_results:
snippet = result.get("markdown") or result.get("snippet", "")
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=result.get("date"), # News results include date
last_updated=None,
)
results.append(search_result)
return SearchResponse(
results=results,

9
litellm/llms/oci/chat/transformation.py
View file

@ -174,10 +174,15 @@ def load_private_key_from_file(file_path: str):
def get_vendor_from_model(model: str) -> OCIVendors:
"""
Extracts the vendor from the model name.
OCI GenAI API uses two apiFormat values:
- "COHERE" for Cohere models (command-r, command-a, etc.)
- "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.)
Args:
model (str): The model name.
model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct").
Returns:
str: The vendor name.
OCIVendors: The vendor enum value.
"""
vendor = model.split(".")[0].lower()
if vendor == "cohere":

0
tests/proxy_e2e_azure_batches_tests/__init__.py → litellm/llms/oci/embed/__init__.py
View file

347
litellm/llms/oci/embed/transformation.py Normal file
View file

@ -0,0 +1,347 @@
"""
OCI Generative AI Embedding Configuration
Supports embedding models available on Oracle Cloud Infrastructure Generative AI service.
Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer).
Supported models:
- cohere.embed-english-v3.0
- cohere.embed-english-light-v3.0
- cohere.embed-multilingual-v3.0
- cohere.embed-multilingual-light-v3.0
- cohere.embed-english-image-v3.0
- cohere.embed-english-light-image-v3.0
- cohere.embed-multilingual-light-image-v3.0
- cohere.embed-v4.0
Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText
"""
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.llms.oci.chat.transformation import OCIChatConfig
from litellm.llms.oci.common_utils import OCIError
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
# Input type mapping from OpenAI conventions to OCI/Cohere conventions
_INPUT_TYPE_MAP = {
"search_document": "SEARCH_DOCUMENT",
"search_query": "SEARCH_QUERY",
"classification": "CLASSIFICATION",
"clustering": "CLUSTERING",
}
class OCIEmbeddingConfig(BaseEmbeddingConfig):
"""
Configuration for OCI Generative AI Embedding API.
The OCI embedding endpoint uses the Cohere embed models hosted on OCI.
Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer).
Usage:
```python
import litellm
response = litellm.embedding(
model="oci/cohere.embed-english-v3.0",
input=["Hello world", "Goodbye world"],
oci_compartment_id="ocid1.compartment.oc1..xxx",
oci_region="us-ashburn-1",
oci_user="ocid1.user.oc1..xxx",
oci_fingerprint="xx:xx:xx:xx",
oci_tenancy="ocid1.tenancy.oc1..xxx",
oci_key_file="~/.oci/key.pem",
)
```
"""
def __init__(self) -> None:
# We reuse OCIChatConfig for signing logic
self._chat_config = OCIChatConfig()
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
if api_base:
return api_base
oci_region = optional_params.get("oci_region", "us-ashburn-1")
return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText"
def get_supported_openai_params(self, model: str) -> list:
return [
"dimensions",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
# Note: OCI Cohere embed does not support custom dimensions natively,
# but we pass it through in case future models support it
if "dimensions" in non_default_params:
optional_params["dimensions"] = non_default_params["dimensions"]
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate OCI credentials for embedding requests.
Supports both OCI SDK Signer and manual credential signing.
"""
oci_signer = optional_params.get("oci_signer")
oci_region = optional_params.get("oci_region", "us-ashburn-1")
api_base = (
api_base
or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
)
if oci_signer is None:
oci_user = optional_params.get("oci_user")
oci_fingerprint = optional_params.get("oci_fingerprint")
oci_tenancy = optional_params.get("oci_tenancy")
oci_key = optional_params.get("oci_key")
oci_key_file = optional_params.get("oci_key_file")
oci_compartment_id = optional_params.get("oci_compartment_id")
if (
not oci_user
or not oci_fingerprint
or not oci_tenancy
or not (oci_key or oci_key_file)
or not oci_compartment_id
):
raise Exception(
"Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id "
"and at least one of oci_key or oci_key_file. "
"Alternatively, provide an oci_signer object from the OCI SDK."
)
from litellm.llms.custom_httpx.http_handler import version
headers.update(
{
"content-type": "application/json",
"user-agent": f"litellm/{version}",
}
)
return headers
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
):
"""Delegate to OCIChatConfig's signing logic."""
return self._chat_config.sign_request(
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
api_key=api_key,
model=model,
stream=stream,
fake_stream=fake_stream,
)
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
api_base: Optional[str] = None,
) -> dict:
"""
Transform the embedding request to OCI format.
OCI embedText API expects:
{
"compartmentId": "...",
"servingMode": {"servingType": "ON_DEMAND", "modelId": "..."},
"inputs": ["text1", "text2"],
"truncate": "END",
"inputType": "SEARCH_DOCUMENT"
}
"""
oci_compartment_id = optional_params.get("oci_compartment_id")
if not oci_compartment_id:
raise Exception(
"kwarg `oci_compartment_id` is required for OCI embedding requests"
)
# Build serving mode
oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND")
if oci_serving_mode == "DEDICATED":
oci_endpoint_id = optional_params.get("oci_endpoint_id", model)
serving_mode = {
"servingType": "DEDICATED",
"endpointId": oci_endpoint_id,
}
else:
serving_mode = {
"servingType": "ON_DEMAND",
"modelId": model,
}
# Normalize input to list of strings
if isinstance(input, str):
inputs = [input]
elif isinstance(input, list):
inputs = []
for item in input:
if isinstance(item, str):
inputs.append(item)
elif isinstance(item, list):
raise ValueError(
"OCI embedding does not support token-array inputs. "
"Please convert token lists to strings before calling embedding()."
)
else:
inputs.append(str(item))
else:
inputs = [str(input)]
# Build request data — OCI embedText API expects inputs, truncate,
# and inputType at the top level alongside compartmentId and servingMode
request_data: Dict[str, Any] = {
"compartmentId": oci_compartment_id,
"servingMode": serving_mode,
"inputs": inputs,
"truncate": optional_params.get("truncate", "END"),
}
# Map input_type if provided
input_type = optional_params.get("input_type")
if input_type:
mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper())
request_data["inputType"] = mapped_type
# Sign the request using the same URL the HTTP handler will POST to
signing_url = self.get_complete_url(
api_base=api_base,
api_key=None,
model=model,
optional_params=optional_params,
litellm_params={},
)
signed_headers, body = self.sign_request(
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=signing_url,
)
headers.update(signed_headers)
return request_data
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
request_data: dict = {},
optional_params: dict = {},
litellm_params: dict = {},
) -> EmbeddingResponse:
"""
Transform OCI embedding response to standard EmbeddingResponse format.
OCI response format:
{
"embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]],
"modelId": "cohere.embed-english-v3.0",
"modelVersion": "3.0",
"inputTextTokenCounts": [5, 4]
}
"""
if raw_response.status_code != 200:
raise OCIError(
message=raw_response.text,
status_code=raw_response.status_code,
)
try:
raw_response_json = raw_response.json()
except Exception:
raise OCIError(
message=raw_response.text,
status_code=raw_response.status_code,
)
embeddings = raw_response_json.get("embeddings", [])
model_id = raw_response_json.get("modelId", model)
# Build response data in OpenAI format
embedding_data = []
for idx, embedding in enumerate(embeddings):
embedding_data.append(
{
"object": "embedding",
"index": idx,
"embedding": embedding,
}
)
model_response.model = model_id
model_response.data = embedding_data
model_response.object = "list"
# Calculate token usage
input_token_counts = raw_response_json.get("inputTextTokenCounts", [])
total_tokens = sum(input_token_counts) if input_token_counts else 0
usage = Usage(
prompt_tokens=total_tokens,
total_tokens=total_tokens,
)
model_response.usage = usage
return model_response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
return OCIError(
message=error_message,
status_code=status_code,
headers=headers if isinstance(headers, httpx.Headers) else None,
)

28
litellm/llms/vertex_ai/vertex_llm_base.py
View file

@ -81,26 +81,26 @@ class VertexBase:
) -> Tuple[Any, str]:
if credentials is not None:
if isinstance(credentials, str):
_is_path = os.path.exists(
credentials
) # credentials is from server config (litellm_params), not user input
verbose_logger.debug(
"Vertex: Loading vertex credentials from %s", credentials
)
verbose_logger.debug(
"Vertex: checking if credentials is a valid path, os.path.exists(%s)=%s, current dir %s",
credentials,
os.path.exists(credentials),
"Vertex: Loading vertex credentials, is_file_path=%s, current dir %s",
_is_path,
os.getcwd(),
)
try:
if os.path.exists(credentials):
json_obj = json.load(open(credentials))
if _is_path:
with open(credentials) as f:
json_obj = json.load(f)
else:
json_obj = json.loads(credentials)
except Exception:
except Exception as e:
raise Exception(
"Unable to load vertex credentials from environment. Got={}".format(
credentials
)
"Unable to load vertex credentials from environment. "
"Ensure the JSON is valid (check for unescaped newlines in private_key). "
"Parse error: {}".format(type(e).__name__)
)
elif isinstance(credentials, dict):
json_obj = credentials
@ -668,8 +668,8 @@ class VertexBase:
## VALIDATION STEP
if _credentials.token is None or not isinstance(_credentials.token, str):
raise ValueError(
"Could not resolve credentials token. Got None or non-string token - {}".format(
_credentials.token
"Could not resolve credentials token. Got None or non-string token (type={})".format(
type(_credentials.token).__name__
)
)

68
litellm/main.py
View file

@ -3792,9 +3792,9 @@ def completion( # type: ignore # noqa: PLR0915
"aws_region_name" not in optional_params
or optional_params["aws_region_name"] is None
):
optional_params[
"aws_region_name"
] = aws_bedrock_client.meta.region_name
optional_params["aws_region_name"] = (
aws_bedrock_client.meta.region_name
)
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
if bedrock_route == "converse":
@ -5668,6 +5668,22 @@ def embedding( # noqa: PLR0915
aembedding=aembedding,
litellm_params={},
)
elif custom_llm_provider == "oci":
response = base_llm_http_handler.embedding(
model=model,
input=input,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
logging_obj=logging,
timeout=timeout,
model_response=EmbeddingResponse(),
optional_params=optional_params,
client=client,
aembedding=aembedding,
litellm_params=litellm_params_dict,
headers=headers,
)
elif custom_llm_provider in litellm._custom_providers:
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
@ -6198,9 +6214,9 @@ def adapter_completion(
new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs)
response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs) # type: ignore
translated_response: Optional[
Union[BaseModel, AdapterCompletionStreamWrapper]
] = None
translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = (
None
)
if isinstance(response, ModelResponse):
translated_response = translation_obj.translate_completion_output_params(
response=response
@ -6380,9 +6396,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
if existing_duration is None:
calculated_duration = calculate_request_duration(file)
if calculated_duration is not None:
response._hidden_params[
"audio_transcription_duration"
] = calculated_duration
response._hidden_params["audio_transcription_duration"] = (
calculated_duration
)
return response
except Exception as e:
@ -6605,9 +6621,9 @@ def transcription(
if existing_duration is None:
calculated_duration = calculate_request_duration(file)
if calculated_duration is not None:
response._hidden_params[
"audio_transcription_duration"
] = calculated_duration
response._hidden_params["audio_transcription_duration"] = (
calculated_duration
)
if response is None:
raise ValueError("Unmapped provider passed in. Unable to get the response.")
@ -6911,9 +6927,9 @@ def speech( # noqa: PLR0915
ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY
] = query_params
litellm_params_dict[
ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY
] = voice_id
litellm_params_dict[ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY] = (
voice_id
)
if api_base is not None:
litellm_params_dict["api_base"] = api_base
@ -7234,7 +7250,8 @@ async def ahealth_check(
if mode is None:
return {
"error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}"
"error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}",
"exception": e,
}
error_to_return = str(e) + "\nstack trace: " + stack_trace
@ -7246,6 +7263,7 @@ async def ahealth_check(
return {
"error": error_to_return,
"raw_request_typed_dict": raw_request_typed_dict,
"exception": e,
}
@ -7492,9 +7510,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(content_chunks) > 0:
response["choices"][0]["message"][
"content"
] = processor.get_combined_content(content_chunks)
response["choices"][0]["message"]["content"] = (
processor.get_combined_content(content_chunks)
)
thinking_blocks = [
chunk
@ -7505,9 +7523,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(thinking_blocks) > 0:
response["choices"][0]["message"][
"thinking_blocks"
] = processor.get_combined_thinking_content(thinking_blocks)
response["choices"][0]["message"]["thinking_blocks"] = (
processor.get_combined_thinking_content(thinking_blocks)
)
reasoning_chunks = [
chunk
@ -7518,9 +7536,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(reasoning_chunks) > 0:
response["choices"][0]["message"][
"reasoning_content"
] = processor.get_combined_reasoning_content(reasoning_chunks)
response["choices"][0]["message"]["reasoning_content"] = (
processor.get_combined_reasoning_content(reasoning_chunks)
)
annotation_chunks = [
chunk

389
litellm/model_prices_and_context_window_backup.json
View file

@ -277,7 +277,15 @@
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.writer.palmyra-x4-v1:0": {
"input_cost_per_token": 2.5e-06,
@ -7814,8 +7822,8 @@
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
@ -7830,6 +7838,26 @@
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
@ -7965,8 +7993,8 @@
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
@ -7981,6 +8009,26 @@
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
"litellm_provider": "bedrock",
@ -23795,7 +23843,8 @@
"output_cost_per_token": 2e-06,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
"supports_response_schema": false,
"supports_vision": true
},
"oci/meta.llama-3.3-70b-instruct": {
"input_cost_per_token": 7.2e-07,
@ -23929,6 +23978,287 @@
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/cohere.command-a-reasoning-08-2025": {
"input_cost_per_token": 1.56e-06,
"litellm_provider": "oci",
"max_input_tokens": 256000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1.56e-06,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/cohere.command-a-vision-07-2025": {
"input_cost_per_token": 1.56e-06,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1.56e-06,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true
},
"oci/cohere.command-a-translate-08-2025": {
"input_cost_per_token": 9e-08,
"litellm_provider": "oci",
"max_input_tokens": 256000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 9e-08,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": false,
"supports_response_schema": false
},
"oci/cohere.command-r-08-2024": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/cohere.command-r-plus-08-2024": {
"input_cost_per_token": 1.56e-06,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1.56e-06,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/meta.llama-3.2-11b-vision-instruct": {
"input_cost_per_token": 2e-06,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 2e-06,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false,
"supports_vision": true
},
"oci/meta.llama-3.1-70b-instruct": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/meta.llama-3.3-70b-instruct-fp8-dynamic": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 7.2e-07,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/xai.grok-4-fast": {
"input_cost_per_token": 5e-06,
"litellm_provider": "oci",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/xai.grok-4.1-fast": {
"input_cost_per_token": 5e-06,
"litellm_provider": "oci",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/xai.grok-4.20": {
"input_cost_per_token": 3e-06,
"litellm_provider": "oci",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/xai.grok-4.20-multi-agent": {
"input_cost_per_token": 3e-06,
"litellm_provider": "oci",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/xai.grok-code-fast-1": {
"input_cost_per_token": 5e-06,
"litellm_provider": "oci",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": false
},
"oci/google.gemini-2.5-pro": {
"input_cost_per_token": 1.25e-06,
"litellm_provider": "oci",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": true
},
"oci/google.gemini-2.5-flash": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "oci",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": true
},
"oci/google.gemini-2.5-flash-lite": {
"input_cost_per_token": 7.5e-08,
"litellm_provider": "oci",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3e-07,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_vision": true
},
"oci/cohere.embed-english-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
},
"oci/cohere.embed-english-light-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 384,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
},
"oci/cohere.embed-multilingual-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
},
"oci/cohere.embed-multilingual-light-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 384,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
},
"oci/cohere.embed-english-image-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1024,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_embedding_image_input": true
},
"oci/cohere.embed-english-light-image-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 384,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_embedding_image_input": true
},
"oci/cohere.embed-multilingual-light-image-v3.0": {
"input_cost_per_token": 1e-07,
"litellm_provider": "oci",
"max_input_tokens": 512,
"max_tokens": 512,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 384,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_embedding_image_input": true
},
"oci/cohere.embed-v4.0": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "oci",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 1536,
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
"supports_embedding_image_input": true
},
"ollama/codegeex4": {
"input_cost_per_token": 0.0,
"litellm_provider": "ollama",
@ -28655,6 +28985,32 @@
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
},
"us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"input_cost_per_token_above_200k_tokens": 6.6e-06,
"output_cost_per_token_above_200k_tokens": 2.475e-05,
"cache_creation_input_token_cost_above_200k_tokens": 8.25e-06,
"cache_read_input_token_cost_above_200k_tokens": 6.6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
},
"au.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06,
"cache_read_input_token_cost": 1.1e-07,
@ -30297,6 +30653,27 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"vertex_ai/claude-haiku-4-5": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_native_streaming": true,
"supports_vision": true
},
"vertex_ai/claude-haiku-4-5@20251001": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_read_input_token_cost": 1e-07,

16
litellm/proxy/_experimental/mcp_server/mcp_context.py Normal file
View file

@ -0,0 +1,16 @@
"""
Shared ContextVars for the MCP server layer.
Lives in its own module to avoid circular imports between
mcp_server_manager.py and server.py.
"""
from contextvars import ContextVar
from typing import Optional
# Set server-side in proxy_server.py route handlers when a request arrives via
# /toolset/{name}/mcp or the toolset fallback in dynamic_mcp_route.
# Never populated from client-supplied headers.
_mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
"_mcp_active_toolset_id", default=None
)

192
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -346,6 +346,8 @@ class MCPServerManager:
aws_session_token=server_config.get("aws_session_token", None),
aws_region_name=server_config.get("aws_region_name", None),
aws_service_name=server_config.get("aws_service_name", None),
aws_role_name=server_config.get("aws_role_name", None),
aws_session_name=server_config.get("aws_session_name", None),
)
self.config_mcp_servers[server_id] = new_server
@ -501,12 +503,12 @@ class MCPServerManager:
)
# Update tool name to server name mapping (for both prefixed and base names)
self.tool_name_to_mcp_server_name_mapping[
base_tool_name
] = server_prefix
self.tool_name_to_mcp_server_name_mapping[
prefixed_tool_name
] = server_prefix
self.tool_name_to_mcp_server_name_mapping[base_tool_name] = (
server_prefix
)
self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = (
server_prefix
)
registered_count += 1
verbose_logger.debug(
@ -686,6 +688,8 @@ class MCPServerManager:
aws_session_token=aws_creds.get("aws_session_token"),
aws_region_name=aws_creds.get("aws_region_name"),
aws_service_name=aws_creds.get("aws_service_name"),
aws_role_name=aws_creds.get("aws_role_name"),
aws_session_name=aws_creds.get("aws_session_name"),
)
return new_server
@ -786,7 +790,18 @@ class MCPServerManager:
f"Allowed MCP Servers for user api key auth: {allowed_mcp_servers}"
)
combined_servers = set(allowed_mcp_servers)
combined_servers.update(allow_all_server_ids)
# Only skip allow_all_keys servers when the request is inside a toolset
# scope. toolset_mcp_route / dynamic_mcp_route set _mcp_active_toolset_id
# before calling the handler — that ContextVar is the reliable signal.
# Using op.mcp_toolsets==[] would false-positive on DB-default rows where
# Postgres initialises the column to ARRAY[]::TEXT[].
from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: PLC0415
_mcp_active_toolset_id,
)
in_toolset_scope = _mcp_active_toolset_id.get() is not None
if not in_toolset_scope:
combined_servers.update(allow_all_server_ids)
if len(combined_servers) == 0:
verbose_logger.debug(
@ -797,6 +812,132 @@ class MCPServerManager:
verbose_logger.warning(f"Failed to get allowed MCP servers: {str(e)}.")
return allow_all_server_ids
async def resolve_toolset_tool_permissions(
self,
toolset_ids: List[str],
) -> Dict[str, List[str]]:
"""
Resolve a list of toolset IDs into a mcp_tool_permissions dict.
Returns: {server_id: [tool_name, ...]} — the union of all tools across
the given toolsets. Results are cached via ``user_api_key_cache`` (a
Redis-backed ``DualCache`` in production) so that cache entries are
shared across workers and cold-cache DB hits are minimised.
"""
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._experimental.mcp_server.toolset_db import list_mcp_toolsets
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
if not toolset_ids or prisma_client is None:
return {}
cache_key = "toolset_perms:" + ",".join(sorted(toolset_ids))
cached = await user_api_key_cache.async_get_cache(key=cache_key)
if cached is not None:
return cached
try:
toolsets = await list_mcp_toolsets(prisma_client, toolset_ids=toolset_ids)
tool_permissions: Dict[str, List[str]] = {}
for toolset in toolsets:
for tool in toolset.tools:
raw_name = tool["tool_name"]
unprefixed, _ = split_server_prefix_from_name(raw_name)
tool_permissions.setdefault(tool["server_id"], [])
if unprefixed not in tool_permissions[tool["server_id"]]:
tool_permissions[tool["server_id"]].append(unprefixed)
await user_api_key_cache.async_set_cache(
key=cache_key,
value=tool_permissions,
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
)
return tool_permissions
except Exception as e:
verbose_logger.warning(f"Failed to resolve toolset permissions: {str(e)}")
return {}
def invalidate_toolset_cache(self, toolset_id: Optional[str] = None) -> None:
"""Evict cached toolset permission entries.
Called after create/update/delete of a toolset so stale data is not served.
The in-memory layer of ``user_api_key_cache`` is cleared immediately;
Redis entries expire naturally after the configured TTL.
Pass toolset_id to evict only entries containing that ID, or None to clear all.
"""
# Clear the in-memory layer of the shared DualCache for affected keys.
# We can't enumerate Redis keys by pattern, so Redis entries expire via TTL.
try:
from litellm.proxy.proxy_server import user_api_key_cache
in_mem = getattr(user_api_key_cache, "in_memory_cache", None)
if in_mem is None:
return
cache_dict = getattr(in_mem, "cache_dict", {})
if toolset_id is None:
keys_to_remove = [k for k in cache_dict if k.startswith("toolset_")]
else:
# Evict permission-cache entries that reference this toolset ID.
# Also evict ALL name-cache entries (toolset_name:*): we can't map
# toolset_id → toolset_name without a DB call, and the name may have
# changed in an update anyway.
keys_to_remove = [
k
for k in cache_dict
if (k.startswith("toolset_perms:") and toolset_id in k)
or k.startswith("toolset_name:")
]
for k in keys_to_remove:
cache_dict.pop(k, None)
except Exception as e:
verbose_logger.warning(
f"invalidate_toolset_cache: failed to evict in-memory entries: {e}"
)
async def get_toolset_by_name_cached(
self,
prisma_client: Any,
toolset_name: str,
) -> Optional[Any]:
"""Return a toolset by name, cached in ``user_api_key_cache`` (Redis-backed
``DualCache`` in production) to avoid a DB hit on every routed request.
Serialisation note: the cache value is stored as a plain JSON-safe dict via
``model_dump(mode="json")`` so that Redis round-trips correctly in multi-worker
deployments. On a cache hit we reconstruct the ``MCPToolset`` Pydantic object
so callers can always use attribute access (e.g. ``toolset.toolset_id``).
"""
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy.proxy_server import user_api_key_cache
from litellm.types.mcp_server.mcp_toolset import MCPToolset
cache_key = f"toolset_name:{toolset_name}"
cached = await user_api_key_cache.async_get_cache(key=cache_key)
if cached is not None:
# Sentinel value used to cache "not found" so we don't re-query for
# names that don't exist.
if cached == "__not_found__":
return None
# Redis deserialises JSON back as a plain dict — reconstruct the model.
if isinstance(cached, dict):
return MCPToolset(**cached)
return cached
from litellm.proxy._experimental.mcp_server.toolset_db import (
get_mcp_toolset_by_name,
)
toolset = await get_mcp_toolset_by_name(prisma_client, toolset_name)
await user_api_key_cache.async_set_cache(
key=cache_key,
value=(
toolset.model_dump(mode="json")
if toolset is not None
else "__not_found__"
),
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
)
return toolset
def filter_server_ids_by_ip(
self, server_ids: List[str], client_ip: Optional[str]
) -> List[str]:
@ -1011,6 +1152,8 @@ class MCPServerManager:
aws_session_token=server.aws_session_token,
aws_region_name=server.aws_region_name,
aws_service_name=server.aws_service_name,
aws_role_name=server.aws_role_name,
aws_session_name=server.aws_session_name,
)
return MCPClient(
@ -1071,6 +1214,24 @@ class MCPServerManager:
tools = global_mcp_tool_registry.convert_tools_to_mcp_sdk_tool_type(
_tools
)
# OpenAPI tools are stored in the registry with their prefix already
# applied (e.g. "test_petstore-getinventory"). Do NOT pass them
# through _create_prefixed_tools — that would add the prefix a second
# time producing "test_petstore-test_petstore-getinventory".
if not add_prefix:
prefix = get_server_prefix(server)
sep = MCP_TOOL_PREFIX_SEPARATOR
tools = [
(
t.model_copy(
update={"name": t.name[len(prefix) + len(sep) :]}
)
if t.name.startswith(f"{prefix}{sep}")
else t
)
for t in tools
]
return tools
else:
tools = await self._fetch_tools_with_timeout(client, server.name)
@ -1571,6 +1732,8 @@ class MCPServerManager:
),
"aws_region_name": credentials_dict.get("aws_region_name"),
"aws_service_name": credentials_dict.get("aws_service_name"),
"aws_role_name": credentials_dict.get("aws_role_name"),
"aws_session_name": credentials_dict.get("aws_session_name"),
}
def _extract_scopes(self, scopes_value: Any) -> Optional[List[str]]:
@ -2410,7 +2573,6 @@ class MCPServerManager:
async def reload_servers_from_database(self):
"""Re-synchronize the in-memory MCP server registry with the database."""
from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
get_prisma_client_or_throw,
)
@ -2421,9 +2583,19 @@ class MCPServerManager:
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
db_mcp_servers = await get_all_mcp_servers(
prisma_client, approval_status="active"
# Load only "active", legacy "approved", and NULL (no approval workflow) rows.
# Pending/rejected servers are excluded at the DB level so we never load them.
from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable
raw_rows = await prisma_client.db.litellm_mcpservertable.find_many(
where={
"OR": [
{"approval_status": None},
{"approval_status": {"in": ["active", "approved"]}},
]
}
)
db_mcp_servers = [LiteLLM_MCPServerTable(**r.model_dump()) for r in raw_rows]
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
previous_registry = self.registry

203
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -1,6 +1,7 @@
"""
LiteLLM MCP Server Routes
"""
# pyright: reportInvalidTypeForm=false, reportArgumentType=false, reportOptionalCall=false
import asyncio
@ -36,6 +37,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
get_request_base_url,
)
from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_active_toolset_id
from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
from litellm.proxy._experimental.mcp_server.utils import (
LITELLM_MCP_SERVER_DESCRIPTION,
@ -170,6 +172,34 @@ if MCP_AVAILABLE:
mcp_info: Optional[MCPInfo] = None
model_config = ConfigDict(arbitrary_types_allowed=True)
def _normalize_resource_contents(contents: list) -> List[ReadResourceContents]:
"""Normalize ResourceContents to ReadResourceContents, preserving meta (MCP 1.26.0+)."""
normalized: List[ReadResourceContents] = []
for content in contents:
meta = getattr(content, "meta", None)
if meta is None and hasattr(content, "model_dump"):
d = content.model_dump()
meta = d.get("meta")
if meta is None:
meta = d.get("_meta")
if isinstance(content, TextResourceContents):
normalized.append(
ReadResourceContents(
content=content.text,
mime_type=content.mimeType,
meta=meta,
)
)
elif isinstance(content, BlobResourceContents):
normalized.append(
ReadResourceContents(
content=content.blob,
mime_type=content.mimeType,
meta=meta,
)
)
return normalized
########################################################
############ Initialize the MCP Server #################
########################################################
@ -630,26 +660,7 @@ if MCP_AVAILABLE:
raw_headers=raw_headers,
)
normalized_contents: List[ReadResourceContents] = []
for content in read_resource_result.contents:
if isinstance(content, TextResourceContents):
text_content: TextResourceContents = content
normalized_contents.append(
ReadResourceContents(
content=text_content.text,
mime_type=text_content.mimeType,
)
)
elif isinstance(content, BlobResourceContents):
blob_content: BlobResourceContents = content
normalized_contents.append(
ReadResourceContents(
content=blob_content.blob,
mime_type=None,
)
)
return normalized_contents
return _normalize_resource_contents(read_resource_result.contents)
########################################################
############ End of MCP Server Routes ##################
@ -1455,6 +1466,49 @@ if MCP_AVAILABLE:
return filtered_tools
async def _merge_toolset_permissions(
user_api_key_auth: Optional[UserAPIKeyAuth],
) -> Optional[UserAPIKeyAuth]:
"""
Resolve mcp_toolsets on the key's object_permission into tool-level permissions
and merge them (union) into object_permission.mcp_tool_permissions.
Returns the (possibly mutated copy of) user_api_key_auth.
"""
if user_api_key_auth is None:
return None
op = user_api_key_auth.object_permission
if op is None:
return user_api_key_auth
toolset_ids = getattr(op, "mcp_toolsets", None) or []
if not toolset_ids:
return user_api_key_auth
toolset_perms = (
await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=toolset_ids
)
)
if not toolset_perms:
return user_api_key_auth
# Merge toolset_perms into existing mcp_tool_permissions (union)
existing = dict(op.mcp_tool_permissions or {})
for server_id, tool_names in toolset_perms.items():
existing_tools = existing.get(server_id, [])
merged = list(set(existing_tools) | set(tool_names))
existing[server_id] = merged
# Build updated object_permission with merged tool permissions and server IDs.
# Union the toolset's server IDs into mcp_servers so downstream server-level
# filtering doesn't silently drop servers that the toolset references but that
# aren't already in the key's explicit mcp_servers list.
merged_servers = list(set(op.mcp_servers or []) | set(existing.keys()))
updated_op = op.model_copy(
update={"mcp_servers": merged_servers, "mcp_tool_permissions": existing}
)
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
async def _list_mcp_tools(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
@ -1479,6 +1533,11 @@ if MCP_AVAILABLE:
"""
if not MCP_AVAILABLE:
return []
# Resolve toolset permissions and merge into the key's object_permission
# so that the existing filter_tools_by_key_team_permissions logic picks them up.
user_api_key_auth = await _merge_toolset_permissions(user_api_key_auth)
# Get tools from managed MCP servers with error handling
managed_tools = []
try:
@ -1822,9 +1881,9 @@ if MCP_AVAILABLE:
"litellm_logging_obj", None
)
if litellm_logging_obj:
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = (
standard_logging_mcp_tool_call
)
litellm_logging_obj.model = f"MCP: {name}"
# Resolve the MCP server early so BYOK checks and credential injection
# apply to ALL dispatch paths (local tool registry AND managed MCP server).
@ -1836,9 +1895,9 @@ if MCP_AVAILABLE:
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
if litellm_logging_obj:
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = (
standard_logging_mcp_tool_call
)
# BYOK: retrieve the stored per-user credential. A single DB call
# both checks existence and fetches the value, avoiding a double query.
@ -2358,6 +2417,63 @@ if MCP_AVAILABLE:
]
return False
async def _apply_toolset_scope(
user_api_key_auth: UserAPIKeyAuth,
toolset_id: str,
) -> UserAPIKeyAuth:
"""
Restrict a key's MCP permissions to a single toolset.
When a request arrives via /toolset/{name}/mcp we override the key's
object_permission so that only the toolset's tools are visible.
Raises HTTPException(403) if the key has an explicit toolset grant list
that does not include toolset_id (i.e. mcp_toolsets is set but empty,
or set to a list that omits this toolset). Admin keys always pass.
"""
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
# Access control: non-admin keys must have this toolset in their grant list.
# Use _user_has_admin_view so that PROXY_ADMIN_VIEW_ONLY is also treated as admin.
is_admin = _user_has_admin_view(user_api_key_auth)
if not is_admin:
op = user_api_key_auth.object_permission
granted = getattr(op, "mcp_toolsets", None) if op else None
# granted=None → key has no explicit toolset grants → deny (same semantics as
# fetch_mcp_toolsets which returns [] for non-admin keys with no grants configured).
# granted=[] or list without toolset_id → also deny.
if granted is None or toolset_id not in granted:
raise HTTPException(
status_code=403,
detail=f"API key does not have access to toolset '{toolset_id}'.",
)
tool_permissions = (
await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=[toolset_id]
)
)
server_ids = list(tool_permissions.keys())
existing_op = user_api_key_auth.object_permission
if existing_op is not None:
updated_op = existing_op.model_copy(
update={
"mcp_servers": server_ids,
"mcp_tool_permissions": tool_permissions,
"mcp_toolsets": [],
# mcp_access_groups is preserved: a key's access-group grants
# remain valid even when the request is scoped to a single toolset.
}
)
else:
updated_op = LiteLLM_ObjectPermissionTable(
object_permission_id="toolset-scope",
mcp_servers=server_ids,
mcp_tool_permissions=tool_permissions,
)
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
@ -2402,6 +2518,21 @@ if MCP_AVAILABLE:
headers={"www-authenticate": authorization_uri},
)
# Strip any client-supplied x-mcp-toolset-id to prevent forgery.
scope["headers"] = [
(k, v)
for k, v in scope.get("headers", [])
if k.lower() != b"x-mcp-toolset-id"
]
# Apply toolset scope if set server-side via ContextVar (set by
# /toolset/{name}/mcp and /{name}/mcp route handlers in proxy_server.py).
active_toolset_id = _mcp_active_toolset_id.get()
if active_toolset_id and user_api_key_auth is not None:
user_api_key_auth = await _apply_toolset_scope(
user_api_key_auth, active_toolset_id
)
# Inject masked debug headers when client sends x-litellm-mcp-debug: true
_debug_headers = MCPDebug.maybe_build_debug_headers(
raw_headers=raw_headers,
@ -2580,17 +2711,15 @@ if MCP_AVAILABLE:
)
auth_context_var.set(auth_user)
def get_auth_context() -> (
Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, Dict[str, str]]],
Optional[Dict[str, str]],
Optional[Dict[str, str]],
Optional[str],
]
):
def get_auth_context() -> Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, Dict[str, str]]],
Optional[Dict[str, str]],
Optional[Dict[str, str]],
Optional[str],
]:
"""
Get the UserAPIKeyAuth from the auth context variable.

117
litellm/proxy/_experimental/mcp_server/toolset_db.py Normal file
View file

@ -0,0 +1,117 @@
import json
from typing import List, Optional
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.proxy.utils import PrismaClient
from litellm.types.mcp_server.mcp_toolset import (
MCPToolset,
NewMCPToolsetRequest,
UpdateMCPToolsetRequest,
)
def _toolset_from_row(row) -> MCPToolset:
data = row.model_dump()
tools = data.get("tools") or []
if isinstance(tools, str):
tools = json.loads(tools)
data["tools"] = tools
return MCPToolset(**data)
async def create_mcp_toolset(
prisma_client: PrismaClient,
data: NewMCPToolsetRequest,
touched_by: str,
) -> MCPToolset:
data_dict = data.model_dump(exclude_none=True)
data_dict["toolset_id"] = str(uuid.uuid4())
data_dict["tools"] = json.dumps(data_dict.get("tools", []))
data_dict["created_by"] = touched_by
data_dict["updated_by"] = touched_by
row = await prisma_client.db.litellm_mcptoolsettable.create(data=data_dict)
return _toolset_from_row(row)
async def get_mcp_toolset(
prisma_client: PrismaClient,
toolset_id: str,
) -> Optional[MCPToolset]:
row = await prisma_client.db.litellm_mcptoolsettable.find_unique(
where={"toolset_id": toolset_id}
)
if row is None:
return None
return _toolset_from_row(row)
async def list_mcp_toolsets(
prisma_client: PrismaClient,
toolset_ids: Optional[List[str]] = None,
) -> List[MCPToolset]:
try:
where = {}
if toolset_ids is not None:
where = {"toolset_id": {"in": toolset_ids}}
rows = await prisma_client.db.litellm_mcptoolsettable.find_many(where=where)
return [_toolset_from_row(r) for r in rows]
except Exception as e:
verbose_proxy_logger.warning(
"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {}".format(
str(e)
)
)
return []
async def get_mcp_toolset_by_name(
prisma_client: PrismaClient,
toolset_name: str,
) -> Optional[MCPToolset]:
row = await prisma_client.db.litellm_mcptoolsettable.find_first(
where={"toolset_name": toolset_name}
)
if row is None:
return None
return _toolset_from_row(row)
async def update_mcp_toolset(
prisma_client: PrismaClient,
data: UpdateMCPToolsetRequest,
touched_by: str,
) -> Optional[MCPToolset]:
data_dict = data.model_dump(exclude_none=True, exclude={"toolset_id"})
if "tools" in data_dict:
data_dict["tools"] = json.dumps(data_dict["tools"])
data_dict["updated_by"] = touched_by
try:
row = await prisma_client.db.litellm_mcptoolsettable.update(
where={"toolset_id": data.toolset_id},
data=data_dict,
)
except Exception as e:
from prisma.errors import RecordNotFoundError
if isinstance(e, RecordNotFoundError):
return None
raise
return _toolset_from_row(row)
async def delete_mcp_toolset(
prisma_client: PrismaClient,
toolset_id: str,
) -> Optional[MCPToolset]:
try:
row = await prisma_client.db.litellm_mcptoolsettable.delete(
where={"toolset_id": toolset_id}
)
except Exception as e:
from prisma.errors import RecordNotFoundError
if isinstance(e, RecordNotFoundError):
return None
raise
return _toolset_from_row(row)

2
litellm/proxy/_experimental/out/404/index.html → litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

34
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,28 +1,28 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js"],"default"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/9a17d35f872a6c38.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/ee9e514b2c2694f7.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a7dc5e0c9d37afe3.js","/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/bf01d87225e5be70.js","/litellm-asset-prefix/_next/static/chunks/b3b05b76472ce110.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/4fc2d71e511309ab.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/360f35fe2e0a4945.js","/litellm-asset-prefix/_next/static/chunks/0d219667baa010f5.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","/litellm-asset-prefix/_next/static/chunks/2e768c2b1dfc8cd5.js"],"default"]
18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
19:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9a17d35f872a6c38.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/ee9e514b2c2694f7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/a7dc5e0c9d37afe3.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/bf01d87225e5be70.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/b3b05b76472ce110.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","async":true}]
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/4fc2d71e511309ab.js","async":true}]
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","async":true}]
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/360f35fe2e0a4945.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/0d219667baa010f5.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","async":true}]
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/2e768c2b1dfc8cd5.js","async":true}]
17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
1a:null

74
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

8
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,8 +1,8 @@
1:"$Sreact.fragment"
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,5 +1,5 @@
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

0
litellm/proxy/_experimental/out/_next/static/Hp-LQxDEAEt-JSJFExm-i/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/-9iBbUN_ohnDf0d-Ux3Ju/_buildManifest.js
View file

Some files were not shown because too many files have changed in this diff Show more