scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-27 01:22:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

Merge branch 'litellm_internal_staging' into litellm_prisma-reconnect-circuit-breaker

Browse source
This commit is contained in:
harish-berri 2026-05-16 16:00:29 -07:00 • committed by GitHub
commit 9c99d33b86
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
845 changed files with 31736 additions and 6971 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
.circleci/config.yml
View file

@ -228,7 +228,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -293,7 +293,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -409,14 +409,25 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml auth_ui_unit_tests_coverage.xml
mv .coverage auth_ui_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
@ -493,13 +504,24 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml router_unit_tests_coverage.xml
mv .coverage router_unit_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- router_unit_tests_coverage.xml
- router_unit_tests_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- *python312_image
@ -603,7 +625,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -646,7 +668,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -688,7 +710,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -732,7 +754,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -814,7 +836,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -856,7 +878,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -930,7 +952,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -972,7 +994,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1015,7 +1037,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1090,7 +1112,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1133,7 +1155,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1184,7 +1206,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -2280,10 +2302,11 @@ jobs:
- run:
name: Combine Coverage
command: |
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
flags: circleci
ui_build:
docker:
@ -2669,6 +2692,8 @@ workflows:
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- litellm_router_unit_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image

94
.github/actions/helm-oci-chart-releaser/action.yml vendored
View file

@ -1,94 +0,0 @@
name: Helm OCI Chart Releaser
description: Push Helm charts to OCI-based (Docker) registries
author: sergeyshaykhullin
branding:
color: yellow
icon: upload-cloud
inputs:
name:
required: true
description: Chart name
repository:
required: true
description: Chart repository name
tag:
required: true
description: Chart version
app_version:
required: true
description: App version
path:
required: false
description: Chart path (Default 'charts/{name}')
registry:
required: true
description: OCI registry
registry_username:
required: true
description: OCI registry username
registry_password:
required: true
description: OCI registry password
update_dependencies:
required: false
default: 'false'
description: Update chart dependencies before packaging (Default 'false')
outputs:
image:
value: ${{ steps.output.outputs.image }}
description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
runs:
using: composite
steps:
- name: Helm | Setup
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:
version: v3.20.0
- name: Helm | Login
shell: bash
env:
REGISTRY_PASSWORD: ${{ inputs.registry_password }}
REGISTRY_USERNAME: ${{ inputs.registry_username }}
REGISTRY: ${{ inputs.registry }}
run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
env:
CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
run: helm dependency update "$CHART_PATH"
- name: Helm | Package
shell: bash
env:
CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
TAG: ${{ inputs.tag }}
APP_VERSION: ${{ inputs.app_version }}
run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
- name: Helm | Push
shell: bash
env:
NAME: ${{ inputs.name }}
TAG: ${{ inputs.tag }}
REGISTRY: ${{ inputs.registry }}
REPOSITORY: ${{ inputs.repository }}
run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
- name: Helm | Logout
shell: bash
env:
REGISTRY: ${{ inputs.registry }}
run: helm registry logout "$REGISTRY"
- name: Helm | Output
id: output
shell: bash
env:
REGISTRY: ${{ inputs.registry }}
REPOSITORY: ${{ inputs.repository }}
NAME: ${{ inputs.name }}
TAG: ${{ inputs.tag }}
run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT

35
.github/workflows/README.md vendored
View file

@ -1,35 +0,0 @@
# Simple PyPI Publishing
A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
## How to Use
1. Go to the **Actions** tab in the GitHub repository
2. Select **Simple PyPI Publish** from the workflow list
3. Click **Run workflow**
4. Enter the version to publish (e.g., `1.74.10`)
## What the Workflow Does
1. **Updates** the version in `pyproject.toml`
2. **Copies** the model prices backup file
3. **Builds** the Python package
4. **Publishes** to PyPI
## Prerequisites
Make sure the following secret is configured in the repository:
- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
## Example Usage
- Version: `1.74.11` → Publishes as v1.74.11
- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
## Features
- ✅ Manual trigger with version input
- ✅ Automatic version updates in `pyproject.toml`
- ✅ Repository safety check (only runs on official repo)
- ✅ Clean package building and publishing
- ✅ Success confirmation with PyPI package link

3
.github/workflows/_test-unit-base.yml vendored
View file

@ -91,7 +91,7 @@ jobs:
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
@ -132,4 +132,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

5
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -132,7 +132,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -144,7 +144,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
@ -186,4 +186,5 @@ jobs:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

92
.github/workflows/llm-translation-testing.yml vendored
View file

@ -1,92 +0,0 @@
name: LLM Translation Tests
on:
workflow_dispatch:
inputs:
release_candidate_tag:
description: "Release candidate tag/version"
required: true
type: string
push:
tags:
- "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
permissions:
contents: read
jobs:
run-llm-translation-tests:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Restore uv dependencies cache
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen
- name: Create test results directory
run: mkdir -p test-results
- name: Run LLM Translation Tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
COMMIT_SHA: ${{ github.sha }}
run: |
python .github/workflows/run_llm_translation_tests.py \
--tag "$RC_TAG" \
--commit "$COMMIT_SHA" \
|| true # Continue even if tests fail
- name: Display test summary
if: always()
run: |
if [ -f "test-results/llm_translation_report.md" ]; then
echo "Test report generated successfully!"
echo "Artifact will contain:"
echo "- test-results/junit.xml (JUnit XML results)"
echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
else
echo "Warning: Test report was not generated"
fi
- name: Upload test artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
path: test-results/
retention-days: 30

131
.github/workflows/mutation-test.yml vendored Normal file
View file

@ -0,0 +1,131 @@
name: "Mutation Test (manual)"
# Manually-triggered mutation testing. Runs mutmut against the scope
# configured in [tool.mutmut] in pyproject.toml (currently the
# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly
# weekly — clicked from the Actions tab when someone wants a fresh report.
#
# Uploads a structured `mutation-report.md` (Meta ACH-style: original +
# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the
# existing tests + a task instruction) as a workflow artifact. Failures
# do not block anything because nothing depends on this workflow.
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: mutation-test-${{ github.ref }}
cancel-in-progress: true
jobs:
mutation:
name: Run mutmut
runs-on: ubuntu-latest
# Whole-folder mutation against ~15 files / ~7.5k LOC can take hours.
# 350 minutes is just under the GitHub-hosted job cap of 360 minutes.
timeout-minutes: 350
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
# mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
# mutation trampolines. uv installs the project as editable by default,
# which puts the original source dir on sys.path via a .pth file and
# shadows the sandbox copy — so tests would never exercise the mutated
# code. Reinstalling non-editable removes the .pth shadow.
- name: Reinstall litellm non-editable (so mutants/ is not shadowed)
run: |
uv pip uninstall litellm
uv pip install . --no-deps
# pytest-retry's pytest_configure hook crashes with
# `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
# via mutmut's in-process pytest.main() call. The entry-point name
# doesn't normalize cleanly with `-p no:<name>`, so just remove the
# package outright. Reruns are wrong for mutation testing anyway —
# rerunning a "failed" mutant test would mask which mutants are killed.
- name: Remove pytest plugins that conflict with mutmut
run: |
uv pip uninstall pytest-retry || true
- name: Run mutmut
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
run: |
set -o pipefail
mkdir -p mutants
uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log
# Generate the structured report. The script embeds the enclosing
# function source for each survivor (via Python AST) and includes the
# existing test files, so an LLM agent has enough context to write
# killing tests without further file lookups. Modeled on Meta's ACH
# prompt template (arXiv 2501.12862).
- name: Generate detailed mutation report
if: always()
run: |
set +e
uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
uv run --no-sync python scripts/mutation_report.py
# The full report can be very long for big test files; the run-page
# summary cuts off at 1 MB. Append the head of the report (summary
# + survivor list) and link out to the artifact for the full body.
{
head -c 900000 mutation-report.md
echo ""
echo ""
echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload mutmut artifacts
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
path: |
mutation-report.md
mutmut-results.txt
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

153
.github/workflows/publish_to_pypi.yml vendored
View file

@ -1,153 +0,0 @@
name: Publish to PyPI
on:
workflow_dispatch:
jobs:
preflight-checks:
name: Preflight Checks
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
# No environment — read-only checks, no approval needed
outputs:
needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
version: ${{ steps.check-litellm.outputs.version }}
steps:
- name: Checkout repo
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Check litellm version on PyPI
id: check-litellm
run: |
VERSION=$(python - <<'PY'
import tomllib
with open("pyproject.toml", "rb") as f:
print(tomllib.load(f)["project"]["version"])
PY
)
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Checking if litellm $VERSION exists on PyPI..."
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
if [ "$HTTP_STATUS" = "200" ]; then
echo "litellm $VERSION already exists on PyPI. Skipping publish."
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
else
echo "litellm $VERSION not found on PyPI. Publish needed."
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
fi
- name: Sanity check proxy-extras version
run: |
# Read pinned version from project optional dependencies
PYPROJECT_VERSION=$(python3 - <<'PY'
import sys
import tomllib
with open("pyproject.toml", "rb") as f:
proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"]
version = None
for requirement in proxy_requirements:
normalized = requirement.split(";", 1)[0].strip()
if not normalized.startswith("litellm-proxy-extras"):
continue
parts = normalized.split("==", 1)
if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras":
candidate = parts[1].strip()
if candidate:
version = candidate
break
if version is None:
print(
"::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy",
file=sys.stderr,
)
sys.exit(1)
print(version)
PY
)
echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
# Check that the pinned version exists on PyPI
echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..."
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json")
if [ "$HTTP_STATUS" != "200" ]; then
echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm."
exit 1
fi
echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed."
publish-litellm:
name: Publish litellm to PyPI
needs: preflight-checks
if: needs.preflight-checks.outputs.needs_publish == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
id-token: write
contents: read
environment: pypi-publish
steps:
- name: Checkout repo
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Copy model prices backup
run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- name: Build package
run: |
rm -rf build dist
uv build
- name: Verify build artifacts
env:
EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
run: |
echo "Contents of dist/:"
ls -la dist/
# Ensure we have both sdist and wheel
ls dist/*.tar.gz
ls dist/*.whl
# Verify built version matches expected
ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
ls dist/
exit 1
}
- name: Validate package metadata
run: |
uv tool run --from 'twine==6.2.0' twine check dist/*
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

28
.github/workflows/read_pyproject_version.yml vendored
View file

@ -1,28 +0,0 @@
name: Read Version from pyproject.toml
on:
push:
branches:
- main # Change this to the default branch of your repository
permissions:
contents: read
jobs:
read-version:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Read version from pyproject.toml
id: read-version
run: |
version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Display version
run: echo "Current version is $LITELLM_VERSION"

27
.github/workflows/results_stats.csv vendored
View file

@ -1,27 +0,0 @@
Date,"Ben
Ashley",Tom Brooks,Jimmy Cooney,"Sue
Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
Total,0,1,1,1,1,1,0,1
1 Date Ben Ashley Tom Brooks Jimmy Cooney Sue Daniels Berlinda Fong Terry Jones Angelina Little Linda Smith
2 10/1 FALSE TRUE TRUE TRUE TRUE TRUE FALSE TRUE
3 10/2 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
4 10/3 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
5 10/4 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
6 10/5 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
7 10/6 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
8 10/7 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
9 10/8 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
10 10/9 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
11 10/10 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
12 10/11 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
13 10/12 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
14 10/13 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
15 10/14 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
16 10/15 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
17 10/16 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
18 10/17 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
19 10/18 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
20 10/19 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
21 10/20 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
22 10/21 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
23 10/22 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
24 10/23 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
25 Total 0 1 1 1 1 1 0 1

229
.github/workflows/run_observatory_tests.yml vendored
View file

@ -1,229 +0,0 @@
name: Run Observatory Tests
on:
workflow_dispatch:
inputs:
tag:
description: "Docker image tag to test (e.g. v1.61.0.rc1)"
required: true
type: string
commit_hash:
description: "Commit hash (defaults to HEAD of current branch)"
required: false
type: string
workflow_call:
inputs:
tag:
description: "Docker image tag to test"
required: true
type: string
commit_hash:
description: "Commit hash of the release"
required: true
type: string
permissions:
contents: read
env:
LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
jobs:
observatory-tests:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Validate tag input
env:
TAG: ${{ inputs.tag }}
run: |
if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
exit 1
fi
- name: Start LiteLLM container
env:
TAG: ${{ inputs.tag }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
WORKSPACE: ${{ github.workspace }}
run: |
docker run -d \
--name litellm-rc \
-p 4000:4000 \
-v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-e AZURE_API_KEY="${AZURE_API_KEY}" \
-e AZURE_API_BASE="${AZURE_API_BASE}" \
"litellm/litellm:${TAG}" \
--config /app/config.yaml --port 4000
- name: Wait for LiteLLM health check
run: |
echo "Waiting for LiteLLM to be ready..."
for i in $(seq 1 30); do
if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
echo "LiteLLM is healthy"
exit 0
fi
echo "Attempt $i/30 - not ready yet, waiting 10s..."
sleep 10
done
echo "LiteLLM failed to start within 5 minutes"
docker logs litellm-rc
exit 1
- name: Start cloudflared tunnel
run: |
# Install cloudflared (pinned version + checksum)
curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c -
chmod +x /usr/local/bin/cloudflared
# Start a quick tunnel (no account needed) and capture the URL
cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
CLOUDFLARED_PID=$!
echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
# Wait for tunnel URL to appear in logs
echo "Waiting for tunnel URL..."
for i in $(seq 1 30); do
TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
if [ -n "$TUNNEL_URL" ]; then
echo "Tunnel URL: $TUNNEL_URL"
echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
exit 0
fi
sleep 2
done
echo "Failed to get tunnel URL"
cat /tmp/cloudflared.log
exit 1
- name: Verify tunnel connectivity
run: |
echo "Testing tunnel at ${TUNNEL_URL}..."
# Quick tunnels need time for DNS propagation; retry to avoid
# transient NXDOMAIN (curl exit code 6) on first attempt.
for i in $(seq 1 10); do
if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
echo "Tunnel is working (attempt $i)"
exit 0
fi
echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
sleep 5
done
echo "Tunnel failed to become reachable after 50s"
cat /tmp/cloudflared.log
exit 1
- name: Trigger observatory test run
id: trigger
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
run: |
PAYLOAD=$(jq -n \
--arg url "${TUNNEL_URL}" \
--arg key "${LITELLM_MASTER_KEY}" \
'{
deployment_url: $url,
api_key: $key,
test_suite: "TestOAIAzureRelease",
models: ["gpt-4o-mini", "gpt-4o"]
}')
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-H "Content-Type: application/json" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-d "$PAYLOAD")
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)
echo "Response ($HTTP_CODE): $BODY"
if [ "$HTTP_CODE" -ge 400 ]; then
echo "Failed to trigger test run"
exit 1
fi
# Extract request_id for polling this specific run
REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
echo "Failed to extract request_id from response"
exit 1
fi
echo "Request ID: $REQUEST_ID"
echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
- name: Poll for test completion
id: poll
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
run: |
TIMEOUT=900 # 15 minutes
INTERVAL=30
ELAPSED=0
while [ $ELAPSED -lt $TIMEOUT ]; do
STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
echo "Test finished with status: $RUN_STATUS"
echo "$STATUS" > /tmp/observatory_result.json
exit 0
fi
sleep $INTERVAL
ELAPSED=$((ELAPSED + INTERVAL))
done
echo "Timed out waiting for test to complete after ${TIMEOUT}s"
exit 1
- name: Verify test results
run: |
RESULT=$(cat /tmp/observatory_result.json)
echo "Full result: $RESULT"
STATUS=$(echo "$RESULT" | jq -r '.status')
TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
ERROR=$(echo "$RESULT" | jq -r '.error // empty')
echo "Status: $STATUS"
echo "Test passed: $TEST_PASSED"
echo "Failure rate: $FAILURE_RATE"
if [ -n "$ERROR" ]; then
echo "Error: $ERROR"
fi
if [ "$STATUS" = "failed" ]; then
echo "Test run failed"
exit 1
fi
if [ "$TEST_PASSED" != "true" ]; then
echo "Tests did not pass (failure rate: $FAILURE_RATE)"
exit 1
fi
echo "All tests passed!"
- name: Print LiteLLM logs on failure
if: failure()
run: |
docker logs litellm-rc 2>/dev/null || true
cat /tmp/cloudflared.log 2>/dev/null || true
- name: Cleanup
if: always()
run: |
kill "$CLOUDFLARED_PID" 2>/dev/null || true
docker rm -f litellm-rc 2>/dev/null || true

48
.github/workflows/scan_duplicate_issues.yml vendored
View file

@ -1,48 +0,0 @@
name: Scan Duplicate Issues (One-Time)
on:
workflow_dispatch:
inputs:
threshold:
description: "Similarity threshold (0-1)"
required: false
default: "0.85"
close:
description: "Actually close duplicates (false = dry run)"
required: false
type: boolean
default: false
jobs:
scan:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Scan for duplicate issues
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_THRESHOLD: ${{ inputs.threshold }}
INPUT_CLOSE: ${{ inputs.close }}
run: |
CLOSE_FLAG=""
if [ "$INPUT_CLOSE" = "true" ]; then
CLOSE_FLAG="--close"
fi
python3 .github/scripts/close_duplicate_issues.py \
--scan \
--repo ${{ github.repository }} \
--threshold "$INPUT_THRESHOLD" \
$CLOSE_FLAG

45
.github/workflows/test-litellm.yml vendored
View file

@ -1,45 +0,0 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
# the same tests in parallel across 10 jobs for faster CI times.
# Kept for manual debugging only.
on:
workflow_dispatch: # Manual trigger only
# pull_request:
# branches: [ main ]
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Install dependencies
run: |
uv lock --check
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Run tests
run: |
uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

2
.github/workflows/test-mcp.yml vendored
View file

@ -43,4 +43,4 @@ jobs:
- name: Run MCP tests
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

38
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -1,38 +0,0 @@
name: "Unit Tests: Caching (Redis)"
# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
# This prevents external PRs from accessing Redis credentials.
on:
push:
branches: [main, "litellm_*"]
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
caching-redis:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
# Redis-only tests that do NOT require provider API keys.
# Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
# test_router_caching.py) are in Phase 3 integration workflows.
test-path: >-
tests/local_testing/test_dual_cache.py
tests/local_testing/test_redis_batch_optimizations.py
tests/local_testing/test_router_utils.py
workers: 2
reruns: 2
timeout-minutes: 20
enable-redis: true
enable-postgres: false
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

2
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -100,6 +100,7 @@ jobs:
test-path: >-
tests/proxy_unit_tests/test_auth_checks.py
tests/proxy_unit_tests/test_user_api_key_auth.py
tests/proxy_unit_tests/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
@ -142,6 +143,7 @@ jobs:
tests/proxy_unit_tests/test_proxy_pass_user_config.py
tests/proxy_unit_tests/test_proxy_token_counter.py
tests/proxy_unit_tests/test_request_size_limit_middleware.py
tests/proxy_unit_tests/test_multipart_bypass_repro.py
workers: 4
dist: loadscope
timeout: 15

54
.github/workflows/update_release.py vendored
View file

@ -1,54 +0,0 @@
import os
import requests
from datetime import datetime
# GitHub API endpoints
GITHUB_API_URL = "https://api.github.com"
REPO_OWNER = "BerriAI"
REPO_NAME = "litellm"
# GitHub personal access token (required for uploading release assets)
GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
# Headers for GitHub API requests
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
"X-GitHub-Api-Version": "2022-11-28",
}
# Get the latest release
releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
response = requests.get(releases_url, headers=headers)
latest_release = response.json()
print("Latest release:", latest_release)
# Upload an asset to the latest release
upload_url = latest_release["upload_url"].split("{?")[0]
asset_name = "results_stats.csv"
asset_path = os.path.join(os.getcwd(), asset_name)
print("upload_url:", upload_url)
with open(asset_path, "rb") as asset_file:
asset_data = asset_file.read()
upload_payload = {
"name": asset_name,
"label": "Load test results",
"created_at": datetime.utcnow().isoformat() + "Z",
}
upload_headers = headers.copy()
upload_headers["Content-Type"] = "application/octet-stream"
upload_response = requests.post(
upload_url,
headers=upload_headers,
data=asset_data,
params=upload_payload,
)
if upload_response.status_code == 201:
print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
else:
print(f"Failed to upload asset. Response: {upload_response.text}")

1
CLAUDE.md
View file

@ -117,6 +117,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
### MCP OAuth / OpenAPI Transport Mapping
- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.

83
backend/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the backend as that user
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4001/tcp
ENTRYPOINT ["uvicorn", "backend.main:app"]
CMD ["--host", "0.0.0.0", "--port", "4001"]

51
backend/main.py Normal file
View file

@ -0,0 +1,51 @@
"""UI backend entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the management/admin surface used by the dashboard. Purely
additive — no existing module is modified.
Run with:
uvicorn backend.main:app --host 0.0.0.0 --port 4001
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# See gateway/main.py for why we assemble DATABASE_URL(s) here before
# importing proxy_server.
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
def _is_backend_route(route) -> bool:
"""Keep the route on the backend if its path is in the management surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Static UI mounts are served by the dedicated UI container, not here.
return False
if path in BACKEND_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)
# See gateway/main.py for why the trim runs inside the lifespan instead of at
# module scope.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _backend_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
yield
app.router.lifespan_context = _backend_lifespan

0
backend/routes/__init__.py Normal file
View file

135
backend/routes/allowlist.py Normal file
View file

@ -0,0 +1,135 @@
"""Path allowlist for the UI backend (control plane) component.
The backend exposes management/admin endpoints consumed by the UI: keys, users,
teams, orgs, customers, budgets, tags, workflows, model management, spend &
analytics, settings (router/cache/cost-tracking/fallbacks), SSO/onboarding,
audit logs, debug, enterprise admin, and UI bootstrap helpers (logo, favicon,
.well-known config).
Anything LLM data-plane is dropped — those run on the gateway component.
"""
BACKEND_PATH_PREFIXES: tuple[str, ...] = (
# Identity / access
"/key/",
"/v2/key/",
"/user/",
"/v2/user/",
"/team/",
"/v2/team/",
"/organization/",
"/customer/",
"/end_user/",
"/sso/",
"/login",
"/v2/login",
"/v3/login",
"/logout",
"/token",
"/onboarding/",
"/audit",
"/oauth/",
"/invitation/",
"/jwt/",
# Models & routing config
"/model/",
"/v1/model/info",
"/v2/model/",
"/model_group",
"/model_access_group/",
"/model_hub/",
"/v1/access_group",
"/access_group/",
"/router/",
"/router_settings",
"/adaptive_router/",
"/fallback",
"/fallbacks",
"/cache_settings",
"/cost_tracking",
"/cost/",
"/credentials",
"/credential",
"/provider/budgets",
# Tools / agents (registry & policy admin)
"/v1/tool/",
"/v1/agents",
# Guardrails admin
"/v2/guardrails/",
# MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
"/v1/mcp/",
"/test/",
"/{mcp_server_name}/",
# Budgets / tags / workflows / memory mgmt
"/budget/",
"/tag/",
"/workflow/",
"/v1/workflows/",
"/project/",
"/memory/",
"/mcp/",
# Spend / analytics
"/spend/",
"/analytics/",
"/global/",
"/user_agent",
"/usage/",
"/daily/",
# CloudZero cost-export admin (init / settings / export / dry-run / delete)
"/cloudzero/",
# Caching admin
"/cache/",
"/caching/",
# Callbacks / hooks
"/active/callbacks",
"/callbacks",
"/team_callback",
# Alerting / email / IP allowlist
"/alerting/",
"/email/",
"/add/allowed_ip",
"/delete/allowed_ip",
"/get/",
# Enterprise admin
"/enterprise/",
# Debug / config / profiling
"/debug/",
"/config/",
"/memory-usage-in-mem-cache",
"/otel-spans",
"/lazy/",
"/in_product_nudges",
# Admin reload / schedule
"/reload/",
"/schedule/",
"/settings",
"/update/",
"/upload/",
# Dev / admin utilities
"/utils/",
# UI bootstrap helpers (assets the dashboard fetches)
"/get_logo_url",
"/get_image",
"/get_favicon",
"/.well-known/",
"/litellm/.well-known/",
"/ui_discovery/",
"/ui-config",
"/sso_settings",
"/public/",
"/robots.txt",
# Health (k8s probes)
"/health",
)
BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/fallback/login",
}
)

10
codecov.yaml
View file

@ -3,6 +3,16 @@ codecov:
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
# a re-upload of a flag replace its prior session instead of accumulating a
# conflicting one, and lets a commit reuse a flag from its parent when that flag
# was not re-uploaded. Required because the same commit can receive the
# push-triggered workflows more than once (re-runs / branches cut at the same
# SHA); flagless overlapping sessions made Codecov drop the largest files.
flag_management:
default_rules:
carryforward: true
component_management:
individual_components:
- component_id: "Router"

36
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -300,6 +300,42 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
# output/error file IDs to managed base64 IDs before the DB write here.
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_hook is not None:
from litellm.proxy._types import UserAPIKeyAuth
_minimal_auth = UserAPIKeyAuth(
user_id=job.created_by or "default-user-id",
team_id=getattr(job, "team_id", None),
)
for _file_attr in ["output_file_id", "error_file_id"]:
_raw_file_id = getattr(response, _file_attr, None)
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(
output_file_id=_raw_file_id,
model_id=model_id,
model_name=str(model_name) if model_name else deployment_info.model_name or None,
)
await managed_files_hook.store_unified_file_id(
file_id=_unified_file_id,
file_object=None,
litellm_parent_otel_span=None,
model_mappings={model_id: _raw_file_id},
user_api_key_dict=_minimal_auth,
)
setattr(response, _file_attr, _unified_file_id)
verbose_proxy_logger.info(
f"CheckBatchCost: converted {_file_attr} "
f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}

83
gateway/Dockerfile Normal file
View file

@ -0,0 +1,83 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN mkdir -p /home/nonroot && \
HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
chown -R nonroot:nonroot /home/nonroot/.cache
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the proxy as that user.
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
COPY --from=builder --chown=nonroot:nonroot /app /app
COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
USER nonroot
EXPOSE 4000/tcp
ENTRYPOINT ["sh", "-c", "exec uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
CMD ["--host", "0.0.0.0", "--port", "4000"]

59
gateway/main.py Normal file
View file

@ -0,0 +1,59 @@
"""Gateway entrypoint.
Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
route table to just the LLM data-plane surface. The trim is purely additive —
no existing module is modified, the full app continues to work via the legacy
entrypoint (`litellm.proxy.proxy_server:app`).
Run with:
uvicorn gateway.main:app --host 0.0.0.0 --port 4000
"""
from contextlib import asynccontextmanager
from fastapi.routing import Mount
# Assemble DATABASE_URL (+ DATABASE_URL_READ_REPLICA) from the discrete
# DATABASE_* env vars before proxy_server imports spin up Prisma. Handles
# both IAM (mint a token) and password auth, writer and reader. The standard
# CLI flow does this in proxy_cli.py; we bypass proxy_cli by uvicorn'ing the
# app directly, so without this Prisma initializes with the placeholder URL
# and every DB-needing endpoint returns "Database not connected".
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
DatabaseURLSettings.from_env().apply_to_env()
from litellm.proxy.proxy_server import app
from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
def _is_gateway_route(route) -> bool:
"""Keep the route on the gateway if its path is in the LLM data-plane surface."""
path = getattr(route, "path", None)
if path is None:
return False
if isinstance(route, Mount):
# Gateway never serves the static UI or its asset bundles.
return False
if path in GATEWAY_EXACT_PATHS:
return True
return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)
# Wrap proxy_server's existing lifespan so the route trim runs *after* its
# startup hooks (and any plugin code those hooks load) have had a chance to
# register routes. A module-load filter would miss routes added during
# startup; running inside the lifespan, after the inner __aenter__, catches
# them while still completing before uvicorn opens the listener.
_proxy_lifespan = app.router.lifespan_context
@asynccontextmanager
async def _gateway_lifespan(app_):
async with _proxy_lifespan(app_):
app_.router.routes = [r for r in app_.router.routes if _is_gateway_route(r)]
yield
app.router.lifespan_context = _gateway_lifespan

0
gateway/routes/__init__.py Normal file
View file

121
gateway/routes/allowlist.py Normal file
View file

@ -0,0 +1,121 @@
"""Path allowlist for the gateway component.
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
responses, vector stores, passthrough providers, realtime websockets, MCP
tool-call endpoints, and operational endpoints (/health, /metrics).
Any path not listed here is dropped from the gateway process so management/UI
endpoints don't ride on the same pods.
Versioned data-plane paths are enumerated explicitly rather than allowing a
blanket `/v1/` or `/v2/` prefix — those broad prefixes would otherwise also
match management routes like `/v1/access_group`, `/v1/tool/{tool_name}/logs`,
`/v2/key/info`, etc.
"""
GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
# OpenAI-compatible data-plane surface (versioned + unversioned)
"/v1/chat/",
"/chat/",
"/v1/completions",
"/completions",
"/v1/embeddings",
"/embeddings",
"/v1/moderations",
"/moderations",
"/v1/audio/",
"/audio/",
"/v1/images/",
"/images/",
"/v1/files",
"/files",
"/v1/batches",
"/batches",
"/v1/fine_tuning/",
"/fine_tuning/",
"/v1/fine-tuning/",
"/fine-tuning/",
"/v1/responses",
"/responses",
"/v1/threads",
"/threads",
"/v1/assistants",
"/assistants",
"/v1/vector_stores",
"/vector_stores",
"/v1/indexes",
"/v1/models",
"/models",
"/openai/",
"/engines/",
# Anthropic / agentic data-plane surface
"/v1/messages",
"/messages",
"/v1/skills",
"/v1/a2a/",
# LiteLLM-native LLM surface
"/v1/rerank",
"/v2/rerank",
"/rerank",
"/v1/ocr",
"/ocr",
"/v1/rag/",
"/rag/",
"/v1/video",
"/v1/videos",
"/video/",
"/videos",
"/v1/search",
"/search",
"/v1/containers",
"/containers",
"/v1/evals",
"/v1/memory",
"/queue/chat/",
# Google data plane (v1beta is the Google AI Studio version)
"/v1beta/",
"/interactions",
# Provider passthrough
"/anthropic/",
"/azure/",
"/azure_ai/",
"/aws/",
"/bedrock/",
"/cohere/",
"/gemini/",
"/google/",
"/vertex_ai/",
"/vertex-ai/",
"/assemblyai/",
"/eu.assemblyai/",
"/langfuse/",
"/vllm/",
"/mistral/",
"/groq/",
"/voyage/",
"/cursor/",
"/milvus/",
"/openai_passthrough/",
# Dynamic provider / toolset passthrough (path templates)
"/{provider}/",
"/toolset/",
# Realtime / streaming
"/v1/realtime",
"/realtime",
# Health & ops
"/health",
"/metrics",
)
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
{
"/",
"/routes",
"/openapi.json",
"/docs",
"/docs/oauth2-redirect",
"/redoc",
"/test",
}
)

8
helm/litellm/Chart.yaml Normal file
View file

@ -0,0 +1,8 @@
apiVersion: v2
name: litellm
description: LiteLLM componentized — gateway, UI backend, and UI as separate services
type: application
version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

49
helm/litellm/templates/NOTES.txt Normal file
View file

@ -0,0 +1,49 @@
LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
Components:
{{- if .Values.gateway.enabled }}
- gateway : Service {{ include "litellm.gateway.fullname" . }} on port {{ .Values.gateway.service.port }}
{{- end }}
{{- if .Values.backend.enabled }}
- backend : Service {{ include "litellm.backend.fullname" . }} on port {{ .Values.backend.service.port }}
{{- end }}
{{- if .Values.ui.enabled }}
- ui : Service {{ include "litellm.ui.fullname" . }} on port {{ .Values.ui.service.port }}
{{- end }}
Port-forward examples:
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.gateway.fullname" . }} {{ .Values.gateway.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }}
Reminders:
- Sensitive values come from Secret references only. Before installing, set:
- masterKey.secretName (Secret with the proxy master key)
- database.writer.{host,port,dbname} (writer connection pieces)
- database.writer.passwordSecret.{name,usernameKey,passwordKey}
(Secret holding the writer DB username + password)
- database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and
omits DATABASE_PASSWORD / DATABASE_URL so the proxy
mints the URL from an IAM token at startup)
- database.reader.host (optional — enables read-replica routing; reader
.passwordSecret.name is required when set, unless
.useIAMAuth is true)
- database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true —
chart emits DATABASE_*_READ_REPLICA env vars and
omits DATABASE_PASSWORD_READ_REPLICA /
DATABASE_URL_READ_REPLICA so the proxy mints the
reader URL from an IAM token at startup)
- redis.passwordSecret.name (optional — set when redis.host is provided and the
cache requires auth)
- redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from
redis.host / redis.port so the proxy's Cache()
constructs a RedisClusterCache; the cluster client
discovers remaining nodes from CLUSTER SLOTS)
- Per-component extras (gateway / backend / ui):
- {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names →
envFrom configMapRef / secretRef)
- {component}.logLevel (renders as LITELLM_LOG)
- gateway.config.proxy_config (rendered into a ConfigMap and mounted at
/app/config/config.yaml; gateway reads it via
CONFIG_FILE_PATH)
- Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

245
helm/litellm/templates/_helpers.tpl Normal file
View file

@ -0,0 +1,245 @@
{{/*
Common naming + label helpers shared by gateway, backend, and ui templates.
*/}}
{{- define "litellm.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.gateway.fullname" -}}
{{- printf "%s-gateway" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.backend.fullname" -}}
{{- printf "%s-backend" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.ui.fullname" -}}
{{- printf "%s-ui" (include "litellm.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "litellm.commonLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
{{- end -}}
{{/*
Per-component selector labels — used in both Service selectors and Deployment matchLabels.
*/}}
{{- define "litellm.gateway.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: gateway
{{- end -}}
{{- define "litellm.backend.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: backend
{{- end -}}
{{- define "litellm.ui.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: ui
{{- end -}}
{{/*
Shared ServiceAccount name used by all three component Deployments. When
`serviceAccount.create` is true and `serviceAccount.name` is empty, default
to the chart fullname. When `create` is false, fall back to the provided
name or the namespace's `default` SA.
*/}}
{{- define "litellm.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/*
Master-key + database + redis env block — shared by gateway, backend, and the
migrations Job.
Invoke with a dict: `(dict "root" $ "component" .Values.gateway)`. `root` is
the chart context (needed for .Values), `component` selects which component's
`extraEnv` / `logLevel` to render.
Sensitive values (master key, DB username + password, Redis password) come
only from referenced Secrets; the chart never accepts inline values for them.
The chart never assembles DATABASE_URL itself. It emits only the discrete
DATABASE_HOST/PORT/USER/NAME/SCHEMA (+ DATABASE_PASSWORD for password auth)
vars; the proxy's entrypoint (DatabaseURLSettings in
litellm/proxy/db/db_url_settings.py) builds the URL from them and
percent-encodes the credentials. Assembling the URL here via Kubernetes
`$(VAR)` substitution would embed the raw secret value, corrupting the URL
whenever the password contains a URL-reserved character (@, /, ?, %, +,
...) — as AWS RDS auto-generated passwords routinely do.
When `database.writer.useIAMAuth: true`, the chart injects
IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
instead of a static password.
The read replica is opt-in via `database.reader.host`. The chart emits
DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
password auth. When `database.reader.useIAMAuth: true` it omits
DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
same way. Reader IAM only takes effect when the writer also uses IAM auth
(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
sets).
*/}}
{{- define "litellm.serverEnv" -}}
{{- $root := .root -}}
{{- $component := .component -}}
- name: LITELLM_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }}
key: {{ $root.Values.masterKey.secretKey | default "master-key" }}
{{- if $component.logLevel }}
- name: LITELLM_LOG
value: {{ $component.logLevel | quote }}
{{- end }}
{{- with $root.Values.database.writer }}
- name: DATABASE_HOST
value: {{ required "database.writer.host is required" .host | quote }}
- name: DATABASE_PORT
value: {{ .port | default 5432 | quote }}
- name: DATABASE_USER
valueFrom:
secretKeyRef:
name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_NAME
value: {{ required "database.writer.dbname is required" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
- name: IAM_TOKEN_DB_AUTH
value: "true"
{{- else }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- with $root.Values.database.reader }}
{{- if .host }}
{{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
{{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
{{- end }}
- name: DATABASE_HOST_READ_REPLICA
value: {{ .host | quote }}
- name: DATABASE_PORT_READ_REPLICA
value: {{ .port | default 5432 | quote }}
- name: DATABASE_NAME_READ_REPLICA
value: {{ required "database.reader.dbname is required when database.reader.host is set" .dbname | quote }}
{{- if .schema }}
- name: DATABASE_SCHEMA_READ_REPLICA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
{{- if .passwordSecret.name }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
{{- end }}
{{- else }}
{{- if not .passwordSecret.name }}
{{- fail "database.reader.passwordSecret.name is required when database.reader.host is set" }}
{{- end }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_PASSWORD_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
The migrations Job (helm.sh/hook: pre-upgrade) is the single owner of
`prisma migrate deploy`. Without this, every gateway/backend pod also runs
Prisma schema-update on startup and contends with the Job — and with each
other — for Prisma's Postgres advisory lock on the writer, which makes the
Job's `migrate deploy` intermittently block until its per-attempt timeout
and retry-exhaust. The Job's entrypoint (migrations/run.py) does not import
proxy_server and never reads DISABLE_SCHEMA_UPDATE, so emitting it here is a
harmless no-op for the Job and authoritative for the app pods.
*/}}
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- if $root.Values.redis.host }}
- name: REDIS_HOST
value: {{ $root.Values.redis.host | quote }}
- name: REDIS_PORT
value: {{ $root.Values.redis.port | quote }}
{{- if $root.Values.redis.passwordSecret.name }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $root.Values.redis.passwordSecret.name }}
key: {{ $root.Values.redis.passwordSecret.passwordKey | default "password" }}
{{- end }}
{{- if $root.Values.redis.cluster }}
{{/* The proxy's Cache() reads REDIS_CLUSTER_NODES as JSON and constructs a
RedisClusterCache when it's set (litellm/caching/caching.py:169-192).
We seed with the single configured endpoint — the cluster client
discovers the remaining nodes from CLUSTER SLOTS at startup. */}}
- name: REDIS_CLUSTER_NODES
value: {{ printf "[{\"host\":%q,\"port\":%v}]" $root.Values.redis.host (int $root.Values.redis.port) | quote }}
{{- end }}
{{- end }}
{{- with $component.extraEnv }}
{{ toYaml . }}
{{- end }}
{{- end -}}
{{/*
Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
lists. Each entry is a resource name; the chart wires the whole ConfigMap /
Secret into the container's env via configMapRef / secretRef.
Invoke with just the component dict, e.g. `.Values.gateway`. Emits nothing
when both lists are empty so the container spec stays clean.
*/}}
{{- define "litellm.envFrom" -}}
{{- $component := . -}}
{{- if or $component.envConfigMaps $component.envSecrets }}
envFrom:
{{- range $component.envConfigMaps }}
- configMapRef:
name: {{ . }}
{{- end }}
{{- range $component.envSecrets }}
- secretRef:
name: {{ . }}
{{- end }}
{{- end }}
{{- end -}}

60
helm/litellm/templates/backend/deployment.yaml Normal file
View file

@ -0,0 +1,60 @@
{{- if .Values.backend.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
selector:
matchLabels:
{{- include "litellm.backend.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.backend.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
ports:
- name: http
containerPort: 4001
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
{{- with .Values.backend.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- with .Values.backend.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/backend/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.backend.fullname" . }}
minReplicas: {{ .Values.backend.hpa.minReplicas }}
maxReplicas: {{ .Values.backend.hpa.maxReplicas }}
metrics:
{{- if .Values.backend.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.backend.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/backend/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.backend.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.backend.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
spec:
type: {{ .Values.backend.service.type }}
ports:
- port: {{ .Values.backend.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.backend.selectorLabels" . | nindent 4 }}
{{- end }}

9
helm/litellm/templates/gateway/configmap.yaml Normal file
View file

@ -0,0 +1,9 @@
{{- if .Values.gateway.config.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.gateway.fullname" . }}-config
data:
config.yaml: |
{{ .Values.gateway.config.proxy_config | toYaml | indent 6 }}
{{- end }}

83
helm/litellm/templates/gateway/deployment.yaml Normal file
View file

@ -0,0 +1,83 @@
{{- if .Values.gateway.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.gateway.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: gateway
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
ports:
- name: http
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.gateway.numWorkers }}
- name: NUM_WORKERS
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if .Values.gateway.config.create }}
volumeMounts:
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
volumes:
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/gateway/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.gateway.fullname" . }}
minReplicas: {{ .Values.gateway.hpa.minReplicas }}
maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
metrics:
{{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/gateway/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.gateway.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
type: {{ .Values.gateway.service.type }}
ports:
- port: {{ .Values.gateway.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
{{- end }}

153
helm/litellm/templates/ingress.yaml Normal file
View file

@ -0,0 +1,153 @@
{{- if .Values.ingress.enabled -}}
{{- $gatewayName := include "litellm.gateway.fullname" . -}}
{{- $backendName := include "litellm.backend.fullname" . -}}
{{- $uiName := include "litellm.ui.fullname" . -}}
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
(e.g. /v1/access_group, /v2/key/info, /v1/tool/*, /v1/agents, /v1/workflows,
/v2/user/info, /v2/team/list, /v2/model/info, /v2/login, /v2/guardrails/*,
/v1/mcp/*) onto the gateway via a broad /v1 or /v2 prefix.
*/}}
{{- $gatewayPrefixes := list
"/v1/chat" "/chat" "/v1/completions" "/completions" "/v1/embeddings" "/embeddings"
"/v1/moderations" "/moderations" "/v1/audio" "/audio" "/v1/images" "/images"
"/v1/files" "/files" "/v1/batches" "/batches" "/v1/fine_tuning" "/fine_tuning"
"/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads"
"/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes"
"/v1/models" "/models" "/openai" "/engines"
"/v1/messages" "/messages" "/v1/skills" "/v1/a2a"
"/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag"
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"
"/v1/realtime" "/realtime"
"/health" "/metrics"
-}}
{{/*
/test is gateway-only as an EXACT path (GATEWAY_EXACT_PATHS), but its
children /test/connection and /test/tools/list are MCP-server management
endpoints kept only on the backend ("/test/" in BACKEND_PATH_PREFIXES).
A Prefix match here would route /test/* to the gateway, which trims those
routes at startup -> 404. So /test is rendered as a standalone Exact path
and /test/* falls through to the backend catch-all.
*/}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- with .Values.ingress.className }}
ingressClassName: {{ . | quote }}
{{- end }}
{{- with .Values.ingress.tls }}
tls:
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- {{- with .Values.ingress.host }}
host: {{ . | quote }}
{{- end }}
http:
paths:
# --- UI (Next.js static export) ---
- path: /
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /favicon.ico
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /litellm-asset-prefix
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /_next
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# /ui/* is where the Next.js SPA serves its login + dashboard
# routes (e.g. /ui/login). Without this, /ui/* falls into the
# catch-all → backend → 404.
- path: /ui
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# Next.js App Router (output: "export", basePath: "") emits the
# RSC/flight payload for every route as a ROOT-level <route>.txt
# (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
# router fetches these on every soft navigation / prefetch as
# <route>.txt?_rsc=<hash> (the query string is irrelevant to path
# matching). They are not under /ui, /_next, or
# /litellm-asset-prefix, so without this rule they fall to the
# backend catch-all → 404 → client-side navigation never settles
# and the login flow spins in an infinite redirect loop
# (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
# export; this rule only routes the request to it. Needs an
# ingress controller whose ImplementationSpecific path is a
# wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
# the AWS Load Balancer Controller.
- path: /*.txt
pathType: ImplementationSpecific
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# --- Gateway data plane ---
# Exact /test only (see the $gatewayPrefixes comment above);
# /test/* MCP management endpoints fall to the backend catch-all.
- path: /test
pathType: Exact
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- range $gatewayPrefixes }}
- path: {{ . }}
pathType: Prefix
backend:
service:
name: {{ $gatewayName }}
port:
number: {{ $gatewayPort }}
{{- end }}
# --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
- path: /
pathType: Prefix
backend:
service:
name: {{ $backendName }}
port:
number: {{ $backendPort }}
{{- end }}

46
helm/litellm/templates/migrations-job.yaml Normal file
View file

@ -0,0 +1,46 @@
{{- if .Values.migrationJob.enabled -}}
# Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against
# the writer database before the gateway and backend Deployments are rolled
# out. Required because the gateway and backend both spin up Prisma at
# startup and assume the LiteLLM schema (LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists.
#
# Running this pre-upgrade closes the window where new application pods would
# otherwise serve traffic against the previous release's unmigrated schema.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: migrations
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: before-hook-creation
helm.sh/hook-weight: "0"
spec:
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
template:
metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 8 }}
app.kubernetes.io/component: migrations
spec:
restartPolicy: Never
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }}
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}

13
helm/litellm/templates/serviceaccount.yaml Normal file
View file

@ -0,0 +1,13 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

70
helm/litellm/templates/ui/deployment.yaml Normal file
View file

@ -0,0 +1,70 @@
{{- if .Values.ui.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
selector:
matchLabels:
{{- include "litellm.ui.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.ui.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: ui
image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.ui.image.pullPolicy }}
ports:
- name: http
containerPort: 3000
protocol: TCP
env:
{{- if .Values.ui.logLevel }}
- name: LITELLM_LOG
value: {{ .Values.ui.logLevel | quote }}
{{- end }}
{{- if .Values.ui.backendUrl }}
- name: LITELLM_BACKEND_URL
value: {{ .Values.ui.backendUrl | quote }}
{{- end }}
{{- with .Values.ui.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.ui | nindent 10 }}
{{- with .Values.ui.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.ui.resources | nindent 12 }}
{{- with .Values.ui.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.ui.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}

33
helm/litellm/templates/ui/hpa.yaml Normal file
View file

@ -0,0 +1,33 @@
{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.ui.fullname" . }}
minReplicas: {{ .Values.ui.hpa.minReplicas }}
maxReplicas: {{ .Values.ui.hpa.maxReplicas }}
metrics:
{{- if .Values.ui.hpa.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.ui.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

18
helm/litellm/templates/ui/service.yaml Normal file
View file

@ -0,0 +1,18 @@
{{- if .Values.ui.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.ui.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
spec:
type: {{ .Values.ui.service.type }}
ports:
- port: {{ .Values.ui.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.ui.selectorLabels" . | nindent 4 }}
{{- end }}

225
helm/litellm/values.yaml Normal file
View file

@ -0,0 +1,225 @@
# LiteLLM helm chart values
nameOverride: ""
fullnameOverride: ""
imagePullSecrets: []
# Optional Ingress wiring the three component Services behind a single L7
# entrypoint. Required when serving the static UI bundle over the network.
ingress:
enabled: false
className: ""
annotations: {}
host: "" # optional; if set, becomes the rule's host
tls: []
# Shared ServiceAccount used by all three component Deployments. Set
# `create: true` to have the chart provision it (e.g. when wiring an EKS
# Pod Identity association by SA name). Set `name` to use an existing SA
# (chart-created or out-of-band). When both are empty / false, pods run
# with the namespace's `default` SA.
serviceAccount:
create: false
automount: true
annotations: {}
name: ""
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
# against the writer database, creating the LiteLLM schema (tables that
# gateway + backend assume exist at startup: LiteLLM_Config,
# LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your
# pipeline runs migrations out-of-band.
#
# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration
# files from `litellm-proxy-extras`) instead of the backend image, so the
# Job doesn't drag in the rest of the proxy and doesn't run `prisma
# generate` — the migration engine doesn't need the generated client.
migrationJob:
enabled: true
backoffLimit: 4
ttlSecondsAfterFinished: 120
resources: {}
image:
repository: ghcr.io/berriai/litellm-migrations
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
# Extra env appended to the migration container. The migration entrypoint
# uses the v2 resolver by default (no diff-and-force recovery — avoids the
# schema thrashing seen during rolling deploys). To opt back into the v1
# resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`.
extraEnv: []
# Required: a master key used by gateway + backend to mint/verify proxy tokens.
# Must reference an existing Secret.
masterKey:
secretName: litellm-master-key-secret # name of a Secret containing the master key
secretKey: master-key
# External Postgres connection.
database:
writer:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-writer-secret
usernameKey: username
passwordKey: password
# Optional read-replica routing. When `reader.host` is set, the proxy routes
# reads (find_*, count, group_by, query_raw/_first) to this endpoint while
# writes stay on the writer. Leave `reader.host` empty to disable.
reader:
host: ""
port: 5432
dbname: ""
schema: ""
useIAMAuth: false
passwordSecret:
name: litellm-reader-secret
usernameKey: username
passwordKey: password
# Optional Redis (caching, rate limiting). Leave host empty to disable.
#
# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster,
# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from
# `host` / `port` as the single seed; the cluster client discovers the
# remaining nodes from CLUSTER SLOTS at startup.
redis:
cluster: false
host: ""
port: 6379
passwordSecret:
name: "" # Leave empty for auth-less Redis
passwordKey: password
# ---------- gateway (LLM data plane) ----------
gateway:
enabled: true
logLevel: INFO
# Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS,
# consumed by the gateway image entrypoint. Default is 1.
numWorkers: 1
extraEnv: [] # Add extra environment variables to the gateway
envConfigMaps: [] # Add extra environment variables to the gateway from config maps
envSecrets: [] # Add extra environment variables to the gateway from secrets
config:
create: true
proxy_config: {}
image:
repository: ghcr.io/berriai/litellm-gateway
tag: "" # defaults to .Chart.AppVersion
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4000
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 10
targetCPUUtilizationPercentage: 70
targetMemoryUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- backend (UI / management API) ----------
backend:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-backend
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 4001
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "2"
memory: 4Gi
livenessProbe:
httpGet: { path: /health/liveliness, port: http }
initialDelaySeconds: 10
periodSeconds: 15
readinessProbe:
httpGet: { path: /health/readiness, port: http }
initialDelaySeconds: 5
periodSeconds: 10
hpa:
enabled: true
minReplicas: 1
maxReplicas: 4
targetCPUUtilizationPercentage: 70
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}
# ---------- ui (Next.js static dashboard) ----------
ui:
enabled: true
logLevel: INFO
extraEnv: []
envConfigMaps: []
envSecrets: []
image:
repository: ghcr.io/berriai/litellm-ui
tag: ""
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 3000
# The dashboard expects to know where to reach the backend API. Set this to
# the externally-routable URL (typically the ingress host + /api or similar).
backendUrl: ""
resources:
requests:
cpu: 500m
memory: 500Mi
limits:
cpu: "1"
memory: 1Gi
livenessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 5
periodSeconds: 20
readinessProbe:
httpGet: { path: /, port: http }
initialDelaySeconds: 2
periodSeconds: 10
hpa:
enabled: false
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 80
podAnnotations: {}
nodeSelector: {}
tolerations: []
affinity: {}

9
license_cache.json
View file

File diff suppressed because one or more lines are too long

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260421135425_add_team_membership_total_spend/migration.sql
View file

@ -1,3 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260513120000_add_delegate_auth_to_upstream_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "delegate_auth_to_upstream" BOOLEAN NOT NULL DEFAULT false;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -323,6 +323,7 @@ model LiteLLM_MCPServerTable {
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(true)
delegate_auth_to_upstream Boolean @default(false)
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.71"
version = "0.4.72"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.71"
version = "0.4.72"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

6
litellm/__init__.py
View file

@ -1426,6 +1426,12 @@ if TYPE_CHECKING:
)
from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig
from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig
from .llms.bedrock.claude_platform.transformation import (
BedrockClaudePlatformConfig as BedrockClaudePlatformConfig,
)
from .llms.bedrock.claude_platform.messages_transformation import (
BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig,
)
from .llms.anthropic.completion.transformation import (
AnthropicTextConfig as AnthropicTextConfig,
)

12
litellm/_lazy_imports_registry.py
View file

@ -131,6 +131,7 @@ LLM_CONFIG_NAMES = (
"OpenrouterConfig",
"DataRobotConfig",
"AnthropicConfig",
"BedrockClaudePlatformConfig",
"AnthropicTextConfig",
"GroqSTTConfig",
"TritonConfig",
@ -170,6 +171,7 @@ LLM_CONFIG_NAMES = (
"SagemakerNovaConfig",
"CohereChatConfig",
"AnthropicMessagesConfig",
"BedrockClaudePlatformMessagesConfig",
"AmazonAnthropicClaudeMessagesConfig",
"AmazonMantleMessagesConfig",
"TogetherAIConfig",
@ -374,7 +376,6 @@ UTILS_MODULE_NAMES = (
"HTTPHandler",
"get_num_retries_from_retry_policy",
"reset_retry_policy",
"get_secret",
"get_coroutine_checker",
"get_litellm_logging_class",
"get_set_callbacks",
@ -610,6 +611,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
"OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"),
"DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"),
"AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"),
"BedrockClaudePlatformConfig": (
".llms.bedrock.claude_platform.transformation",
"BedrockClaudePlatformConfig",
),
"AnthropicTextConfig": (
".llms.anthropic.completion.transformation",
"AnthropicTextConfig",
@ -712,6 +717,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.anthropic.experimental_pass_through.messages.transformation",
"AnthropicMessagesConfig",
),
"BedrockClaudePlatformMessagesConfig": (
".llms.bedrock.claude_platform.messages_transformation",
"BedrockClaudePlatformMessagesConfig",
),
"AmazonAnthropicClaudeMessagesConfig": (
".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation",
"AmazonAnthropicClaudeMessagesConfig",
@ -1274,7 +1283,6 @@ _UTILS_MODULE_IMPORT_MAP = {
"litellm.router_utils.get_retry_from_policy",
"reset_retry_policy",
),
"get_secret": ("litellm.secret_managers.main", "get_secret"),
"get_coroutine_checker": (
"litellm.litellm_core_utils.cached_imports",
"get_coroutine_checker",

1
litellm/_logging.py
View file

@ -404,6 +404,7 @@ def _turn_on_debug():
def _disable_debugging():
"""Disable the package, router, and proxy verbose loggers."""
verbose_logger.disabled = True
verbose_router_logger.disabled = True
verbose_proxy_logger.disabled = True

68
litellm/_redis.py
View file

@ -41,7 +41,7 @@ def _get_redis_kwargs():
"retry",
}
include_args = [
include_args = {
"url",
"redis_connect_func",
"gcp_service_account",
@ -50,9 +50,9 @@ def _get_redis_kwargs():
"azure_client_id",
"azure_tenant_id",
"azure_client_secret",
]
}
available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
available_args = {x for x in arg_spec.args if x not in exclude_args} | include_args
return available_args
@ -84,23 +84,23 @@ def _get_redis_cluster_kwargs(client=None):
# Only allow primitive arguments
exclude_args = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"}
available_args = [x for x in arg_spec.args if x not in exclude_args]
available_args.append("password")
available_args.append("username")
available_args.append("ssl")
available_args.append("ssl_cert_reqs")
available_args.append("ssl_check_hostname")
available_args.append("ssl_ca_certs")
available_args.append(
"redis_connect_func"
) # Needed for sync clusters and IAM detection
available_args.append("gcp_service_account")
available_args.append("gcp_ssl_ca_certs")
available_args.append("azure_redis_ad_token")
available_args.append("azure_client_id")
available_args.append("azure_tenant_id")
available_args.append("azure_client_secret")
available_args.append("max_connections")
available_args = {x for x in arg_spec.args if x not in exclude_args}
available_args |= {
"password",
"username",
"ssl",
"ssl_cert_reqs",
"ssl_check_hostname",
"ssl_ca_certs",
"redis_connect_func", # Needed for sync clusters and IAM detection
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
"azure_client_id",
"azure_tenant_id",
"azure_client_secret",
"max_connections",
}
return available_args
@ -479,10 +479,24 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs) # type: ignore
def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
connection_kwargs = {}
args = _get_redis_kwargs()
for arg in redis_kwargs:
if arg in args:
connection_kwargs[arg] = redis_kwargs[arg]
return connection_kwargs
def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
sentinel_nodes = redis_kwargs.get("sentinel_nodes")
sentinel_password = redis_kwargs.get("sentinel_password")
service_name = redis_kwargs.get("service_name")
connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
if not sentinel_nodes or not service_name:
raise ValueError(
@ -494,19 +508,22 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
# Set up the Sentinel client
sentinel = redis.Sentinel(
sentinel_nodes,
socket_timeout=REDIS_SOCKET_TIMEOUT,
password=sentinel_password,
sentinel_kwargs=sentinel_kwargs,
)
# Return the master instance for the given service
return sentinel.master_for(service_name)
return sentinel.master_for(service_name, **connection_kwargs)
def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
sentinel_nodes = redis_kwargs.get("sentinel_nodes")
sentinel_password = redis_kwargs.get("sentinel_password")
service_name = redis_kwargs.get("service_name")
connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
if not sentinel_nodes or not service_name:
raise ValueError(
@ -518,13 +535,12 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
# Set up the Sentinel client
sentinel = async_redis.Sentinel(
sentinel_nodes,
socket_timeout=REDIS_SOCKET_TIMEOUT,
password=sentinel_password,
sentinel_kwargs=sentinel_kwargs,
)
# Return the master instance for the given service
return sentinel.master_for(service_name)
return sentinel.master_for(service_name, **connection_kwargs)
def get_redis_client(**env_overrides):

23
litellm/batches/batch_utils.py
View file

@ -113,8 +113,11 @@ def _batch_cost_calculator(
"""
Calculate the cost of a batch based on the output file id
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
batch_cost, _ = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)
@ -136,10 +139,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
model_name: Optional[str] = None,
) -> Tuple[float, Usage]:
"""
Calculate both cost and usage from Vertex AI batch responses.
Calculate both cost and usage from raw Vertex AI batch responses.
Vertex AI batch output lines have format:
{"request": ..., "status": "", "response": {"candidates": [...], "usageMetadata": {...}}}
Used only when ``litellm.disable_vertex_batch_output_transformation = True``.
In that case the GCS predictions.jsonl is returned as-is, with each line in
the native Vertex format:
{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
"""
@ -362,8 +368,11 @@ def _get_batch_job_total_usage_from_file_content(
"""
Get the tokens of a batch job from the file content
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
_, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)

12
litellm/budget_manager.py
View file

@ -178,6 +178,18 @@ class BudgetManager:
return list(self.user_dict.keys())
def reset_cost(self, user):
"""
Reset the tracked spend for a user back to zero.
Clears both the aggregate ``current_cost`` and the per-model
``model_cost`` breakdown stored for the given user.
Args:
user: The user identifier whose cost should be reset.
Returns:
dict: ``{"user": <updated user record>}`` reflecting the reset state.
"""
self.user_dict[user]["current_cost"] = 0
self.user_dict[user]["model_cost"] = {}
return {"user": self.user_dict[user]}

9
litellm/constants.py
View file

@ -1569,6 +1569,15 @@ DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
)
DEFAULT_ACCESS_GROUP_CACHE_TTL = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600))
# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
# callers from forcing a DB query per request for unknown names, while bounding
# staleness so a transient DB error (which surfaces as an empty list) cannot
# hide a real group for long.
DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL = 10
# Maximum number of comma-separated MCP server / access-group tokens accepted
# in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
# fan-out an authenticated caller can trigger by stuffing the path with tokens.
DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS = 16
# Sentry Scrubbing Configuration
SENTRY_DENYLIST = [

20
litellm/cost_calculator.py
View file

@ -2120,6 +2120,26 @@ def batch_cost_calculator(
)
except Exception:
model_info = None
elif not any(
model_info.get(k) is not None
for k in (
"input_cost_per_token_batches",
"input_cost_per_token",
"output_cost_per_token_batches",
"output_cost_per_token",
)
):
# model_info was provided (e.g. deployment metadata with only id/db_model)
# but carries no pricing fields. Fall back to the global pricing table so
# that standard model pricing is used instead of silently returning $0.
try:
global_info = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if global_info:
model_info = global_info
except Exception:
pass
if not model_info:
return 0.0, 0.0

29
litellm/integrations/custom_guardrail.py
View file

@ -888,6 +888,15 @@ def log_guardrail_information(func):
- pre_call
- during_call
- post_call
Some guardrails (e.g. ``block_code_execution``) call
``add_standard_logging_guardrail_information_to_request_data`` directly
from inside the wrapped function so they can record a richer payload
(structured detections, tracing detail) than this decorator's
"allow"/"mask"/raw-response default. To avoid double-recording in that
case (which would emit two spans, two Datadog records, two spend-log
entries, etc.), snapshot the entry count before invocation: if the
wrapped function already appended its own entry, skip the auto-record.
"""
import functools
import inspect
@ -907,6 +916,16 @@ def log_guardrail_information(func):
return GuardrailEventHooks.post_call
return None
def _count_recorded_guardrail_entries(request_data: dict) -> int:
total = 0
for container_key in ("metadata", "litellm_metadata"):
container = request_data.get(container_key)
if isinstance(container, dict):
entries = container.get("standard_logging_guardrail_information")
if isinstance(entries, list):
total += len(entries)
return total
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now() # Move start_time inside the wrapper
@ -919,8 +938,11 @@ def log_guardrail_information(func):
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
entries_before = _count_recorded_guardrail_entries(request_data)
try:
response = await func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
return response
return self._process_response(
response=response,
request_data=request_data,
@ -931,6 +953,8 @@ def log_guardrail_information(func):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
raise
return self._process_error(
e=e,
request_data=request_data,
@ -952,8 +976,11 @@ def log_guardrail_information(func):
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
entries_before = _count_recorded_guardrail_entries(request_data)
try:
response = func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
return response
return self._process_response(
response=response,
request_data=request_data,
@ -962,6 +989,8 @@ def log_guardrail_information(func):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
raise
return self._process_error(
e=e,
request_data=request_data,

21
litellm/integrations/custom_logger.py
View file

@ -697,6 +697,27 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
return AgenticLoopPlan(run_agentic_loop=False)
async def async_post_agentic_loop_response_hook(
self,
response: Any,
plan: AgenticLoopPlan,
kwargs: Dict,
) -> Any:
"""
Post-process the response returned by the agentic-loop follow-up call.
Called after BaseLLMHTTPHandler executes ``AgenticLoopPlan.request_patch``
and receives the final response from the provider. Lets callbacks shape
what the client sees without bypassing the loop's safety / observability
machinery (depth tracking, fingerprinting, etc.).
Use ``plan.metadata`` to carry whatever the build step decided to expose
for post-processing (e.g. native tool_result blocks to inject).
Default returns ``response`` unchanged.
"""
return response
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,

419
litellm/integrations/opentelemetry.py
View file

@ -1,7 +1,7 @@
import os
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -10,6 +10,12 @@ from litellm.integrations._types.open_inference import (
SpanAttributes,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
OTEL_SEMCONV_STABILITY_OPT_IN_ENV,
OTELGenAISemconvMixin,
OTELSemconvCategory,
parse_semconv_opt_in,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.services import ServiceLoggerPayload
@ -53,6 +59,11 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
# OTel-standard names. status is also kept under error.code for back compat.
HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
HTTP_ROUTE_ATTRIBUTE = "http.route"
URL_PATH_ATTRIBUTE = "url.path"
PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -85,6 +96,7 @@ class OpenTelemetryConfig:
# Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
capture_message_content: Optional[str] = None
semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set)
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -110,6 +122,11 @@ class OpenTelemetryConfig:
self.ignore_context_propagation = str_to_bool(
os.getenv("OTEL_IGNORE_CONTEXT_PROPAGATION")
)
# Resolve the env opt-in once here so self.semconv_stability_opt_in is the
# single source of truth: the union of programmatic and env categories.
self.semconv_stability_opt_in |= parse_semconv_opt_in(
os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV)
)
@classmethod
def from_env(cls):
@ -157,7 +174,7 @@ class OpenTelemetryConfig:
)
class OpenTelemetry(CustomLogger):
class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
def __init__(
self,
config: Optional[OpenTelemetryConfig] = None,
@ -237,7 +254,14 @@ class OpenTelemetry(CustomLogger):
not isinstance(cb, OpenTelemetry) for cb in litellm.service_callback
):
litellm.service_callback.append(self)
setattr(proxy_server, "open_telemetry_logger", self)
# avoid proxy logger ownership being overwritten by later
# handlers. Multiple integrations (default OTEL, Langfuse OTEL,
# Arize OTEL, etc.) may initialize in sequence; without this guard,
# the last one silently replaces the first and breaks expected
# routing for proxy_server.open_telemetry_logger consumers.
# Behavior: first-registered wins.
if getattr(proxy_server, "open_telemetry_logger", None) is None:
setattr(proxy_server, "open_telemetry_logger", self)
def _get_or_create_provider(
self,
@ -648,6 +672,31 @@ class OpenTelemetry(CustomLogger):
parent_otel_span = user_api_key_dict.parent_otel_span
if parent_otel_span is not None:
parent_otel_span.set_status(Status(StatusCode.ERROR))
# Stamp structured error attrs on the SERVER span itself; the
# failure path otherwise only sets its status (_handle_failure
# records on the litellm_request child span). Inline import:
# litellm_logging <-> integrations is circular.
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
traceback_str=traceback_str,
)
self._record_exception_on_span(
span=parent_otel_span,
kwargs={
"exception": original_exception,
"standard_logging_object": {"error_information": error_information},
},
)
# Pre-request latency (request_data carries the propagated
# metadata on the failure path; omitted if it failed before handoff).
self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
_span_name = "Failed Proxy Server Request"
# Exception Logging Child Span
@ -684,6 +733,14 @@ class OpenTelemetry(CustomLogger):
ctx, _ = self._get_span_context(kwargs, default_span=parent_span)
# Pre-request latency on the SERVER span (success path).
self.set_preprocessing_duration_attribute(parent_span, kwargs)
# http.response.status_code on the SERVER span (success path).
# A successful proxy response is HTTP 200; the failure path sets
# this from the error code in _record_exception_on_span.
self.set_response_status_code_attribute(parent_span, 200)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
@ -794,12 +851,100 @@ class OpenTelemetry(CustomLogger):
# End of Team/Key Based Logging Control Flow
#########################################################
def _emit_once(self, kwargs: dict, *scope: object) -> bool:
"""Return True the first time this handler is asked to emit a span
for the given (handler, scope) on this kwargs; False on repeats.
Used to suppress duplicate span emission for two distinct patterns:
1. **Handler-level dual-fire**: streaming code paths trigger both
the sync and async callback for one request, so ``_handle_success``
/ ``_handle_failure`` would otherwise produce two
``litellm_request`` spans. Scope: ``("success",)`` / ``("failure",)``.
2. **Payload-driven multi-entrypoint emission**: a span loop that
reads entries from ``standard_logging_payload`` (currently only
guardrails) is invoked from multiple lifecycle points
(post-call hooks, success callback, failure callback). The list
can be re-read with mutated entries between calls, so dedupe
must be at entry granularity. Scope: the entry's stable identity.
``scope`` parts can be any hashable identity. The marker is stored
in ``kwargs["litellm_params"]["metadata"]["_otel_internal"]`` so it
is request-local (kwargs is shared across the sync/async callbacks
and lifecycle hooks for one request).
"""
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
litellm_params = {}
kwargs["litellm_params"] = litellm_params
_metadata = litellm_params.get("metadata")
if not isinstance(_metadata, dict):
_metadata = {}
litellm_params["metadata"] = _metadata
_otel_internal = _metadata.get("_otel_internal")
if not isinstance(_otel_internal, dict):
_otel_internal = {}
_metadata["_otel_internal"] = _otel_internal
spans_logged = _otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):
spans_logged = {}
_otel_internal["spans_logged"] = spans_logged
dedupe_key = (self.__class__.__name__, id(self), *scope)
if spans_logged.get(dedupe_key) is True:
return False
spans_logged[dedupe_key] = True
return True
def _end_proxy_span_from_kwargs(self, kwargs: dict, end_time) -> None:
"""Close the proxy-level parent span if it is still recording.
This helper retrieves the proxy span directly from kwargs metadata
and closes it after all child spans have been recorded.
Only called from the success path. The failure path deliberately
leaves the proxy span open so ``async_post_call_failure_hook`` can
append the ``"Failed Proxy Server Request"`` child span before
closing it.
Only spans named ``LITELLM_PROXY_REQUEST_SPAN_NAME`` are closed —
externally provided spans must not be closed by LiteLLM.
"""
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {}) or {}
proxy_span = _metadata.get("litellm_parent_otel_span", None)
if (
proxy_span is not None
and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
and hasattr(proxy_span, "is_recording")
and proxy_span.is_recording()
):
proxy_span.end(end_time=self._to_ns(end_time))
def _handle_success(self, kwargs, response_obj, start_time, end_time):
"""Create the litellm_request span then close the proxy span."""
verbose_logger.debug(
"OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
kwargs,
self.config,
)
# sync + async success handlers can both fire for one
# request (notably in streaming code paths). Guard against duplicate
# span writes — but still close the proxy span on the skip path so
# the trace doesn't leak an open root span.
if not self._emit_once(kwargs, "success"):
verbose_logger.debug(
"OpenTelemetry: skipping duplicate success span for handler=%s",
self.__class__.__name__,
)
self._end_proxy_span_from_kwargs(kwargs, end_time)
return
ctx, parent_span = self._get_span_context(kwargs)
if self.config.ignore_context_propagation:
@ -859,7 +1004,7 @@ class OpenTelemetry(CustomLogger):
# 6. Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
# However, proxy-created spans should be closed here.
if (
parent_span is not None
and hasattr(parent_span, "name")
@ -867,6 +1012,11 @@ class OpenTelemetry(CustomLogger):
):
parent_span.end(end_time=self._to_ns(end_time))
# close the proxy span explicitly from kwargs metadata
# after all child spans (litellm_request, guardrail, raw_request)
# have been fully recorded and exported.
self._end_proxy_span_from_kwargs(kwargs, end_time)
def _start_primary_span(
self,
kwargs,
@ -879,13 +1029,14 @@ class OpenTelemetry(CustomLogger):
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
# Always create a new span
# The parent relationship is preserved through the context parameter
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=context,
)
span_kwargs: Dict[str, Any] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": context,
}
if self._gen_ai_semconv_latest_experimental:
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
span.set_status(Status(StatusCode.OK))
self.set_attributes(span, kwargs, response_obj)
@ -898,6 +1049,10 @@ class OpenTelemetry(CustomLogger):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
# raw_gen_ai_request is non-standard in semconv mode.
if self._gen_ai_semconv_latest_experimental:
return
if not self._capture_in_span():
return
@ -923,7 +1078,11 @@ class OpenTelemetry(CustomLogger):
provider = params.get("custom_llm_provider", "Unknown")
common_attrs = {
"gen_ai.operation.name": "chat",
"gen_ai.operation.name": (
self._gen_ai_operation_name(kwargs)
if self._gen_ai_semconv_latest_experimental
else "chat"
),
"gen_ai.system": provider,
"gen_ai.request.model": kwargs.get("model"),
"gen_ai.framework": "litellm",
@ -1146,6 +1305,24 @@ class OpenTelemetry(CustomLogger):
response_duration_seconds, attributes=common_attrs
)
@staticmethod
def _otel_log_types():
"""Resolve ``(LogRecord, SeverityNumber)`` across OTEL SDK versions.
``LogRecord`` moved out of ``opentelemetry.sdk._logs`` in OTEL >= 1.39.0
(open-telemetry/opentelemetry-python#4676). Imports stay function-local
because the SDK is an optional dependency.
"""
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import LogRecord # OTEL < 1.39.0
except ImportError:
from opentelemetry.sdk._logs._internal import ( # OTEL >= 1.39.0
LogRecord,
)
return LogRecord, SeverityNumber
def _emit_semantic_logs(self, kwargs, response_obj, span: Span):
if not self.config.enable_events:
return
@ -1159,16 +1336,7 @@ class OpenTelemetry(CustomLogger):
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import ( # type: ignore[attr-defined] # OTEL < 1.39.0
LogRecord as SdkLogRecord,
)
except ImportError:
from opentelemetry.sdk._logs._internal import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL >= 1.39.0
)
SdkLogRecord, SeverityNumber = self._otel_log_types()
# Resolve through the handler's own LoggerProvider (which may be a
# private one when skip_set_global=True) rather than the module-level
@ -1180,6 +1348,16 @@ class OpenTelemetry(CustomLogger):
"custom_llm_provider", "Unknown"
)
if self._gen_ai_semconv_latest_experimental:
self._emit_inference_details_event(
kwargs=kwargs,
response_obj=response_obj,
provider=provider,
otel_logger=otel_logger,
parent_ctx=parent_ctx,
)
return
# per-message events
for msg in kwargs.get("messages", []):
role = msg.get("role", "user")
@ -1296,6 +1474,21 @@ class OpenTelemetry(CustomLogger):
for guardrail_information in guardrail_information_list:
start_time_float = guardrail_information.get("start_time")
end_time_float = guardrail_information.get("end_time")
# ``_create_guardrail_span`` is called from three lifecycle
# points (``async_post_call_success_hook``, ``_handle_success``,
# ``_handle_failure``) and re-reads the (mutating) entry list
# each time. Dedupe at entry granularity so a single real
# guardrail invocation produces exactly one span per handler.
if not self._emit_once(
kwargs,
"guardrail",
guardrail_information.get("guardrail_name"),
start_time_float,
guardrail_information.get("guardrail_mode"),
):
continue
start_time_datetime = datetime.now()
if start_time_float is not None:
start_time_datetime = datetime.fromtimestamp(start_time_float)
@ -1349,6 +1542,21 @@ class OpenTelemetry(CustomLogger):
kwargs,
self.config,
)
# sync + async failure handlers can both fire for one
# request (notably in streaming code paths), producing two
# semantically identical ERROR spans. Unlike the success path, the
# proxy span is intentionally left open here so that
# ``async_post_call_failure_hook`` can append the
# "Failed Proxy Server Request" child span before closing it —
# there is no proxy-span side-effect to preserve on the skip path.
if not self._emit_once(kwargs, "failure"):
verbose_logger.debug(
"OpenTelemetry: skipping duplicate failure span for handler=%s",
self.__class__.__name__,
)
return
_parent_context, parent_otel_span = self._get_span_context(kwargs)
if self.config.ignore_context_propagation:
@ -1366,11 +1574,14 @@ class OpenTelemetry(CustomLogger):
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
)
span_kwargs: Dict[str, Any] = {
"name": self._get_span_name(kwargs),
"start_time": self._to_ns(start_time),
"context": _parent_context,
}
if self._gen_ai_semconv_latest_experimental:
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
@ -1454,6 +1665,19 @@ class OpenTelemetry(CustomLogger):
value=error_information["error_code"],
)
# Also expose under the OTel-standard name as an int
# (error_code is a str, may be non-numeric).
_error_code_val = error_information["error_code"]
if _error_code_val is not None:
try:
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(_error_code_val),
)
except (ValueError, TypeError):
pass
if error_information.get("error_class"):
self.safe_set_attribute(
span=span,
@ -1652,11 +1876,21 @@ class OpenTelemetry(CustomLogger):
)
# The Generative AI Provider: Azure, OpenAI, etc.
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM.value,
value=litellm_params.get("custom_llm_provider", "Unknown"),
)
provider_name = litellm_params.get("custom_llm_provider", "Unknown")
# Latest-experimental semconv replaced gen_ai.system with
# gen_ai.provider.name; emit only the conformant key in that mode.
if self._gen_ai_semconv_latest_experimental:
self.safe_set_attribute(
span=span,
key="gen_ai.provider.name",
value=provider_name,
)
else:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM.value,
value=provider_name,
)
# The maximum number of tokens the LLM generates for a request.
if optional_params.get("max_tokens"):
@ -1682,11 +1916,17 @@ class OpenTelemetry(CustomLogger):
value=optional_params.get("top_p"),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_IS_STREAMING.value,
value=str(optional_params.get("stream", False)),
)
if self._gen_ai_semconv_latest_experimental:
# Semconv emits gen_ai.request.stream (only when streaming) via
# _set_semconv_request_attributes; skip the legacy llm.is_streaming.
self._set_semconv_request_attributes(span, optional_params)
self._set_semconv_cache_token_attributes(span, standard_logging_payload)
else:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_IS_STREAMING.value,
value=str(optional_params.get("stream", False)),
)
if optional_params.get("user"):
self.safe_set_attribute(
@ -1807,14 +2047,18 @@ class OpenTelemetry(CustomLogger):
value=safe_dumps(transformed_system_instructions),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
value=(
if self._gen_ai_semconv_latest_experimental:
operation_name = self._gen_ai_operation_name(kwargs)
else:
operation_name = (
"chat"
if standard_logging_payload.get("call_type") == "completion"
else standard_logging_payload.get("call_type") or "chat"
),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OPERATION_NAME.value,
value=operation_name,
)
if standard_logging_payload.get("request_id"):
@ -2151,6 +2395,10 @@ class OpenTelemetry(CustomLogger):
if generation_name:
return generation_name
if self._gen_ai_semconv_latest_experimental:
model = kwargs.get("model") or "unknown"
return f"{self._gen_ai_operation_name(kwargs)} {model}"
return LITELLM_REQUEST_SPAN_NAME
def get_traceparent_from_header(self, headers):
@ -2188,7 +2436,7 @@ class OpenTelemetry(CustomLogger):
verbose_logger.debug(
"OpenTelemetry: Using explicit parent span from metadata"
)
return trace.set_span_in_context(parent_otel_span), parent_otel_span
return trace.set_span_in_context(parent_otel_span), None
# Priority 2: HTTP traceparent header
if traceparent is not None:
@ -2692,3 +2940,86 @@ class OpenTelemetry(CustomLogger):
context=self.get_traceparent_from_header(headers=headers),
kind=self.span_kind.SERVER,
)
def set_proxy_request_route_attributes(
self,
span: Optional[Span],
*,
url_path: Optional[str] = None,
http_route: Optional[str] = None,
) -> None:
"""
Set OTel-standard ``http.route`` / ``url.path`` on the proxy SERVER
span. Called from the auth path, the only point where both the
SERVER span and the request are in hand. No-op if span/value missing.
"""
if span is None:
return
if url_path:
self.safe_set_attribute(span=span, key=URL_PATH_ATTRIBUTE, value=url_path)
if http_route:
self.safe_set_attribute(
span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
)
def set_response_status_code_attribute(
self, span: Optional[Span], status_code: Optional[int]
) -> None:
"""
Set OTel-standard ``http.response.status_code`` (int) on the proxy
SERVER span. The failure path sets this from the error code in
``_record_exception_on_span``; this is the success-path counterpart
so the attribute is present on every SERVER span regardless of
outcome (required by the HTTP semconv, and needed for error-ratio /
status-breakdown dashboards). No-op if span/value missing.
"""
if span is None or status_code is None:
return
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(status_code),
)
def set_preprocessing_duration_attribute(
self, span: Optional[Span], container: Any
) -> None:
"""
Set ``litellm.preprocessing.duration_ms`` (proxy-receive -> first
provider handoff) on the proxy SERVER span. ``litellm_received_at``
rides request metadata; ``first_api_call_start_time`` is the
set-once first-handoff instant (retries/backoff excluded). Works
uniformly for the success (model_call_details) and failure
(request_data) containers. No-op if span/either anchor is missing.
"""
if span is None or not isinstance(container, dict):
return
received_at = None
# first_api_call_start_time is top-level (never in user metadata).
first_handoff = container.get("first_api_call_start_time")
_lp = container.get("litellm_params")
for _md in (
(_lp or {}).get("metadata") if isinstance(_lp, dict) else None,
container.get("metadata"),
container.get("litellm_metadata"),
):
if isinstance(_md, dict):
received_at = received_at or _md.get("litellm_received_at")
if received_at is None or first_handoff is None:
return
try:
start_ts = self._to_timestamp(received_at)
end_ts = self._to_timestamp(first_handoff)
except Exception:
return
if start_ts is None or end_ts is None:
return
duration_ms = (end_ts - start_ts) * 1000.0
# Clock skew → omit rather than emit a negative latency.
if duration_ms < 0:
return
self.safe_set_attribute(
span=span,
key=PREPROCESSING_DURATION_MS_ATTRIBUTE,
value=duration_ms,
)

271
litellm/integrations/opentelemetry_utils/gen_ai_semconv.py Normal file
View file

@ -0,0 +1,271 @@
"""OTEL GenAI ``gen_ai_latest_experimental`` semantic conventions.
Setting ``OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`` switches the
emitted traces to the experimental OTEL GenAI conventions
(https://opentelemetry.io/docs/specs/semconv/gen-ai/). Concretely, versus the
default LiteLLM output:
Request span:
- name is ``{operation} {model}`` (e.g. ``chat gpt-4``) instead of
``litellm_request``; span kind is ``CLIENT``.
- ``gen_ai.operation.name`` is the actual operation (``chat`` /
``text_completion`` / ``embeddings``) instead of always ``chat``.
- the provider is reported as ``gen_ai.provider.name``; the superseded
``gen_ai.system`` and the legacy ``llm.is_streaming`` are dropped.
- adds ``gen_ai.request.{frequency_penalty,presence_penalty,top_k,seed}``,
``gen_ai.request.stop_sequences`` (a string array),
``gen_ai.request.stream`` (only when streaming),
``gen_ai.request.choice.count`` (only when n > 1), and
``gen_ai.usage.cache_{creation,read}.input_tokens``.
- the non-standard ``raw_gen_ai_request`` child span is no longer created.
Events:
- the per-message ``gen_ai.content.prompt`` / per-choice
``gen_ai.content.completion`` log events are replaced by a single
``gen_ai.client.inference.operation.details`` log event carrying
``gen_ai.input.messages`` / ``gen_ai.output.messages`` (message content
included only when content capture is enabled).
"""
from datetime import datetime
from enum import Enum
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import OpenTelemetryConfig
Span = Union[_Span, Any]
else:
Span = Any
# OTEL_SEMCONV_STABILITY_OPT_IN is a comma-separated list of category-specific
# opt-in values. See https://opentelemetry.io/docs/specs/semconv/gen-ai/
OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN"
class OTELSemconvCategory(Enum):
GEN_AI_LATEST_EXPERIMENTAL = "gen_ai_latest_experimental"
# Reverse lookup: opt-in token string -> OTELSemconvCategory.
_SEMCONV_CATEGORY_BY_VALUE = {
category.value: category for category in OTELSemconvCategory
}
# LiteLLM optional_params key -> OTEL gen_ai semconv span attribute.
_SEMCONV_REQUEST_ATTRIBUTES = {
"frequency_penalty": "gen_ai.request.frequency_penalty",
"presence_penalty": "gen_ai.request.presence_penalty",
"top_k": "gen_ai.request.top_k",
"seed": "gen_ai.request.seed",
}
# usage_object key -> OTEL gen_ai semconv cache-token span attribute.
_SEMCONV_CACHE_TOKEN_ATTRIBUTES = {
"cache_creation_input_tokens": "gen_ai.usage.cache_creation.input_tokens",
"cache_read_input_tokens": "gen_ai.usage.cache_read.input_tokens",
}
# Name of the consolidated GenAI inference event (replaces the legacy
# per-message gen_ai.content.prompt / per-choice gen_ai.content.completion).
_INFERENCE_DETAILS_EVENT_NAME = "gen_ai.client.inference.operation.details"
def parse_semconv_opt_in(raw: Optional[str]) -> Set[OTELSemconvCategory]:
"""Parse the comma-separated OTEL_SEMCONV_STABILITY_OPT_IN value into the
set of recognized categories. Unknown tokens are ignored per the spec."""
if not raw:
return set()
return {
_SEMCONV_CATEGORY_BY_VALUE[token]
for token in (part.strip() for part in raw.split(","))
if token in _SEMCONV_CATEGORY_BY_VALUE
}
class OTELGenAISemconvMixin:
"""OTEL GenAI ``gen_ai_latest_experimental`` semantic-convention behavior.
Mixed into ``OpenTelemetry`` (its only host). Every member is internal to
the OTEL integration; the leading underscore marks "subsystem-internal",
not "class-private" (the host lives in a sibling module).
Members the host calls (the mixin -> host contract):
- ``_gen_ai_semconv_latest_experimental`` -- opt-in gate; guards every
semconv code path in ``opentelemetry.py``.
- ``_gen_ai_operation_name`` -- LiteLLM ``call_type`` -> spec
``gen_ai.operation.name``.
- ``_set_semconv_request_attributes`` /
``_set_semconv_cache_token_attributes`` -- add the ``gen_ai.request.*``
/ ``gen_ai.usage.cache_*`` span attributes.
- ``_emit_inference_details_event`` -- emit the consolidated event.
Helpers the host must provide (declared under ``TYPE_CHECKING`` below):
``config``, ``safe_set_attribute``, ``_capture_in_event``,
``_transform_messages_to_otel_semantic_conventions``,
``_transform_choices_to_otel_semantic_conventions``, ``_to_ns``,
``_otel_log_types``.
"""
if TYPE_CHECKING:
config: "OpenTelemetryConfig"
def safe_set_attribute(self, span: Span, key: str, value: Any) -> None: ...
def _capture_in_event(self) -> bool: ...
def _transform_messages_to_otel_semantic_conventions(
self, messages: Union[List[dict], str]
) -> List[dict]: ...
def _transform_choices_to_otel_semantic_conventions(
self, choices: List[dict]
) -> List[dict]: ...
def _to_ns(self, dt: datetime) -> int: ...
def _otel_log_types(self) -> Tuple[Any, Any]: ...
@property
def _gen_ai_semconv_latest_experimental(self) -> bool:
"""Whether the ``gen_ai_latest_experimental`` opt-in is active.
Every semconv behavior is gated on this; ``False`` => legacy output.
"""
return (
OTELSemconvCategory.GEN_AI_LATEST_EXPERIMENTAL
in self.config.semconv_stability_opt_in
)
@staticmethod
def _gen_ai_operation_name(kwargs: dict) -> str:
"""Map a LiteLLM ``call_type`` to spec ``gen_ai.operation.name``.
Substring match (e.g. ``aembedding`` -> ``embeddings``); defaults to
``chat``.
"""
call_type = kwargs.get("call_type", "") or ""
match call_type:
case s if "embedding" in s:
return "embeddings"
case s if "text_completion" in s:
return "text_completion"
case _:
return "chat"
def _set_semconv_request_attributes(
self, span: Span, optional_params: dict
) -> None:
"""Add ``gen_ai.request.*`` span attributes from ``optional_params``.
Covers the sampling params plus the conditionally-required
``stop_sequences`` / ``stream`` / ``choice.count`` per the spec.
"""
for source_key, semconv_key in _SEMCONV_REQUEST_ATTRIBUTES.items():
value = optional_params.get(source_key)
if value is not None:
self.safe_set_attribute(span=span, key=semconv_key, value=value)
stop = optional_params.get("stop")
if stop is not None:
# Spec types this as string[]. safe_set_attribute coerces to a
# primitive, so set the array directly via the span API.
stop_list = stop if isinstance(stop, list) else [stop]
span.set_attribute(
"gen_ai.request.stop_sequences", [str(s) for s in stop_list]
)
# Conditionally required: set only when the request is streaming.
if optional_params.get("stream"):
self.safe_set_attribute(span=span, key="gen_ai.request.stream", value=True)
# Conditionally required per spec ("if available and != 1"). Valid n is
# an int >= 1, so n > 1 is equivalent for conformant input while
# suppressing nonsensical values (0, negative, non-int).
n = optional_params.get("n")
if isinstance(n, int) and n > 1:
self.safe_set_attribute(
span=span, key="gen_ai.request.choice.count", value=n
)
def _set_semconv_cache_token_attributes(
self, span: Span, standard_logging_payload
) -> None:
"""Add ``gen_ai.usage.cache_*.input_tokens`` from the usage object.
No-op when the payload or the usage values are missing/zero.
"""
if not standard_logging_payload:
return
usage = (standard_logging_payload.get("metadata") or {}).get(
"usage_object"
) or {}
for source_key, semconv_key in _SEMCONV_CACHE_TOKEN_ATTRIBUTES.items():
value = usage.get(source_key)
if value:
self.safe_set_attribute(span=span, key=semconv_key, value=value)
def _build_inference_details_attrs(
self, kwargs: dict, response_obj: dict, provider: str
) -> Dict[str, Any]:
"""Build the attribute payload for the inference-details event.
Always includes provider/operation; input/output messages are added
only when content capture is enabled and non-empty. Mixin-internal.
"""
attrs: Dict[str, Any] = {
"event_name": _INFERENCE_DETAILS_EVENT_NAME,
"gen_ai.provider.name": provider,
"gen_ai.operation.name": self._gen_ai_operation_name(kwargs),
}
if not self._capture_in_event():
return attrs
input_messages = self._transform_messages_to_otel_semantic_conventions(
kwargs.get("messages") or []
)
output_messages = self._transform_choices_to_otel_semantic_conventions(
response_obj.get("choices", [])
)
if input_messages:
attrs["gen_ai.input.messages"] = safe_dumps(input_messages)
if output_messages:
attrs["gen_ai.output.messages"] = safe_dumps(output_messages)
return attrs
def _emit_inference_details_event(
self,
kwargs: dict,
response_obj: dict,
provider: str,
otel_logger,
parent_ctx,
) -> None:
"""Emit the consolidated ``gen_ai.client.inference.operation.details``
log event, correlated to the request span via ``parent_ctx``.
Replaces the legacy per-message / per-choice content events.
"""
LogRecord, SeverityNumber = self._otel_log_types()
log_record = LogRecord(
timestamp=self._to_ns(datetime.now()),
trace_id=parent_ctx.trace_id,
span_id=parent_ctx.span_id,
trace_flags=parent_ctx.trace_flags,
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=None,
attributes=self._build_inference_details_attrs(
kwargs, response_obj, provider
),
)
otel_logger.emit(log_record)

104
litellm/integrations/prometheus.py
View file

@ -1226,6 +1226,17 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
# Provider-agnostic fallback: providers like Bedrock and Vertex don't return
# x-ratelimit-remaining-* headers, so the gauges above only fire for OpenAI /
# Anthropic / Azure. When the proxy router has tpm/rpm configured for the
# model_group, derive remaining from configured-limit minus current usage so
# the same metric is populated for any provider.
await self._async_set_router_remaining_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
label_context=label_context,
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
@ -2199,6 +2210,99 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
async def _async_set_router_remaining_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
) -> None:
"""
Populate ``litellm_remaining_tokens_metric`` /
``litellm_remaining_requests_metric`` from the router's internal usage
counters when the upstream provider did not return
``x-ratelimit-remaining-*`` response headers.
OpenAI / Anthropic / Azure return remaining tokens/requests in response
headers, but Bedrock and Vertex AI do not. This fallback computes
``configured_limit - current_usage`` via
``Router.get_remaining_model_group_usage`` so the same gauges are
emitted for every provider when tpm/rpm is configured on the
deployment.
"""
try:
additional_headers = (
standard_logging_payload.get("hidden_params", {}) or {}
).get("additional_headers") or {}
already_have_tokens = (
additional_headers.get("x_ratelimit_remaining_tokens") is not None
)
already_have_requests = (
additional_headers.get("x_ratelimit_remaining_requests") is not None
)
if already_have_tokens and already_have_requests:
return
model_group = standard_logging_payload.get("model_group")
if not model_group:
return
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
llm_router = None
if llm_router is None:
return
try:
remaining_usage = await llm_router.get_remaining_model_group_usage(
model_group
)
except Exception as e:
verbose_logger.exception(
"Prometheus: get_remaining_model_group_usage failed for "
"model_group=%s: %s",
model_group,
e,
)
return
if not remaining_usage:
return
remaining_tokens = remaining_usage.get("x-ratelimit-remaining-tokens")
remaining_requests = remaining_usage.get("x-ratelimit-remaining-requests")
if not already_have_tokens and remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_tokens_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_tokens_metric.labels(**_labels).set(
remaining_tokens
)
if not already_have_requests and remaining_requests is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_requests_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_requests_metric.labels(**_labels).set(
remaining_requests
)
except Exception as e:
verbose_logger.exception(
"Prometheus Error: _async_set_router_remaining_metrics. "
"Exception occured - {}".format(str(e))
)
def set_llm_deployment_success_metrics(
self,
request_kwargs: dict,

247
litellm/integrations/websearch_interception/handler.py
View file

@ -19,12 +19,14 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.tools import (
get_litellm_web_search_tool,
get_litellm_web_search_tool_openai,
is_anthropic_native_web_search_tool,
is_web_search_tool,
is_web_search_tool_chat_completion,
)
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
@ -36,6 +38,16 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
# Key used to flag, on per-request kwargs, that the originating client sent
# an Anthropic-native ``web_search_*`` tool — meaning the final response
# should include ``web_search_tool_result`` content blocks so the client
# (e.g. Claude Desktop's citations panel) can render sources.
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY = "_websearch_interception_emit_native_blocks"
# Key on ``AgenticLoopPlan.metadata`` carrying the list of pre-built
# ``web_search_tool_result`` blocks to inject into the final response.
WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY = "websearch_native_blocks"
class WebSearchInterceptionLogger(CustomLogger):
"""
@ -152,22 +164,55 @@ class WebSearchInterceptionLogger(CustomLogger):
f"(provider={provider_str}, query='{query}')"
)
# Execute search
# Native clients (Claude Desktop / Cowork / Anthropic SDK) make a
# standalone /v1/messages sub-request just for the search, and they
# expect the response in native shape with server_tool_use +
# web_search_tool_result content blocks so the citations panel can
# render. The agentic-loop post-hook never fires on this path because
# there is no model call — emit the native blocks here instead.
native_tool = next(
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
# Execute search — keep the structured SearchResponse so the native
# block can carry per-result url/title/page_age.
try:
search_result_text = await self._execute_search(query)
search_result_text, structured = await self._execute_search(query)
except Exception as e:
verbose_logger.error(
f"WebSearchInterception: Short-circuit search failed: {e}"
)
search_result_text = f"Search failed: {e}"
search_result_text, structured = f"Search failed: {e}", None
content: List[Dict[str, Any]] = []
if native_tool is not None:
tool_use_id = f"srvtoolu_{uuid.uuid4().hex}"
tool_name = native_tool.get("name") or "web_search"
content.append(
{
"type": "server_tool_use",
"id": tool_use_id,
"name": tool_name,
"input": {"query": query},
}
)
content.append(
WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=structured,
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
# Build synthetic Anthropic response
response: Dict[str, Any] = {
"id": f"msg_{str(uuid.uuid4())}",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": search_result_text}],
"content": content,
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 0, "output_tokens": 0},
@ -175,7 +220,8 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, "
f"returning synthetic response ({len(search_result_text)} chars)"
f"returning synthetic response ({len(search_result_text)} chars, "
f"native_blocks={native_tool is not None})"
)
return response
@ -219,6 +265,14 @@ class WebSearchInterceptionLogger(CustomLogger):
"WebSearchInterception: Converting native web_search tools to LiteLLM standard"
)
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (matches async_pre_request_hook). This
# deployment hook fires before async_pre_request_hook on some paths,
# so flagging here ensures the signal isn't lost regardless of order.
if any(is_anthropic_native_web_search_tool(t) for t in tools):
kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
# Convert native/custom web_search tools to LiteLLM standard
converted_tools = []
for tool in tools:
@ -342,6 +396,14 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}"
)
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag
# is read by async_build_agentic_loop_plan; the leading underscore
# prefix ensures it is stripped before the follow-up call kwargs.
if any(is_anthropic_native_web_search_tool(t) for t in tools):
kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
# Convert native web search tools to LiteLLM standard
converted_tools = []
for tool in tools:
@ -591,7 +653,7 @@ class WebSearchInterceptionLogger(CustomLogger):
) -> AgenticLoopPlan:
tool_calls = tools["tool_calls"]
thinking_blocks = tools.get("thinking_blocks", [])
request_patch = await self._build_anthropic_request_patch(
request_patch, structured_results = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
@ -600,12 +662,92 @@ class WebSearchInterceptionLogger(CustomLogger):
logging_obj=logging_obj,
kwargs=kwargs,
)
metadata: Dict[str, Any] = {
"tool_type": "websearch",
"response_format": "anthropic",
}
# If the client request originally carried a native web_search_* tool,
# pre-build the Anthropic-native ``web_search_tool_result`` blocks now
# (while we still have the structured SearchResponse list) and stash
# them on plan metadata for the post-hook to inject.
if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
metadata[WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY] = (
self._build_native_result_blocks(
tool_calls=tool_calls,
structured_results=structured_results,
)
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=request_patch,
metadata={"tool_type": "websearch", "response_format": "anthropic"},
metadata=metadata,
)
async def async_post_agentic_loop_response_hook(
self,
response: Any,
plan: AgenticLoopPlan,
kwargs: Dict,
) -> Any:
"""
Inject Anthropic-native ``web_search_tool_result`` blocks into the
final response when the originating client used a native
``web_search_*`` tool.
See ``WebSearchTransformation.build_web_search_tool_result_block`` for
the block shape. The blocks are prepended to ``response.content`` so
Anthropic-native clients (Claude Desktop, the Anthropic SDK) can
render citations / sources alongside the model's textual reply.
"""
native_blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY)
if not native_blocks:
return response
return self._inject_native_blocks(response, native_blocks)
@staticmethod
def _build_native_result_blocks(
tool_calls: List[Dict],
structured_results: List[Optional[SearchResponse]],
) -> List[Dict[str, Any]]:
"""Build one ``web_search_tool_result`` block per tool_call."""
blocks: List[Dict[str, Any]] = []
for i, tool_call in enumerate(tool_calls):
tool_use_id = tool_call.get("id") or ""
structured = structured_results[i] if i < len(structured_results) else None
blocks.append(
WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=structured,
)
)
return blocks
@staticmethod
def _inject_native_blocks(
response: Any, native_blocks: List[Dict[str, Any]]
) -> Any:
"""Prepend native blocks to response content, dict or object form."""
if not native_blocks:
return response
if isinstance(response, dict):
existing = response.get("content") or []
response["content"] = list(native_blocks) + list(existing)
return response
existing = getattr(response, "content", None) or []
try:
response.content = list(native_blocks) + list(existing)
except (AttributeError, TypeError):
# Object refused write — fall through and leave the response
# untouched rather than crash the request.
verbose_logger.debug(
"WebSearchInterception: could not inject native blocks into "
f"response of type {type(response).__name__}"
)
return response
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
@ -733,7 +875,7 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs: Dict,
) -> Any:
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch = await self._build_anthropic_request_patch(
request_patch, structured_results = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
@ -755,7 +897,7 @@ class WebSearchInterceptionLogger(CustomLogger):
if max_tokens is None:
max_tokens = cast(int, kwargs.get("max_tokens", 1024))
return await anthropic_messages.acreate(
response = await anthropic_messages.acreate(
max_tokens=max_tokens,
messages=request_patch.messages,
model=request_patch.model or model,
@ -763,6 +905,18 @@ class WebSearchInterceptionLogger(CustomLogger):
**request_patch.kwargs,
)
# Legacy path: the new path goes through the typed plan + core
# dispatcher which runs the post-hook automatically. Mirror the
# native-block injection here so both paths behave identically.
if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
native_blocks = self._build_native_result_blocks(
tool_calls=tool_calls,
structured_results=structured_results,
)
response = self._inject_native_blocks(response, native_blocks)
return response
async def _build_anthropic_request_patch(
self,
model: str,
@ -772,8 +926,16 @@ class WebSearchInterceptionLogger(CustomLogger):
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
kwargs: Dict,
) -> AgenticLoopRequestPatch:
"""Execute litellm.search() and build follow-up request patch."""
) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]:
"""
Execute litellm.search() and build follow-up request patch.
Returns the patch alongside the parallel list of structured
``SearchResponse`` objects (one per tool_call, ``None`` when the
search failed or the tool_call had no query). The caller uses these
to optionally build Anthropic-native ``web_search_tool_result``
content blocks for the final response.
"""
# Extract search queries from tool_use blocks
search_tasks = []
@ -797,23 +959,38 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
# Handle any exceptions in search results
# Split the gathered (text, structured) tuples into two parallel lists.
# The text list feeds the follow-up model call; the structured list
# is returned to the caller for native-block emission.
final_search_results: List[str] = []
structured_results: List[Optional[SearchResponse]] = []
for i, result in enumerate(search_results):
if isinstance(result, Exception):
verbose_logger.error(
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
)
final_search_results.append(f"Search failed: {str(result)}")
elif isinstance(result, str):
# Explicitly cast to str for type checker
final_search_results.append(cast(str, result))
structured_results.append(None)
elif isinstance(result, tuple) and len(result) == 2:
text_value, structured_value = result
final_search_results.append(
cast(str, text_value)
if isinstance(text_value, str)
else str(text_value)
)
structured_results.append(
structured_value
if isinstance(structured_value, SearchResponse)
else None
)
else:
# Should never happen, but handle for type safety
# Defensive: legacy callers / unexpected shape — preserve text,
# drop structure.
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
)
final_search_results.append(str(result))
structured_results.append(None)
# Build assistant and user messages using transformation
assistant_message, user_message = WebSearchTransformation.transform_response(
@ -859,16 +1036,26 @@ class WebSearchInterceptionLogger(CustomLogger):
len(follow_up_messages),
len(final_search_results),
)
return AgenticLoopRequestPatch(
patch = AgenticLoopRequestPatch(
model=full_model_name,
messages=follow_up_messages,
max_tokens=max_tokens,
optional_params=optional_params_without_max_tokens,
kwargs=kwargs_for_followup,
)
return patch, structured_results
async def _execute_search(self, query: str) -> str:
"""Execute a single web search using router's search tools"""
async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
"""
Execute a single web search using router's search tools.
Returns both the formatted text (fed back to the model in the follow-up
call) and the structured ``SearchResponse`` (preserved so callers can
build Anthropic-native ``web_search_tool_result`` blocks for clients
that requested a native ``web_search_*`` tool). The structured value
is None on the failure path so callers can still emit an empty result
block rather than dropping the search entirely.
"""
try:
# Import router from proxy_server
try:
@ -934,7 +1121,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars"
)
return search_result_text
return search_result_text, result
except Exception as e:
verbose_logger.error(
f"WebSearchInterception: Search failed for '{query}': {str(e)}"
@ -1015,7 +1202,8 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
# Handle any exceptions in search results
# Chat-completion path only needs text — OpenAI tool_result format
# has no equivalent of Anthropic's web_search_tool_result block.
final_search_results: List[str] = []
for i, result in enumerate(search_results):
if isinstance(result, Exception):
@ -1023,8 +1211,13 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
)
final_search_results.append(f"Search failed: {str(result)}")
elif isinstance(result, str):
final_search_results.append(cast(str, result))
elif isinstance(result, tuple) and len(result) == 2:
text_value, _ = result
final_search_results.append(
cast(str, text_value)
if isinstance(text_value, str)
else str(text_value)
)
else:
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
@ -1112,9 +1305,11 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs=kwargs_for_followup,
)
async def _create_empty_search_result(self) -> str:
async def _create_empty_search_result(
self,
) -> Tuple[str, Optional[SearchResponse]]:
"""Create an empty search result for tool calls without queries"""
return "No search query provided"
return "No search query provided", None
@staticmethod
def initialize_from_proxy_config(

47
litellm/integrations/websearch_interception/tools.py
View file

@ -126,6 +126,27 @@ def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
return False
def is_anthropic_native_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is an Anthropic-native ``web_search_*`` tool.
Native clients (Anthropic SDK, Claude Desktop, Anthropic Console) send
tools like ``{"type": "web_search_20250305", "name": "web_search"}`` and
expect the response to contain ``web_search_tool_result`` content blocks
so that citations can be rendered. This helper identifies that contract
so the agentic loop can emit native-format blocks for those clients
without affecting clients that send the LiteLLM standard tool.
Returns False for the LiteLLM standard tool (``litellm_web_search``),
the OpenAI-shaped variant, the bare ``WebSearch`` legacy name, and the
bare ``web_search`` name (Claude Code style).
"""
tool_type = tool.get("type", "")
if not isinstance(tool_type, str):
return False
return tool_type.startswith("web_search_") and tool_type != "function"
def is_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is a web search tool (native or LiteLLM standard).
@ -135,7 +156,22 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
- OpenAI format: type == "function" with function.name == "litellm_web_search"
- Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
- Claude Code: name == "web_search" with a type field
- Custom: name == "WebSearch" (legacy format)
- Custom: name == "WebSearch" (legacy interception marker — only matched
when input_schema is absent; see note below)
Note on the legacy ``WebSearch`` name:
Clients like Claude Desktop / Cowork ship a *client-side* tool called
``WebSearch`` (a fully-formed Anthropic client tool with its own
``input_schema``) that they handle themselves. Treating that as our
interception marker hijacks it server-side and the client's own tool
handler never fires — which means Cowork's separate native
``web_search_20250305`` sub-request (where citation data actually
flows) never gets made.
Real Anthropic client tools always carry an ``input_schema`` (the API
rejects them otherwise), so a bare ``{name: "WebSearch"}`` with no
schema is the only thing that could be a legacy interception marker.
Gate the match on schema absence to keep both groups working.
Args:
tool: Tool dictionary to check
@ -152,6 +188,10 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
True
>>> is_web_search_tool({"name": "calculator"})
False
>>> is_web_search_tool({"name": "WebSearch"}) # legacy interception marker
True
>>> is_web_search_tool({"name": "WebSearch", "input_schema": {"type": "object"}}) # Cowork client tool
False
"""
tool_name = tool.get("name", "")
tool_type = tool.get("type", "")
@ -175,8 +215,9 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
if tool_name == "web_search" and tool_type:
return True
# Check for legacy WebSearch format
if tool_name == "WebSearch":
# Legacy "WebSearch" interception marker — only when no schema is
# present, so real client-side WebSearch tools (Cowork) pass through.
if tool_name == "WebSearch" and "input_schema" not in tool:
return True
return False

66
litellm/integrations/websearch_interception/transformation.py
View file

@ -100,11 +100,14 @@ class WebSearchTransformation:
block_id = getattr(block, "id", None)
block_input = getattr(block, "input", {})
# Check for LiteLLM standard or legacy web search tools
# Handles: litellm_web_search, WebSearch, web_search
# Detect tool_use blocks that came from interception. After
# pre-request conversion the model always sees
# ``litellm_web_search``; the bare ``web_search`` entry handles
# callers that bypass our pre-request hooks (e.g. direct
# litellm.acompletion). "WebSearch" is intentionally omitted —
# see is_web_search_tool for the Cowork rationale.
if block_type == "tool_use" and block_name in (
LITELLM_WEB_SEARCH_TOOL_NAME,
"WebSearch",
"web_search",
):
# Convert to dict for easier handling
@ -190,10 +193,12 @@ class WebSearchTransformation:
getattr(function, "arguments", None) if function else None
)
# Check for LiteLLM standard or legacy web search tools
# Detect function-style web search tool_calls. ``WebSearch`` is
# intentionally omitted — see is_web_search_tool for the Cowork
# rationale (clients ship their own client-side ``WebSearch`` and
# we must not hijack it).
if tool_type == "function" and function_name in (
LITELLM_WEB_SEARCH_TOOL_NAME,
"WebSearch",
"web_search",
):
# Parse arguments (might be JSON string)
@ -350,6 +355,57 @@ class WebSearchTransformation:
return assistant_message, tool_messages
@staticmethod
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: Optional[SearchResponse],
) -> Dict[str, Any]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
Native Anthropic clients (Claude Desktop, the Anthropic SDK, the
Anthropic Console) expect search-tool results to be returned as
structured ``web_search_tool_result`` blocks so that citations and
source links can be rendered. The agentic loop currently feeds the
model a flat text blob in the follow-up call (which is correct — the
model needs readable evidence). This helper produces the *additional*
block that should accompany the model's text reply when the original
request used a native ``web_search_*`` tool.
Spec reference:
https://docs.anthropic.com/en/api/web-search-tool
Args:
tool_use_id: The ``tool_use_id`` the model emitted on the first
turn. Must match exactly so the client can pair the result
with its tool_use block.
search_response: Structured ``SearchResponse`` from
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
"""
items: List[Dict[str, Any]] = []
if search_response is not None:
results = getattr(search_response, "results", None) or []
for r in results:
url = getattr(r, "url", "") or ""
title = getattr(r, "title", "") or ""
page_age = getattr(r, "date", None) or getattr(r, "last_updated", None)
items.append(
{
"type": "web_search_result",
"url": url,
"title": title,
"page_age": page_age,
"encrypted_content": "",
}
)
return {
"type": "web_search_tool_result",
"tool_use_id": tool_use_id,
"content": items,
}
@staticmethod
def format_search_response(result: SearchResponse) -> str:
"""

45
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -53,8 +53,19 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
# Raw bytes
filename = "audio.wav"
file_content = bytes(audio_file)
elif isinstance(audio_file, (str, os.PathLike)):
# File path or PathLike
elif isinstance(audio_file, str):
# Bare strings are rejected — see extract_file_data for the same
# rationale: in a proxy request handler the string is
# attacker-controlled, and opening it as a path is an arbitrary
# file read.
raise ValueError(
"process_audio_file does not accept bare str inputs. Pass bytes, "
"an open file handle, a (filename, content) tuple, or a "
"pathlib.Path."
)
elif isinstance(audio_file, os.PathLike):
# File path or PathLike — PathLike is a Python-level type that
# HTTP form values can't fabricate.
file_path = str(audio_file)
with open(file_path, "rb") as f:
file_content = f.read()
@ -66,8 +77,14 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
content = audio_file[1]
if isinstance(content, (bytes, bytearray)):
file_content = bytes(content)
elif isinstance(content, (str, os.PathLike)):
# File path or PathLike
elif isinstance(content, str):
raise ValueError(
"process_audio_file does not accept bare str tuple "
"contents. Pass bytes, an open file handle, or a "
"pathlib.Path."
)
elif isinstance(content, os.PathLike):
# PathLike: SDK convenience for local-file uploads.
with open(str(content), "rb") as f:
file_content = f.read()
elif hasattr(content, "read"):
@ -149,7 +166,14 @@ def get_audio_file_content_hash(file_obj: FileTypes) -> str:
try:
if isinstance(file_content_obj, (bytes, bytearray)):
file_content = bytes(file_content_obj)
elif isinstance(file_content_obj, (str, os.PathLike)):
elif isinstance(file_content_obj, str):
# Bare strings are not treated as file paths in this helper —
# the cache-key path is reached from request handlers where the
# value is attacker-controlled. Fall back to hashing the string
# itself rather than opening it.
fallback_filename = file_content_obj
file_content = None
elif isinstance(file_content_obj, os.PathLike):
try:
with open(str(file_content_obj), "rb") as f:
file_content = f.read()
@ -229,8 +253,15 @@ def calculate_request_duration(file: FileTypes) -> Optional[float]:
if isinstance(file, (bytes, bytearray)):
# Raw bytes
file_content = bytes(file)
elif isinstance(file, (str, os.PathLike)):
# File path
elif isinstance(file, str):
# Bare strings are rejected — see extract_file_data.
raise ValueError(
"calculate_request_duration does not accept bare str inputs. "
"Pass bytes, an open file handle, a (filename, content) "
"tuple, or a pathlib.Path."
)
elif isinstance(file, os.PathLike):
# File path (PathLike): SDK convenience.
with open(str(file), "rb") as f:
file_content = f.read()
elif isinstance(file, tuple):

12
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1050,6 +1050,16 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.model_call_details["api_call_start_time"] = datetime.datetime.now()
# Set-once first provider-handoff instant. api_call_start_time
# is overwritten on every retry, so it can't measure one-time
# preprocessing; pinning the first attempt excludes retry loops
# + backoff. Logging object only — must NOT go into
# litellm_params["metadata"] (caller request metadata, typed
# Dict[str, str], echoed downstream; a datetime breaks it).
if self.model_call_details.get("first_api_call_start_time") is None:
self.model_call_details["first_api_call_start_time"] = (
self.model_call_details["api_call_start_time"]
)
# Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
callbacks = litellm.input_callback + (self.dynamic_input_callbacks or [])
for callback in callbacks:
@ -1212,7 +1222,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Log the exact result from the LLM API, for streaming - log the type of response received
litellm.error_logs["POST_CALL"] = locals()
if isinstance(original_response, dict):
original_response = json.dumps(original_response)
original_response = json.dumps(original_response, default=str)
try:
self.model_call_details["input"] = input
self.model_call_details["api_key"] = api_key

25
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -755,14 +755,25 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
else:
file_content = file_data
# Convert content to bytes
if isinstance(file_content, (str, PathLike)):
# If it's a path, open and read the file
# Extract filename from path if not already set
if isinstance(file_content, str):
# Bare string inputs are rejected: when this helper runs in a proxy
# request handler the string came from an attacker-controlled form
# field, and opening it as a path is an arbitrary file read on the
# proxy host. SDK callers who want to upload from a path should
# either pass a pathlib.Path (a PathLike instance — see the branch
# below) or open the file themselves and pass the handle / bytes.
raise ValueError(
"extract_file_data does not accept bare str inputs. Pass bytes, "
"an open file handle, a (filename, content) tuple, or a "
"pathlib.Path. To upload a local file from a path, call "
"open(path, 'rb') yourself."
)
if isinstance(file_content, PathLike):
# PathLike (pathlib.Path) is a Python-level type that HTTP form
# values can't fabricate. Treat as a local file path for SDK
# convenience.
if filename is None:
if isinstance(file_content, PathLike):
filename = Path(file_content).name
else:
filename = Path(str(file_content)).name
filename = Path(file_content).name
with open(file_content, "rb") as f:
content = f.read()
elif isinstance(file_content, io.IOBase):

5
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4977,8 +4977,9 @@ class BedrockConverseMessagesProcessor:
)
if reasoning_text and not reasoning_text.get("signature"):
reasoning_text_text = reasoning_text["text"]
assistants_part = BedrockContentBlock(text=reasoning_text_text)
assistant_parts.append(assistants_part)
if reasoning_text_text.strip():
assistants_part = BedrockContentBlock(text=reasoning_text_text)
assistant_parts.append(assistants_part)
else:
filtered_thinking_blocks.append(block)
if len(filtered_thinking_blocks) > 0:

43
litellm/llms/anthropic/common_utils.py
View file

@ -832,6 +832,49 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
def strip_empty_text_blocks_from_anthropic_messages(
messages: List[Any],
) -> List[Any]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks removed.
Anthropic's API rejects requests containing such blocks with
``"messages: text content blocks must be non-empty"``, but assistant
messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
back as conversation history, which then causes the next request to 400
on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already
handles this in ``anthropic_messages_pt``; this helper provides the
equivalent guarantee for the native Anthropic Messages path.
Messages whose content is a list and becomes empty after stripping are
omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
The caller's list and its content blocks are never mutated; modified
messages are returned as shallow copies with a fresh content list.
"""
out: List[Any] = []
for m in messages:
if not isinstance(m, dict) or not isinstance(m.get("content"), list):
out.append(m)
continue
content = m["content"]
filtered = [b for b in content if not _is_empty_text_block(b)]
if len(filtered) == len(content):
out.append(m)
elif filtered:
out.append({**m, "content": filtered})
return out
def _is_empty_text_block(block: Any) -> bool:
if not isinstance(block, dict) or block.get("type") != "text":
return False
text = block.get("text")
return not isinstance(text, str) or not text.strip()
def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
openai_headers = {}
if "anthropic-ratelimit-requests-limit" in headers:

11
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -1299,9 +1299,18 @@ class LiteLLMAnthropicMessagesAdapter:
else truncated_name
)
# Strip Gemini thought-signature suffix from id (mirrors streaming
# path below); base64 chars (+ / =) violate Anthropic's
# `^[a-zA-Z0-9_-]+$` tool_use.id pattern when replayed.
raw_id = tool_call.id or ""
base_id = (
raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
if THOUGHT_SIGNATURE_SEPARATOR in raw_id
else raw_id
)
tool_use_block = AnthropicResponseContentBlockToolUse(
type="tool_use",
id=tool_call.id,
id=base_id,
name=original_name,
input=parse_tool_call_arguments(
tool_call.function.arguments,

22
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,9 @@ from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, c
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
strip_empty_text_blocks_from_anthropic_messages,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -188,8 +191,20 @@ async def anthropic_messages(
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
Async: Make llm api request in Anthropic /messages API spec
Async: Make llm api request in Anthropic /messages API spec.
Runs the empty-text-block sanitizer before any backend dispatch.
"""
# Anthropic's API rejects requests containing empty / whitespace-only
# text content blocks with "messages: text content blocks must be
# non-empty". Multi-turn tool-use clients (e.g. Claude Code) routinely
# loop assistant responses that contain {"type": "text", "text": ""}
# alongside tool_use blocks back as conversation history, which then
# causes the next /v1/messages call to 400. /v1/chat/completions
# already handles this in anthropic_messages_pt; sanitize the native
# Anthropic Messages path here for the same guarantee. See #22930.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
original_stream = stream or kwargs.get(
"_websearch_interception_converted_stream", False
)
@ -336,6 +351,11 @@ def anthropic_messages_handler(
"""
from litellm.types.utils import LlmProviders
# Sanitize empty text blocks here too so the sync entry point
# (litellm.messages.create -> anthropic_messages_handler) gets the same
# protection as the async wrapper. Idempotent when called twice.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
metadata = validate_anthropic_api_metadata(metadata)
local_vars = locals()

48
litellm/llms/azure/image_edit/transformation.py
View file

@ -3,8 +3,10 @@ from typing import Optional, cast
import httpx
import litellm
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import _add_path_to_api_base
@ -30,20 +32,42 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
litellm_params: Optional[dict] = None,
api_base: Optional[str] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
"""
Validate Azure environment and set up authentication headers.
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
Delegates to ``BaseAzureLLM._base_validate_azure_environment`` so the
Azure image-edit route uses the same auth resolution as every other
Azure provider (videos, vector_stores, responses, containers, ...):
- prefers the Azure-style ``api-key`` header when an API key is available
- falls back to ``Authorization: Bearer <azure_ad_token>`` only when AAD
auth is configured
The previous implementation unconditionally set
``Authorization: Bearer <api_key>``, which is correct for OpenAI direct
but not for Azure OpenAI / API Management gateways that expect the
``api-key`` header. Subscription-key-based deployments (e.g., behind
Azure APIM) responded with ``401 "Access denied due to missing
subscription key"``.
API-key precedence (matches ``AzureVideosConfig``):
- ``litellm_params["api_key"]`` is the source of truth.
- The positional ``api_key`` kwarg only fills in when
``litellm_params["api_key"]`` is empty.
- This is a deliberate change from the old ``or`` chain (where the
positional ``api_key`` argument won) so behavior matches every other
Azure ``validate_environment`` implementation. In production the only
caller (``llm_http_handler.image_edit``) sources both values from
the same ``litellm_params.api_key``, so the precedence only matters
for direct callers of this method.
"""
params = GenericLiteLLMParams(**(litellm_params or {}))
if api_key is not None and params.api_key is None:
params.api_key = api_key
return BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=params
)
return headers
def get_complete_url(
self,

8
litellm/llms/bedrock/base_aws_llm.py
View file

@ -1428,7 +1428,13 @@ class BaseAWSLLM:
def _sign_request(
self,
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
service_name: Literal[
"bedrock",
"sagemaker",
"bedrock-agentcore",
"s3vectors",
"aws-external-anthropic",
],
headers: dict,
optional_params: dict,
request_data: dict,

4
litellm/llms/bedrock/chat/mantle/transformation.py
View file

@ -21,7 +21,9 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
MANTLE_ENDPOINT_TEMPLATE = (
"https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
)
class AmazonMantleConfig(AmazonAnthropicClaudeConfig):

8
litellm/llms/bedrock/claude_platform/__init__.py Normal file
View file

@ -0,0 +1,8 @@
from .transformation import (
BedrockClaudePlatformConfig,
)
from .messages_transformation import (
BedrockClaudePlatformMessagesConfig,
)
__all__ = ["BedrockClaudePlatformConfig", "BedrockClaudePlatformMessagesConfig"]

107
litellm/llms/bedrock/claude_platform/common_utils.py Normal file
View file

@ -0,0 +1,107 @@
from typing import Literal, Optional, Tuple
import litellm
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.secret_managers.main import get_secret_str
CLAUDE_PLATFORM_SERVICE_NAME: Literal["aws-external-anthropic"] = (
"aws-external-anthropic"
)
CLAUDE_PLATFORM_BEDROCK_ROUTE = "claude_platform/"
def strip_claude_platform_route(model: str) -> str:
if model.startswith(CLAUDE_PLATFORM_BEDROCK_ROUTE):
return model.replace(CLAUDE_PLATFORM_BEDROCK_ROUTE, "", 1)
return model
class BedrockClaudePlatformMixin(BaseAWSLLM):
@staticmethod
def _get_workspace_id(optional_params: dict, litellm_params: dict) -> Optional[str]:
workspace_id = (
optional_params.get("workspace_id")
or litellm_params.get("workspace_id")
or optional_params.get("aws_workspace_id")
or litellm_params.get("aws_workspace_id")
or optional_params.get("anthropic-workspace-id")
or litellm_params.get("anthropic-workspace-id")
)
if workspace_id is None:
workspace_id = optional_params.get(
"anthropic_workspace_id"
) or litellm_params.get("anthropic_workspace_id")
if workspace_id is not None:
return str(workspace_id)
return get_secret_str("ANTHROPIC_AWS_WORKSPACE_ID") or get_secret_str(
"ANTHROPIC_WORKSPACE_ID"
)
def _get_required_aws_region_name(self, optional_params: dict) -> str:
aws_region_name = (
optional_params.get("aws_region_name")
or get_secret_str("AWS_REGION_NAME")
or get_secret_str("AWS_REGION")
or get_secret_str("AWS_DEFAULT_REGION")
)
if aws_region_name is None:
raise litellm.AuthenticationError(
message=(
"Missing AWS region for Claude Platform on AWS. Pass "
"`aws_region_name` or set a standard AWS region environment value."
),
llm_provider="bedrock",
model="",
)
self._validate_aws_region_name(str(aws_region_name))
return str(aws_region_name)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
api_base = (
api_base
or litellm.api_base
or get_secret_str("ANTHROPIC_AWS_BASE_URL")
or get_secret_str("ANTHROPIC_AWS_API_BASE")
)
if api_base is None:
aws_region_name = self._get_required_aws_region_name(optional_params)
api_base = (
f"https://{CLAUDE_PLATFORM_SERVICE_NAME}.{aws_region_name}.api.aws"
)
if not api_base.endswith("/v1/messages"):
api_base = f"{api_base.rstrip('/')}/v1/messages"
return api_base
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
) -> Tuple[dict, Optional[bytes]]:
if api_key or get_secret_str("ANTHROPIC_AWS_API_KEY"):
return headers, None
return self._sign_request(
service_name=CLAUDE_PLATFORM_SERVICE_NAME,
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
model=model,
stream=stream,
fake_stream=fake_stream,
)

71
litellm/llms/bedrock/claude_platform/messages_transformation.py Normal file
View file

@ -0,0 +1,71 @@
from typing import Any, Dict, List, Optional, Tuple
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from .common_utils import BedrockClaudePlatformMixin, strip_claude_platform_route
class BedrockClaudePlatformMessagesConfig(
BedrockClaudePlatformMixin, AnthropicMessagesConfig
):
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: List[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Tuple[dict, Optional[str]]:
workspace_id = self._get_workspace_id(optional_params, litellm_params)
if workspace_id is None:
raise litellm.AuthenticationError(
message=(
"Missing workspace ID for Claude Platform on AWS. Pass "
"`workspace_id` or configure the provider workspace setting."
),
llm_provider="bedrock",
model=model,
)
resolved_api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY")
headers = {
**headers,
"anthropic-version": headers.get(
"anthropic-version", DEFAULT_ANTHROPIC_API_VERSION
),
"content-type": headers.get("content-type", "application/json"),
"anthropic-workspace-id": workspace_id,
}
if resolved_api_key and "x-api-key" not in headers:
headers["x-api-key"] = resolved_api_key
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
)
return headers, api_base
def transform_anthropic_messages_request(
self,
model: str,
messages: List[Dict],
anthropic_messages_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
return super().transform_anthropic_messages_request(
model=strip_claude_platform_route(model),
messages=messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)

94
litellm/llms/bedrock/claude_platform/transformation.py Normal file
View file

@ -0,0 +1,94 @@
from typing import Any, Dict, List, Optional
import litellm
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from .common_utils import BedrockClaudePlatformMixin
class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig):
"""
Bedrock Claude Platform uses Anthropic's Messages API with AWS gateway auth.
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Dict:
workspace_id = self._get_workspace_id(optional_params, litellm_params)
if workspace_id is None:
raise litellm.AuthenticationError(
message=(
"Missing workspace ID for Claude Platform on AWS. Pass "
"`workspace_id` or configure the provider workspace setting."
),
llm_provider="bedrock",
model=model,
)
api_key = api_key or get_secret_str("ANTHROPIC_AWS_API_KEY")
anthropic_headers = self.get_anthropic_headers(
api_key=api_key,
auth_token=None,
computer_tool_used=self.is_computer_tool_used(
tools=optional_params.get("tools")
),
prompt_caching_set=self.is_cache_control_set(messages=messages),
pdf_used=self.is_pdf_used(messages=messages),
file_id_used=self.is_file_id_used(messages=messages),
mcp_server_used=self.is_mcp_server_used(
mcp_servers=optional_params.get("mcp_servers")
),
web_search_tool_used=self.is_web_search_tool_used(
tools=optional_params.get("tools")
),
tool_search_used=self.is_tool_search_used(
tools=optional_params.get("tools")
),
programmatic_tool_calling_used=self.is_programmatic_tool_calling_used(
tools=optional_params.get("tools")
),
input_examples_used=self.is_input_examples_used(
tools=optional_params.get("tools")
),
effort_used=self.is_effort_used(
optional_params=optional_params, model=model
),
user_anthropic_beta_headers=self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
),
code_execution_tool_used=self.is_code_execution_tool_used(
tools=optional_params.get("tools")
),
container_with_skills_used=self.is_container_with_skills_used(
optional_params=optional_params
),
)
anthropic_headers["anthropic-workspace-id"] = workspace_id
return {**headers, **anthropic_headers}
def get_model_response_iterator(
self,
streaming_response: Any,
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> Any:
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
return ModelResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=bool(json_mode),
)

43
litellm/llms/bedrock/common_utils.py
View file

@ -692,6 +692,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
) -> Literal[
"converse",
"invoke",
"claude_platform",
"converse_like",
"agent",
"agentcore",
@ -706,6 +707,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
str,
Literal[
"invoke",
"claude_platform",
"converse_like",
"converse",
"agent",
@ -716,6 +718,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
],
] = {
"invoke/": "invoke",
"claude_platform/": "claude_platform",
"converse_like/": "converse_like",
"converse/": "converse",
"agent/": "agent",
@ -753,6 +756,36 @@ class BedrockModelInfo(BaseLLMModelInfo):
"""
return "converse/" in model
@staticmethod
def _explicit_claude_platform_route(model: str) -> bool:
"""
Check if the model is an explicit Claude Platform on AWS route.
"""
return "claude_platform/" in model
@staticmethod
def get_claude_platform_model(model: str) -> str:
"""
Strip the Claude Platform route prefix from a Bedrock model name.
"""
return model.replace("claude_platform/", "", 1)
@staticmethod
def map_claude_platform_auth_params(
passed_params: dict, optional_params: dict
) -> dict:
"""
Map Claude Platform route auth params that are not OpenAI request params.
"""
for key in (
"workspace_id",
"aws_workspace_id",
"anthropic_workspace_id",
):
if key in passed_params:
optional_params[key] = passed_params[key]
return optional_params
@staticmethod
def _explicit_invoke_route(model: str) -> bool:
"""
@ -815,6 +848,12 @@ class BedrockModelInfo(BaseLLMModelInfo):
All other routes should return None since they will go through litellm.completion
"""
#########################################################
# Claude Platform route uses Anthropic Messages API via the AWS gateway.
#########################################################
if BedrockModelInfo._explicit_claude_platform_route(model):
return litellm.BedrockClaudePlatformMessagesConfig()
#########################################################
# Converse routes should go through litellm.completion()
if BedrockModelInfo._explicit_converse_route(model):
@ -860,7 +899,9 @@ def get_bedrock_chat_config(model: str):
base_model = BedrockModelInfo.get_base_model(model)
# Handle explicit routes first
if bedrock_route == "converse" or bedrock_route == "converse_like":
if bedrock_route == "claude_platform":
return litellm.BedrockClaudePlatformConfig()
elif bedrock_route == "converse" or bedrock_route == "converse_like":
return litellm.AmazonConverseConfig()
elif bedrock_route == "openai":
return litellm.AmazonBedrockOpenAIConfig()

4
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -20,7 +20,9 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
MANTLE_ENDPOINT_TEMPLATE = (
"https://bedrock-mantle.{region}.api.aws/anthropic/v1/messages"
)
class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):

7
litellm/llms/custom_httpx/http_handler.py
View file

@ -485,11 +485,16 @@ class MaskedHTTPStatusError(httpx.HTTPStatusError):
if k.lower() not in ("content-encoding", "content-length")
}
try:
request_content = original_error.request.content
except httpx.RequestNotRead:
request_content = b""
masked_request = httpx.Request(
method=original_error.request.method,
url=masked_url,
headers=original_error.request.headers,
content=original_error.request.content,
content=request_content,
)
super().__init__(

21
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -4634,6 +4634,7 @@ class BaseLLMHTTPHandler:
fingerprints: List[str],
fingerprint: str,
stream: bool = False,
callback: Optional[Any] = None,
) -> Any:
from litellm.anthropic_interface import messages as anthropic_messages
@ -4675,7 +4676,7 @@ class BaseLLMHTTPHandler:
kwargs_for_followup["max_agentic_loops"] = max_loops
kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
return await anthropic_messages.acreate(
response = await anthropic_messages.acreate(
**{
"max_tokens": max_tokens,
"messages": patch.messages,
@ -4686,6 +4687,23 @@ class BaseLLMHTTPHandler:
}
)
if callback is not None:
try:
response = await callback.async_post_agentic_loop_response_hook(
response=response, plan=plan, kwargs=kwargs
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.exception(
"LiteLLM.AgenticHookError: Exception in "
"async_post_agentic_loop_response_hook [call_id=%s model=%s]: %s",
_call_id,
model,
str(e),
)
return response
async def _execute_chat_completion_agentic_plan(
self,
plan: AgenticLoopPlan,
@ -4869,6 +4887,7 @@ class BaseLLMHTTPHandler:
fingerprints=fingerprints,
fingerprint=fingerprint,
stream=stream,
callback=callback,
)
except Exception as e:
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")

11
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -241,10 +241,13 @@ class FireworksAIConfig(OpenAIGPTConfig):
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
filter_value_from_dict(cast(dict, message), "cache_control")
# Remove fields not permitted by FireworksAI that may cause:
# "Not permitted, field: 'messages[n].provider_specific_fields'"
if isinstance(message, dict) and "provider_specific_fields" in message:
cast(dict, message).pop("provider_specific_fields", None)
# Remove fields not permitted by FireworksAI (additionalProperties: false
# on their ChatMessage schema) that may cause:
# "Extra inputs are not permitted, field: 'messages[n].<field>'"
if isinstance(message, dict):
m = cast(dict, message)
m.pop("provider_specific_fields", None)
m.pop("thinking_blocks", None)
return messages

53
litellm/llms/gemini/google_genai/transformation.py
View file

@ -2,6 +2,7 @@
Transformation for Calling Google models in their native format.
"""
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
import httpx
@ -11,6 +12,10 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.base_llm.google_genai.transformation import (
BaseGoogleGenAIGenerateContentConfig,
)
from litellm.llms.vertex_ai.common_utils import (
_build_vertex_schema,
supports_response_json_schema,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.types.router import GenericLiteLLMParams
@ -302,6 +307,52 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
litellm_params=litellm_params,
)
@staticmethod
def _normalize_response_schema(
generate_content_config_dict: Dict, model: str
) -> None:
schema_key = next(
(
k
for k in ("responseSchema", "response_schema")
if k in generate_content_config_dict
),
None,
)
json_schema_key = next(
(
k
for k in ("responseJsonSchema", "response_json_schema")
if k in generate_content_config_dict
),
None,
)
if schema_key is None:
return
value = generate_content_config_dict[schema_key]
if not isinstance(value, dict):
return
if supports_response_json_schema(model):
if json_schema_key is not None:
generate_content_config_dict.pop(schema_key)
return
generate_content_config_dict.pop(schema_key)
new_json_schema_key = (
"response_json_schema"
if schema_key == "response_schema"
else "responseJsonSchema"
)
generate_content_config_dict[new_json_schema_key] = value
else:
if json_schema_key is not None:
generate_content_config_dict.pop(json_schema_key)
generate_content_config_dict[schema_key] = _build_vertex_schema(
parameters=deepcopy(value), add_property_ordering=True
)
def transform_generate_content_request(
self,
model: str,
@ -315,6 +366,8 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
GenerateContentRequestDict,
)
self._normalize_response_schema(generate_content_config_dict, model)
typed_generate_content_request = GenerateContentRequestDict(
model=model,
contents=contents,

4
litellm/llms/ollama/chat/transformation.py
View file

@ -507,10 +507,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
# PROCESS REASONING CONTENT
reasoning_content: Optional[str] = None
content: Optional[str] = None
if chunk["message"].get("thinking") is not None:
if chunk["message"].get("thinking"):
reasoning_content = chunk["message"].get("thinking")
self.started_reasoning_content = True
elif chunk["message"].get("content") is not None:
if chunk["message"].get("content"):
if (
self.started_reasoning_content
and not self.finished_reasoning_content

2
litellm/llms/ollama/common_utils.py
View file

@ -108,7 +108,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
continue
nm = entry.get("name") or entry.get("model")
if isinstance(nm, str):
names.add(nm)
names.add(nm if nm.startswith("ollama/") else f"ollama/{nm}")
except Exception as e:
verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}")
# If tags endpoint fails, fall back to static list

3
litellm/llms/vertex_ai/google_genai/transformation.py
View file

@ -79,6 +79,9 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
Transform the generate content request for Vertex AI.
Since Vertex AI natively supports Google GenAI format, we can pass most fields directly.
"""
if generate_content_config_dict:
self._normalize_response_schema(generate_content_config_dict, model)
# Build the request in Google GenAI format that Vertex AI expects
result = {
"model": model,

34
litellm/main.py
View file

@ -1528,7 +1528,11 @@ def completion( # type: ignore # noqa: PLR0915
"logit_bias": logit_bias,
"user": user,
# params to identify the model
"model": model,
"model": (
model_info.get("base_model")
if isinstance(model_info, dict) and model_info.get("base_model")
else model
),
"custom_llm_provider": custom_llm_provider,
"response_format": response_format,
"seed": seed,
@ -3858,7 +3862,33 @@ def completion( # type: ignore # noqa: PLR0915
)
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
if bedrock_route == "converse":
if bedrock_route == "claude_platform":
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model,
provider=LlmProviders.BEDROCK,
)
model = BedrockModelInfo.get_claude_platform_model(model)
response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="bedrock",
timeout=timeout,
headers=headers,
encoding=_get_encoding(),
api_key=api_key,
logging_obj=logging,
client=client,
provider_config=provider_config,
)
return response
elif bedrock_route == "converse":
model = model.replace("converse/", "")
response = bedrock_converse_chat_completion.completion(
model=model,

431
litellm/model_prices_and_context_window_backup.json
View file

@ -2112,6 +2112,380 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"azure_ai/gpt-5.4": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-2026-03-05": {
"cache_read_input_token_cost": 2.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_priority": 3e-05,
"output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-pro-2026-03-05": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "azure_ai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "responses",
"output_cost_per_token": 0.00018,
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_priority": 0.00036,
"output_cost_per_token_above_272k_tokens_priority": 0.00054,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-pro",
"supported_endpoints": [
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": true
},
"azure_ai/gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_above_272k_tokens": 1.5e-07,
"cache_read_input_token_cost_priority": 1.5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 3e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_above_272k_tokens": 1.5e-06,
"input_cost_per_token_priority": 1.5e-06,
"input_cost_per_token_above_272k_tokens_priority": 3e-06,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_above_272k_tokens": 6.75e-06,
"output_cost_per_token_priority": 9e-06,
"output_cost_per_token_above_272k_tokens_priority": 1.35e-05,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-mini",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-5.4-nano-2026-03-17": {
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"cache_read_input_token_cost_priority": 4e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"input_cost_per_token_priority": 4e-07,
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.25e-06,
"output_cost_per_token_above_272k_tokens": 1.875e-06,
"output_cost_per_token_priority": 2.5e-06,
"output_cost_per_token_above_272k_tokens_priority": 3.75e-06,
"source": "https://ai.azure.com/catalog/models/gpt-5.4-nano",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true,
"supports_web_search": true,
"supports_none_reasoning_effort": true,
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/model_router": {
"input_cost_per_token": 1.4e-07,
"output_cost_per_token": 0,
@ -3521,7 +3895,7 @@
"supports_tool_choice": true
},
"azure/gpt-4o-mini-transcribe": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "azure",
"max_input_tokens": 16000,
@ -3596,7 +3970,7 @@
"supports_tool_choice": true
},
"azure/gpt-4o-transcribe": {
"input_cost_per_audio_token": 6e-06,
"input_cost_per_audio_token": 2.5e-06,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 16000,
@ -3608,7 +3982,7 @@
]
},
"azure/gpt-4o-transcribe-diarize": {
"input_cost_per_audio_token": 6e-06,
"input_cost_per_audio_token": 2.5e-06,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 16000,
@ -8974,7 +9348,7 @@
"supports_vision": true
},
"gpt-4o-transcribe-diarize": {
"input_cost_per_audio_token": 6e-06,
"input_cost_per_audio_token": 2.5e-06,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 16000,
@ -15551,14 +15925,17 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_token": 1.5e-07,
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
"mode": "embedding",
"output_cost_per_token": 0,
"output_vector_size": 3072,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal",
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supports_multimodal": true,
"uses_embed_content": true
},
@ -15573,7 +15950,7 @@
"mode": "embedding",
"output_cost_per_token": 0,
"output_vector_size": 3072,
"source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal",
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supports_multimodal": true,
"uses_embed_content": true
},
@ -18988,7 +19365,7 @@
"supports_vision": true
},
"gpt-4o-mini-transcribe": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 16000,
@ -19118,7 +19495,7 @@
"supports_vision": true
},
"gpt-4o-transcribe": {
"input_cost_per_audio_token": 6e-06,
"input_cost_per_audio_token": 2.5e-06,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 16000,
@ -21104,6 +21481,38 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"gpt-realtime-2": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image": 5e-06,
"input_cost_per_token": 4e-06,
"litellm_provider": "openai",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
@ -38898,7 +39307,7 @@
]
},
"gpt-4o-mini-transcribe-2025-03-20": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 16000,
@ -38910,7 +39319,7 @@
]
},
"gpt-4o-mini-transcribe-2025-12-15": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 16000,

25
litellm/ocr/main.py
View file

@ -10,7 +10,6 @@ import os
import re
from functools import partial
from io import IOBase
from pathlib import Path
from typing import Any, Coroutine, Dict, Optional, Union
import httpx
@ -376,11 +375,13 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str,
with an inline base64 data URI.
Accepts document dicts like:
{"type": "file", "file": "/path/to/document.pdf"} # file path string
{"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path
{"type": "file", "file": <binary file-like object>} # file-like object (BinaryIO)
{"type": "file", "file": b"raw bytes"} # raw bytes
Bare ``str`` paths are not accepted — pass a ``pathlib.Path`` or
``open(path, "rb")`` instead. See the str check below for the rationale.
Returns:
{"type": "document_url", "document_url": "data:<mime>;base64,<data>"}
or {"type": "image_url", "image_url": "data:<mime>;base64,<data>"}
@ -389,14 +390,28 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str,
if file_input is None:
raise ValueError(
"document with type='file' must include a 'file' field containing "
"a file path (str), pathlib.Path, file-like object, or bytes"
"a pathlib.Path, file-like object, or bytes"
)
file_bytes: bytes
mime_type: str = "application/octet-stream"
file_name: Optional[str] = None
if isinstance(file_input, (str, Path)):
if isinstance(file_input, str):
# Bare strings are rejected here. The OCR ``document`` accepts a
# ``{"type": "file", "file": <value>}`` shape, and when this helper
# runs in a proxy request handler ``<value>`` is attacker-controlled.
# Opening it as a path is an arbitrary local file read on the proxy
# host, which is then base64-encoded and forwarded to the OCR
# provider — an exfiltration primitive.
raise ValueError(
"OCR file input does not accept bare str values. Pass bytes, "
"a pathlib.Path, or a file-like object. To OCR a local file "
"from a path, call open(path, 'rb') yourself."
)
if isinstance(file_input, os.PathLike):
# os.PathLike (pathlib.Path and custom __fspath__ classes) is a
# Python-level type that HTTP form values can't fabricate.
file_path = str(file_input)
if not os.path.isfile(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
@ -417,7 +432,7 @@ def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str,
else:
raise ValueError(
f"Unsupported file input type: {type(file_input)}. "
"Expected str (file path), pathlib.Path, bytes, or a file-like object."
"Expected pathlib.Path, bytes, or a file-like object."
)
if not file_bytes:

156
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,3 +1,4 @@
import re
from typing import Dict, List, Optional, Set, Tuple, cast
from fastapi import HTTPException
@ -122,6 +123,24 @@ class MCPRequestHandler:
# cannot be smuggled via query string, hostname, or a deeper URL segment.
if request.url.path.startswith("/.well-known/"):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
not litellm_api_key
and MCPRequestHandler._target_servers_delegate_auth_to_upstream( # noqa: E501
path=request.url.path, mcp_servers=mcp_servers
)
):
# Operator opted this oauth2 server into upstream-delegated auth
# (PKCE passthrough): skip LiteLLM API-key/SSO entirely so the
# client authenticates directly with the upstream MCP server.
# Fires ONLY when neither x-litellm-api-key nor Authorization is
# present. If any LiteLLM key is supplied (primary or secondary
# header), we fall through so user_id is resolved, spend/rate
# limiting apply, and any stored OAuth token can be retrieved
# and forwarded upstream. Gated by
# _target_servers_delegate_auth_to_upstream, which only returns
# True when EVERY target is auth_type=oauth2 AND has the
# delegate_auth_to_upstream flag set — fails closed otherwise.
validated_user_api_key_auth = UserAPIKeyAuth()
elif has_explicit_litellm_key:
# Explicit x-litellm-api-key provided - always validate normally
validated_user_api_key_auth = await user_api_key_auth(
@ -181,23 +200,62 @@ class MCPRequestHandler:
@staticmethod
def _extract_target_server_names_from_path(path: str) -> List[str]:
"""
Extract the target MCP server name from the standard MCP transport
URL patterns: ``/mcp/{server_name}[/...]`` and
Extract the target MCP server name(s) from the standard MCP transport
URL patterns: ``/mcp/{server_name_or_csv}[/...]`` and
``/{server_name}/mcp[/...]``. Returns ``[]`` for any other path so
callers fail closed when the target cannot be resolved.
Mirrors the regex-based parser in ``server.py::_get_mcp_servers_in_path``
so the names used for auth gating match the names used for downstream
filtering. Without this alignment, an attacker could craft
``/mcp/<delegated_server>/<garbage>`` so that auth treats the request
as targeting the delegate server (bypassing LiteLLM auth) while
downstream filtering sees a different (non-existent) target and falls
back to the caller's full allowed-server set.
REST/admin endpoints, OAuth2 server endpoints
(``/{server_name}/authorize``, ``/token`` etc.), and ``.well-known``
discovery routes intentionally fall through — those flows do not need
OAuth2 token passthrough. Clients aggregating multiple servers should
use ``x-mcp-servers``, which takes precedence over path parsing.
use ``x-mcp-servers`` on a path that does not encode a target.
"""
# ``/{server_name}/mcp[/...]`` form — single server. The literal
# ``mcp`` must be the second segment (not the first, which would be
# the ``/mcp/...`` form handled below). This branch must stay in sync
# with ``server.py::_get_mcp_servers_in_path``, which also accepts the
# un-rewritten form (some entry points may skip the
# ``dynamic_mcp_route`` rewrite).
segments = [s for s in path.split("/") if s]
if len(segments) >= 2 and segments[0] == "mcp":
return [segments[1]]
if len(segments) >= 2 and segments[1] == "mcp":
if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp":
return [segments[0]]
return []
# ``/mcp/...`` form — server name(s) may contain a slash (e.g.
# ``custom_solutions/user_123``) and may be a comma-separated list.
# Use the same parsing logic as ``_get_mcp_servers_in_path`` so the
# parsed names match downstream routing.
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if not mcp_path_match:
return []
servers_and_path = mcp_path_match.group(1)
if not servers_and_path:
return []
if "," in servers_and_path:
# Comma-separated servers, possibly followed by a trailing path.
path_match = re.search(r"/([^/,]+(?:/[^/,]+)*)$", servers_and_path)
if path_match:
servers_part = servers_and_path[: -(len(path_match.group(1)) + 1)]
else:
servers_part = servers_and_path
return [s.strip() for s in servers_part.split(",") if s.strip()]
# Single-server case — server name may contain at most one slash.
single_server_match = re.match(
r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path
)
if single_server_match:
return [single_server_match.group(1)]
return [servers_and_path]
@staticmethod
def _target_servers_use_oauth2(path: str, mcp_servers: Optional[List[str]]) -> bool:
@ -217,13 +275,13 @@ class MCPRequestHandler:
)
from litellm.types.mcp import MCPAuth
# Use the x-mcp-servers header verbatim when present (including the
# explicitly-empty list, which means "no targets" → fail closed).
# Only fall back to path parsing when the header was absent entirely.
target_names = (
mcp_servers
if mcp_servers is not None
else MCPRequestHandler._extract_target_server_names_from_path(path)
# Resolve the same target list downstream routing will use. For
# ``/mcp/...`` routes, ``extract_mcp_auth_context`` overrides the
# ``x-mcp-servers`` header with path-derived names, so we must mirror
# that here — otherwise a caller could set the header to a permissive
# server while the path targets a stricter one (header/path TOCTOU).
target_names = MCPRequestHandler._resolve_target_server_names(
path=path, mcp_servers_header=mcp_servers
)
if not target_names:
return False
@ -234,6 +292,76 @@ class MCPRequestHandler:
return False
return True
@staticmethod
def _target_servers_delegate_auth_to_upstream(
path: str, mcp_servers: Optional[List[str]]
) -> bool:
"""
True only when EVERY MCP server the request targets is configured for
``auth_type == oauth2`` AND has ``delegate_auth_to_upstream=True``.
Fails closed when any target does not opt in or cannot be resolved.
Used by :meth:`process_mcp_request` to skip LiteLLM API-key/SSO auth
entirely (PKCE passthrough) so the client authenticates directly with
the upstream MCP server. Mixed-target requests (e.g. one delegated +
one non-delegated server) fall back to normal LiteLLM auth.
"""
# Inline imports avoid a circular dependency: mcp_server_manager imports
# from this module.
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.types.mcp import MCPAuth
# See _target_servers_use_oauth2: must mirror the downstream
# header-vs-path override or an attacker could set
# ``x-mcp-servers`` to a delegate-enabled server while the URL path
# targets a non-delegate server, skipping LiteLLM auth for it.
target_names = MCPRequestHandler._resolve_target_server_names(
path=path, mcp_servers_header=mcp_servers
)
if not target_names:
return False
for name in target_names:
server = global_mcp_server_manager.get_mcp_server_by_name(name)
if server is None or server.auth_type != MCPAuth.oauth2:
return False
# `is True` is intentional: opt-in must be an explicit boolean
# True. A MagicMock attribute (in tests) or any other truthy
# non-bool must not silently enable the bypass.
if getattr(server, "delegate_auth_to_upstream", False) is not True:
return False
# Never delegate for M2M (client_credentials) servers: LiteLLM
# fetches the upstream token automatically using stored credentials,
# so allowing anonymous bypass would let any external caller invoke
# tools authenticated as LiteLLM's service account.
if server.has_client_credentials:
return False
return True
@staticmethod
def _resolve_target_server_names(
path: str, mcp_servers_header: Optional[List[str]]
) -> List[str]:
"""
Resolve the target MCP server names exactly as downstream routing
does (``server.py::extract_mcp_auth_context``).
For ``/mcp/...`` paths, downstream routing **overrides** any
``x-mcp-servers`` header value with the path-derived names. Mirror
that here so an attacker cannot use a permissive header value to
flip an auth gate while the path targets a stricter server
(header/path TOCTOU). For non-``/mcp/...`` paths (where the path
does not encode targets), fall back to the header.
"""
path_targets = MCPRequestHandler._extract_target_server_names_from_path(path)
if path_targets:
return path_targets
# Path did not resolve to /mcp/... targets — trust the header
# (including an explicitly empty list, which means "no targets").
return mcp_servers_header if mcp_servers_header is not None else []
@staticmethod
def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]:
"""

25
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -86,7 +86,9 @@ def decode_state_hash(encrypted_state: str) -> dict:
def _get_validated_client_redirect_uri(
request: Request, state_data: Dict[str, Any]
) -> str:
"""Return a trusted (same-origin or loopback) client redirect URI from OAuth state."""
"""Return a trusted (same-origin, loopback, or ops-allowlisted)
client redirect URI from OAuth state.
"""
redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url")
if not redirect_uri or not isinstance(redirect_uri, str):
raise HTTPException(status_code=400, detail="Invalid redirect URI")
@ -296,11 +298,10 @@ async def authorize_with_server(
status_code=400, detail="MCP server authorization url is not set"
)
# Loopback OR same-origin redirect_uri. The URI is encrypted into the
# OAuth state and decoded on /callback to redirect the user back;
# restricting to trusted origins blocks the open-redirect +
# code-theft primitive (VERIA-57 root cause B). Loopback supports
# native MCP clients; same-origin supports the proxy's own UI callback.
# Trusted redirect_uri: same-origin, loopback, or ops-allowlisted.
# The URI is encrypted into the OAuth state and decoded on
# /callback to redirect the user back; a non-trusted URI would be
# an open-redirect + code-theft primitive (VERIA-57 root cause B).
validate_trusted_redirect_uri(request, redirect_uri)
parsed = urlparse(redirect_uri)
base_url = urlunparse(parsed._replace(query=""))
@ -623,12 +624,12 @@ async def callback(request: Request, code: str, state: str):
state_data = decode_state_hash(state)
original_state = state_data["original_state"]
# Re-validate at the sink. /authorize rejects untrusted
# redirect_uri before encoding into state, but encrypted states
# minted before that check was added have no expiry and remain
# valid indefinitely. Validating here (same-origin OR loopback)
# blocks the open-redirect + code-theft primitive even for pre-fix
# states while allowing the UI's same-origin callback to work.
# Re-validate the client redirect URI at the sink. /authorize
# rejects untrusted URIs before encoding them into state, but
# encrypted states minted before that check was added have no
# expiry and remain valid indefinitely. Validating here blocks
# the open-redirect + code-theft primitive even for pre-fix
# states while permitting same-origin / allowlisted clients.
redirect_uri = _get_validated_client_redirect_uri(request, state_data)
params = {"code": code, "state": original_state}

154
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -145,6 +145,30 @@ def _warn_on_server_name_fields(
_warn("server_name", server_name)
def _warn_internal_delegate_pkce_if_applicable(
server: MCPServer, *, source: str
) -> None:
"""Surface internal + upstream PKCE delegate in logs for operators."""
if server.auth_type != MCPAuth.oauth2:
return
if getattr(server, "delegate_auth_to_upstream", False) is not True:
return
if getattr(server, "available_on_public_internet", True):
return
if server.has_client_credentials:
return
label = get_server_prefix(server)
verbose_logger.warning(
"MCP server %r (id=%s, source=%s): internal-only (available_on_public_internet=false) "
"with delegate_auth_to_upstream=true. Anonymous callers can reach the upstream OAuth2 "
"/authorize flow and complete PKCE without a LiteLLM API key session; ensure the "
"upstream IdP and network enforce your access policy.",
label,
server.server_id,
source,
)
def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]:
"""
Deserialize optional JSON mappings stored in the database.
@ -297,32 +321,6 @@ class MCPServerManager:
)()
name_for_prefix = get_server_prefix(temp_server)
# Use alias for name if present, else server_name
alias = server_config.get("alias", None)
# Apply mcp_aliases mapping if provided
if mcp_aliases and alias is None:
# Check if this server_name has an alias in mcp_aliases
for alias_name, target_server_name in mcp_aliases.items():
if (
target_server_name == server_name
and alias_name not in used_aliases
):
alias = alias_name
used_aliases.add(alias_name)
verbose_logger.debug(
f"Mapped alias '{alias_name}' to server '{server_name}'"
)
break
# Create a temporary server object to use with get_server_prefix utility
temp_server = type(
"TempServer",
(),
{"alias": alias, "server_name": server_name, "server_id": None},
)()
name_for_prefix = get_server_prefix(temp_server)
server_url = server_config.get("url", None) or ""
# Generate stable server ID based on parameters
server_id = self._generate_stable_server_id(
@ -402,6 +400,9 @@ class MCPServerManager:
available_on_public_internet=bool(
server_config.get("available_on_public_internet", True)
),
delegate_auth_to_upstream=bool(
server_config.get("delegate_auth_to_upstream", False)
),
# AWS SigV4 fields
aws_access_key_id=server_config.get("aws_access_key_id", None),
aws_secret_access_key=server_config.get("aws_secret_access_key", None),
@ -422,6 +423,7 @@ class MCPServerManager:
),
)
self._assign_unique_short_prefix(new_server)
_warn_internal_delegate_pkce_if_applicable(new_server, source="config")
self.config_mcp_servers[server_id] = new_server
# Check if this is an OpenAPI-based server
@ -599,16 +601,57 @@ class MCPServerManager:
)
raise e
def _cleanup_server_tool_routing_artifacts(self, server: MCPServer) -> None:
"""Drop OpenAPI global tools and name-mapping rows owned by ``server``.
When a server leaves ``self.registry`` (eviction, ``remove_server``, etc.),
OpenAPI tools remain in ``global_mcp_tool_registry`` and
``tool_name_to_mcp_server_name_mapping`` unless removed here. Stale
mappings make ``_get_mcp_server_from_tool_name`` resolve to a prefix that
no longer exists in the live registry.
"""
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
prefix_root = normalize_server_name(get_server_prefix(server))
if server.spec_path and prefix_root:
openapi_key_prefix = prefix_root + MCP_TOOL_PREFIX_SEPARATOR
global_mcp_tool_registry.unregister_tools_with_prefix(openapi_key_prefix)
owned_raw: Set[str] = set()
for p in iter_known_server_prefixes(server):
if p:
owned_raw.add(p)
if server.name:
owned_raw.add(server.name)
owned_normalized = {normalize_server_name(x) for x in owned_raw}
stale_mapping_keys: List[str] = []
for tool_name, mapped_server in list(
self.tool_name_to_mcp_server_name_mapping.items()
):
if mapped_server in owned_raw:
stale_mapping_keys.append(tool_name)
elif normalize_server_name(str(mapped_server)) in owned_normalized:
stale_mapping_keys.append(tool_name)
for key in stale_mapping_keys:
del self.tool_name_to_mcp_server_name_mapping[key]
def remove_server(self, mcp_server: LiteLLM_MCPServerTable):
"""
Remove a server from the registry
"""
if mcp_server.server_name in self.get_registry():
del self.registry[mcp_server.server_name]
verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_name}")
elif mcp_server.server_id in self.get_registry():
del self.registry[mcp_server.server_id]
verbose_logger.debug(f"Removed MCP Server: {mcp_server.server_id}")
evicted: Optional[MCPServer] = self.registry.pop(mcp_server.server_id, None)
if evicted is None and mcp_server.server_name:
evicted = self.registry.pop(mcp_server.server_name, None)
if evicted is not None:
verbose_logger.debug(
"Removed MCP Server: %s", mcp_server.server_id or mcp_server.server_name
)
self._cleanup_server_tool_routing_artifacts(evicted)
else:
verbose_logger.warning(
f"Server ID {mcp_server.server_id} not found in registry"
@ -755,6 +798,9 @@ class MCPServerManager:
available_on_public_internet=bool(
getattr(mcp_server, "available_on_public_internet", True)
),
delegate_auth_to_upstream=bool(
getattr(mcp_server, "delegate_auth_to_upstream", False)
),
created_at=getattr(mcp_server, "created_at", None),
updated_at=getattr(mcp_server, "updated_at", None),
tool_name_to_display_name=_deserialize_json_dict(
@ -787,6 +833,7 @@ class MCPServerManager:
)
or "urn:ietf:params:oauth:token-type:access_token",
)
_warn_internal_delegate_pkce_if_applicable(new_server, source="database")
return new_server
async def _maybe_register_openapi_tools(
@ -806,6 +853,13 @@ class MCPServerManager:
self.initialize_tool_name_to_mcp_server_name_mapping()
async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
# The runtime registry is the allowlist for tool calls and health
# probes (which spawn the underlying transport, including stdio
# subprocesses). Match the eligibility set used by the bulk DB
# filter in reload_servers_from_database() — NULL is legacy and
# "approved" is a legacy alias for "active".
if mcp_server.approval_status not in (None, "active", "approved"):
return
try:
if mcp_server.server_id not in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
@ -819,6 +873,16 @@ class MCPServerManager:
raise e
async def update_server(self, mcp_server: LiteLLM_MCPServerTable):
# If a previously-active server has been moved out of the active
# state, evict any stale registry entry so subsequent tool calls and
# health probes can't reach it.
if mcp_server.approval_status not in (None, "active", "approved"):
evicted = self.registry.pop(mcp_server.server_id, None)
if evicted is None and mcp_server.server_name:
evicted = self.registry.pop(mcp_server.server_name, None)
if evicted is not None:
self._cleanup_server_tool_routing_artifacts(evicted)
return
try:
if mcp_server.server_id in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
@ -909,6 +973,31 @@ class MCPServerManager:
if not in_toolset_scope:
combined_servers.update(allow_all_server_ids)
# For anonymous callers (no user_id, no role), also surface any
# servers the operator has opted into upstream-delegated auth.
# These servers handle their own auth at the upstream level, so
# LiteLLM granting access here does not bypass any security gate.
is_anonymous = not (
user_api_key_auth
and (
getattr(user_api_key_auth, "user_id", None)
or getattr(user_api_key_auth, "user_role", None)
or getattr(user_api_key_auth, "api_key", None)
)
)
if is_anonymous:
delegate_server_ids = [
server.server_id
for server in self.get_registry().values()
if getattr(server, "auth_type", None) == MCPAuth.oauth2
and getattr(server, "delegate_auth_to_upstream", False) is True
# M2M servers must not be exposed anonymously: an
# unauthenticated caller would get LiteLLM to proxy tool
# calls using its stored client_credentials.
and not server.has_client_credentials
]
combined_servers.update(delegate_server_ids)
if len(combined_servers) == 0:
verbose_logger.debug(
"No allowed MCP Servers found for user api key auth."
@ -3474,6 +3563,7 @@ class MCPServerManager:
registration_url=server.registration_url,
allow_all_keys=server.allow_all_keys,
available_on_public_internet=server.available_on_public_internet,
delegate_auth_to_upstream=server.delegate_auth_to_upstream,
is_byok=server.is_byok,
byok_description=server.byok_description,
byok_api_key_help_url=server.byok_api_key_help_url,

180
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -1,7 +1,9 @@
"""Shared helpers for the MCP OAuth authorization endpoints
(BYOK + discoverable / pass-through OAuth proxy)."""
import os
from ipaddress import ip_address
from typing import List, Optional
from urllib.parse import urlparse, urlunparse
from fastapi import HTTPException, Request
@ -13,6 +15,20 @@ from litellm.proxy.auth.ip_address_utils import IPAddressUtils
# must not be cached — both success and error bodies may reveal secrets.
TOKEN_NO_CACHE_HEADERS = {"Cache-Control": "no-store", "Pragma": "no-cache"}
# Stripped from netloc before same-origin comparison so
# ``llm.example.com`` matches ``llm.example.com:443`` (load balancers
# routinely set X-Forwarded-Port: 443 even when the client URL has no
# explicit port, which would otherwise break a literal netloc compare).
_DEFAULT_PORTS = {"http": 80, "https": 443}
# Env var for ops to allowlist additional redirect_uri origins beyond
# same-origin + loopback — needed for first-party OAuth clients hosted
# on sister domains (e.g. a web app on app.example.com registering as
# an OAuth client of the MCP proxy on llm.example.com). Comma-separated;
# each entry is ``host`` or ``host:port``; a ``*.`` prefix matches any
# subdomain. HTTPS only.
_TRUSTED_REDIRECT_ORIGINS_ENV = "MCP_TRUSTED_REDIRECT_ORIGINS"
def get_request_base_url(request: Request) -> str:
"""
@ -96,22 +112,106 @@ def validate_loopback_redirect_uri(redirect_uri: str) -> None:
raise HTTPException(status_code=400, detail="invalid_request")
def _strip_default_port(scheme: str, netloc: str) -> str:
"""Return ``netloc`` lowercased with the scheme's default port
stripped. ``Llm.Example.com:443`` with scheme ``https`` becomes
``llm.example.com``. Used so a literal netloc comparison between
the proxy's origin and the client redirect_uri survives a load-
balancer that sets ``X-Forwarded-Port: 443``.
"""
if not netloc:
return netloc
lowered = netloc.lower()
if lowered.startswith("["):
# IPv6 literal: port (if any) appears after the "]".
close = lowered.rfind("]")
if close != -1 and lowered[close + 1 :].startswith(":"):
try:
port = int(lowered[close + 2 :])
except ValueError:
return lowered
if _DEFAULT_PORTS.get(scheme) == port:
return lowered[: close + 1]
return lowered
if ":" in lowered:
host, _, port_str = lowered.rpartition(":")
try:
port = int(port_str)
except ValueError:
return lowered
if _DEFAULT_PORTS.get(scheme) == port:
return host
return lowered
def _parse_trusted_redirect_origins() -> List[str]:
"""Parse ``MCP_TRUSTED_REDIRECT_ORIGINS`` into normalized entries.
Empty / unset env var → empty list. Entries are lowercased and any
scheme / path component the operator included is stripped. Default
``:443`` is also stripped from non-wildcard entries so
``app.example.com:443`` matches a redirect_netloc whose own ``:443``
has already been normalized away — the allowlist path is https-only,
so ``:443`` is the only default port that can legitimately appear.
"""
raw = os.environ.get(_TRUSTED_REDIRECT_ORIGINS_ENV, "").strip()
if not raw:
return []
entries: List[str] = []
for token in raw.split(","):
entry = token.strip().lower()
if not entry:
continue
if "://" in entry:
entry = entry.split("://", 1)[1]
entry = entry.split("/", 1)[0]
if not entry:
continue
# Wildcards don't express port constraints; leave them alone.
if not entry.startswith("*."):
entry = _strip_default_port("https", entry)
if entry:
entries.append(entry)
return entries
def _matches_trusted_origin_entry(netloc: str, entry: str) -> bool:
"""``entry`` is either ``host[:port]`` (exact match after port
normalization) or ``*.suffix`` (subdomain wildcard; matches any
strictly-deeper subdomain of ``suffix`` but not ``suffix`` itself).
``netloc`` is the already-port-normalized, lowercased netloc of
the redirect_uri being validated.
"""
if entry.startswith("*."):
suffix = entry[2:]
if not suffix or suffix.startswith("."):
return False
# Strip port from netloc for wildcard host comparison;
# wildcards don't express port constraints.
host = netloc.split(":", 1)[0] if ":" in netloc else netloc
return host != suffix and host.endswith("." + suffix)
return netloc == entry
def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
"""Accept same-origin (proxy's own origin) OR loopback ``redirect_uri``.
"""Accept ``redirect_uri`` when it is (a) same-origin with the
proxy's own request origin, (b) loopback, or (c) listed in the
``MCP_TRUSTED_REDIRECT_ORIGINS`` ops allowlist.
Same-origin is required for the LiteLLM UI's OAuth flow: the UI
redirects to ``<proxy>/ui/mcp/oauth/callback`` which is not loopback
but is on the proxy's own trusted HTTPS origin. An attacker cannot
host content on the proxy's own origin without already owning the
proxy, so the open-redirect / code-theft primitive that motivated
:func:`validate_loopback_redirect_uri` does not apply here.
Same-origin is VERIA-57's threat-model-safe equivalent of loopback:
an attacker who can host content on the proxy's own HTTPS origin
has already compromised the proxy, so the open-redirect + code-
theft primitive that motivated the loopback-only rule does not
apply. The same reasoning extends to ops-trusted first-party
hosts (e.g. an internal web app registering as an OAuth client of
the proxy on a sister domain).
Loopback continues to be accepted for native MCP clients (per
OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3).
Allowlisted non-loopback hosts are accepted only when the
redirect_uri scheme is ``https`` — an attacker on the network
cannot elevate to https without controlling the host's TLS key.
Use this in the discoverable OAuth proxy endpoints that serve both
native clients and the proxy's own UI. BYOK endpoints that only
support native clients should keep
native clients and the proxy's UI / cross-origin web clients. The
BYOK endpoints, which only serve native MCP clients, retain
:func:`validate_loopback_redirect_uri`.
"""
try:
@ -122,26 +222,53 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
raise HTTPException(status_code=400, detail="invalid_request")
if parsed.fragment:
raise HTTPException(status_code=400, detail="invalid_request")
if not parsed.netloc or parsed.username is not None or parsed.password is not None:
raise HTTPException(status_code=400, detail="invalid_request")
# Reject userinfo (``user:pass@host``) outright: OAuth redirect_uris
# have no legitimate reason to carry credentials, and allowing them
# opens a host-confusion attack where the netloc *looks* allowlisted
# (``app.example.com:443@attacker.example``) but the browser navigates
# to the post-``@`` host and hands the authorization code to the
# attacker. We compare against ``hostname`` after this, but defense in
# depth keeps malformed netloc strings from reaching the wildcard
# splitter.
if parsed.username is not None or parsed.password is not None:
raise HTTPException(status_code=400, detail="invalid_request")
# Reject backslash in netloc: urlparse keeps ``\`` as part of netloc,
# but browsers normalize ``\`` to ``/`` for http(s) URLs and treat it
# as the start of the path. An attacker can exploit that split by
# crafting ``https://attacker.net\app.example.com/cb`` — urlparse sees
# ``attacker.net\app.example.com`` (matches ``*.example.com``) while
# the browser navigates to ``attacker.net`` with the auth code.
if "\\" in parsed.netloc:
raise HTTPException(status_code=400, detail="invalid_request")
# Same-origin: scheme + netloc (host[:port]) must match the proxy's
# own base URL at this request (honouring trusted X-Forwarded-*).
redirect_netloc = _strip_default_port(parsed.scheme, parsed.netloc)
# (a) Same-origin. Swallow ``get_request_base_url`` failures so the
# loopback + allowlist paths remain reachable when the origin can't
# be determined (e.g. request came from an untrusted proxy and
# ``get_request_base_url`` raised).
proxy_base: Optional[str] = None
try:
proxy_base = urlparse(get_request_base_url(request))
if (
parsed.netloc
and parsed.scheme == proxy_base.scheme
and parsed.netloc.lower() == proxy_base.netloc.lower()
):
return
proxy_base = get_request_base_url(request)
except Exception as exc:
# If we can't determine the proxy's origin, fall through to
# loopback. Log so the failure is diagnosable in production.
verbose_logger.warning(
"validate_trusted_redirect_uri: could not determine proxy origin, "
"falling back to loopback-only check. error=%s",
"falling back to loopback + allowlist. error=%s",
exc,
)
proxy_base = None
if proxy_base:
proxy_parsed = urlparse(proxy_base)
if (
parsed.scheme == proxy_parsed.scheme
and redirect_netloc
== _strip_default_port(proxy_parsed.scheme, proxy_parsed.netloc)
):
return
# (b) Loopback — same rule as validate_loopback_redirect_uri.
host = (parsed.hostname or "").lower()
if host == "localhost":
return
@ -150,4 +277,11 @@ def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
return
except ValueError:
pass
# (c) Ops allowlist. https only.
if parsed.scheme == "https":
for entry in _parse_trusted_redirect_origins():
if _matches_trusted_origin_entry(redirect_netloc, entry):
return
raise HTTPException(status_code=400, detail="invalid_request")

189
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -23,6 +23,7 @@ from typing import (
cast,
)
import httpx
from fastapi import FastAPI, HTTPException
from pydantic import AnyUrl, ConfigDict
from starlette.requests import Request as StarletteRequest
@ -51,13 +52,17 @@ from litellm.proxy._experimental.mcp_server.utils import (
get_server_prefix,
iter_known_server_prefixes,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
get_chain_id_from_headers,
)
from litellm.types.mcp import MCPAuth
from litellm.types.mcp import MCPAuth, MCPSpecVersion
from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall
from litellm.utils import Rules, client, function_setup
@ -1332,8 +1337,24 @@ if MCP_AVAILABLE:
raw_headers=raw_headers,
)
# If no OAuth2 token came from request headers, fall back to pre-fetched creds
if extra_headers is None and server.auth_type == MCPAuth.oauth2:
# Prefer server-stored per-user OAuth when configured, so a stale
# Authorization header from the MCP client cannot override Redis/DB
# (same issue as call_tool in mcp_server_manager: VS Code caches tokens).
if (
server.auth_type == MCPAuth.oauth2
and getattr(server, "needs_user_oauth_token", False)
and user_api_key_auth is not None
):
db_headers = await _get_user_oauth_extra_headers_from_db(
server,
user_api_key_auth,
prefetched_creds=_prefetched_oauth_creds,
)
if db_headers:
extra_headers = db_headers
# If still no OAuth2 token, fall back to pre-fetched creds (non-stale-client path)
elif extra_headers is None and server.auth_type == MCPAuth.oauth2:
extra_headers = await _get_user_oauth_extra_headers_from_db(
server,
user_api_key_auth,
@ -2536,6 +2557,10 @@ if MCP_AVAILABLE:
import re
mcp_servers_from_path: Optional[List[str]] = None
segments = [s for s in path.split("/") if s]
if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp":
return [segments[0]]
# Match /mcp/<servers_and_maybe_path>
# Where servers can be comma-separated list of server names
# Server names can contain slashes (e.g., "custom_solutions/user_123")
@ -2754,6 +2779,157 @@ if MCP_AVAILABLE:
)
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
def _get_forwarded_auth_from_scope(scope: Scope) -> Optional[str]:
"""Return the upstream-bound ``Authorization`` header value, or None.
Only returns the ``Authorization`` header when ``x-litellm-api-key`` is
also present. In that case ``Authorization`` is unambiguously the
upstream token the caller wants forwarded to the MCP server. When
``x-litellm-api-key`` is absent the ``Authorization`` header may itself
be the LiteLLM proxy API key (backward-compat path in
``MCPRequestHandler.process_mcp_request``), and forwarding it upstream
would leak the proxy key to a third-party MCP server.
"""
authorization = None
has_litellm_key_header = False
for key, value in scope.get("headers", []):
key_lower = key.lower()
if key_lower == b"authorization":
authorization = value.decode("latin-1")
elif key_lower == b"x-litellm-api-key":
has_litellm_key_header = True
if not has_litellm_key_header:
return None
return authorization
async def _probe_upstream_auth(
url: str,
auth_header: str,
timeout: float = 5.0,
) -> tuple:
"""JSON-RPC initialize-probe the upstream URL to check whether the token is accepted.
Uses POST so StreamableHTTP MCP servers run the same auth path as a
real client request. Returns (status_code, www_authenticate).
Fails-open with (200, None) on network errors so a transient hiccup
does not block valid requests.
Uses the public ``AsyncHTTPHandler.post()`` interface and catches
``httpx.HTTPStatusError`` separately so the 401/403 we want to surface
is not swallowed by the broad fail-open ``except Exception`` below.
"""
client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.MCP,
params={"timeout": timeout},
)
probe_payload = {
"jsonrpc": "2.0",
"id": "litellm-mcp-auth-probe",
"method": "initialize",
"params": {
"protocolVersion": MCPSpecVersion.jun_2025.value,
"capabilities": {},
"clientInfo": {
"name": "litellm-mcp-auth-probe",
"version": "1.0.0",
},
},
}
probe_headers = {
"Authorization": auth_header,
"Accept": "application/json, text/event-stream",
}
try:
resp = await client.post(
url=url,
headers=probe_headers,
json=probe_payload,
timeout=timeout,
)
return resp.status_code, resp.headers.get("www-authenticate")
except httpx.HTTPStatusError as exc:
# AsyncHTTPHandler.post() calls raise_for_status(); a 401/403 from
# upstream lands here. Return its status so the caller can map it
# to the appropriate response.
return exc.response.status_code, exc.response.headers.get(
"www-authenticate"
)
except Exception as exc:
verbose_logger.debug(
f"_probe_upstream_auth: probe to {url} failed ({exc}), allowing request through"
)
return 200, None
async def _check_passthrough_upstream_auth(
scope: Scope,
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_servers: Optional[List[str]],
client_ip: Optional[str],
) -> None:
"""Probe pass-through upstream servers in parallel before the MCP session starts.
Only servers the caller's key is already authorized to reach are probed —
the list is derived from _get_allowed_mcp_servers so that a user cannot
trigger an upstream probe against a server their key is not permitted for.
The MCP SDK commits HTTP 200 headers before invoking handlers, so a 401
can only be returned before that point. This function raises HTTPException(401)
with a WWW-Authenticate header if any upstream rejects the client token.
Fails-open: network errors are logged and the request is allowed through.
"""
forwarded_auth = _get_forwarded_auth_from_scope(scope)
if not forwarded_auth:
return
# Use the authorized server set, not the raw user-supplied names, so that
# a caller cannot force a probe to a server their key is not allowed to use.
allowed_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
client_ip=client_ip,
)
passthrough_servers = [
srv
for srv in allowed_servers
if srv.extra_headers
and any(h.lower() == "authorization" for h in srv.extra_headers)
# Exclude M2M servers: _prepare_mcp_server_headers skips caller
# Authorization when has_client_credentials is set, so probing
# those with the caller's token would send the wrong credential.
and not srv.has_client_credentials
]
if not passthrough_servers:
return
probe_results = await asyncio.gather(
*[
_probe_upstream_auth(srv.url or "", forwarded_auth)
for srv in passthrough_servers
]
)
request = StarletteRequest(scope)
base_url = get_request_base_url(request)
for srv, (probe_status, _) in zip(passthrough_servers, probe_results):
if probe_status == 401:
# Token is missing or expired — direct the client to re-authorize.
authorization_uri = (
f"Bearer authorization_uri="
f"{base_url}/.well-known/oauth-authorization-server/{srv.name}"
)
raise HTTPException(
status_code=401,
detail="Unauthorized",
headers={"WWW-Authenticate": authorization_uri},
)
if probe_status == 403:
# Token is valid but the caller lacks permission — do not hint
# at re-authorization (RFC 9110: a fresh token with the same
# scopes would just hit 403 again and loop indefinitely).
raise HTTPException(
status_code=403,
detail="Forbidden",
)
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
@ -2827,6 +3003,13 @@ if MCP_AVAILABLE:
user_api_key_auth, active_toolset_id
)
# Pre-flight auth check for pass-through servers. Must run after
# toolset scoping so the probe list is derived from the fully-authorized
# server set, not the raw user-supplied names.
await _check_passthrough_upstream_auth(
scope, user_api_key_auth, mcp_servers, _client_ip
)
# Inject masked debug headers when client sends x-litellm-mcp-debug: true
_debug_headers = MCPDebug.maybe_build_debug_headers(
raw_headers=raw_headers,

27
litellm/proxy/_experimental/mcp_server/tool_registry.py
View file

@ -59,6 +59,22 @@ class MCPToolRegistry:
]
return list(self.tools.values())
def unregister_tools_with_prefix(self, prefix: str) -> int:
"""Remove tools whose registered name starts with ``prefix``.
Used when an OpenAPI-backed MCP server leaves the runtime registry so
stale tool handlers cannot be invoked after eviction.
"""
if not prefix:
return 0
removed = 0
for name in list(self.tools.keys()):
if name.startswith(prefix):
del self.tools[name]
removed += 1
verbose_logger.debug("Unregistered MCP tool %s", name)
return removed
def convert_tools_to_mcp_sdk_tool_type(
self, tools: List[MCPTool]
) -> List["MCPToolSDKTool"]:
@ -76,13 +92,20 @@ class MCPToolRegistry:
]
def load_tools_from_config(
self, mcp_tools_config: Optional[Dict[str, Any]] = None
self,
mcp_tools_config: Optional[Dict[str, Any]] = None,
config_file_path: Optional[str] = None,
) -> None:
"""
Load and register tools from the proxy config
Args:
mcp_tools_config: The mcp_tools config from the proxy config
config_file_path: Path to the operator's config.yaml. Threaded
through to ``get_instance_fn`` so an ``s3://``/``gcs://``
``handler`` declared in the YAML resolves; callers from a
non-YAML path must leave this ``None`` so the runtime gate
fires.
"""
if mcp_tools_config is None:
raise ValueError(
@ -105,7 +128,7 @@ class MCPToolRegistry:
# First check if it's a module path (e.g., "module.submodule.function")
if handler_name is None:
raise ValueError(f"handler is required for tool {name}")
handler = get_instance_fn(handler_name)
handler = get_instance_fn(handler_name, config_file_path)
if handler is None:
verbose_logger.warning(

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

30
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,30 +1,30 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js"],"default"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js"],"default"]
1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
1b:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
0:{"buildId":"LpD6ruZoEpvYpT5IvMEoa","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0b3d09ff6c6e4335.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b1c98cc932a0ab19.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/e7e5bfdf70ba79ab.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/7a9066dcd4a390ff.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/da1c7742cc6fe8b4.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/bd02f158353d9cea.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/ad02f56c287539eb.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/a8f7c8c5eeb6e042.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/6188170a32c9a3c3.js","async":true}]
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/20acf4fa815c638e.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/e1a670efcb966aaa.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/3c0e9dc19dbbd4ed.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","async":true}]
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true}]
17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","async":true}]
18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/ca7a3fdb635fb7dc.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/659ce28f2cb74401.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/934dbc43f8c1abde.js","async":true}]
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/d6ab357d1bbb53f0.js","async":true}]
17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/4a0ccb5ed3d0c33f.js","async":true}]
19:["$","$L1a",null,{"children":["$","$1b",null,{"name":"Next.MetadataOutlet","children":"$@1c"}]}]
1c:null

Some files were not shown because too many files have changed in this diff Show more