scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

Merge remote-tracking branch 'origin/main' into fix-sap-creds

Browse source
This commit is contained in:
Vasilisa Parshikova 2026-03-26 12:55:20 +04:00
commit 7e83c1caed
1429 changed files with 65511 additions and 16279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1098
.circleci/config.yml
View file

File diff suppressed because it is too large Load diff

36
.claude/settings.json
View file

@ -1,36 +0,0 @@
{
"permissions": {
"allow": [
"Bash(git show:*)",
"Bash(git worktree add:*)",
"Read(//Users/krrishdholakia/Documents/litellm/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
"Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
"Bash(python:*)",
"Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
"Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
"Bash(poetry run pytest:*)",
"Bash(git add:*)",
"Bash(git commit:*)",
"Bash(poetry run python:*)",
"Bash(poetry run pip:*)",
"Bash(git reset:*)",
"Bash(git cherry-pick:*)",
"Bash(git checkout:*)",
"Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
"Read(//Users/krrishdholakia/Documents/**)",
"Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
"Bash(ls:*)"
],
"additionalDirectories": [
"/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
"/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
"/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
]
}
}

2
.github/ISSUE_TEMPLATE/config.yml vendored
View file

@ -1,7 +1,7 @@
blank_issues_enabled: true
contact_links:
- name: Schedule Demo
url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
url: https://enterprise.litellm.ai/demo
about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
- name: Discord
url: https://discord.com/invite/wuPM9dRgDw

4
.github/pull_request_template.md vendored
View file

@ -11,6 +11,10 @@
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
## Delays in PR merge?
If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
## CI (LiteLLM team)
> **CI status guideline:**

6
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -2,7 +2,7 @@ name: Updates model_prices_and_context_window.json and Create Pull Request
on:
schedule:
- cron: "0 0 * * 0" # Run every Sundays at midnight
- cron: "0 0 * * 0" # Run every Sundays at midnight
#- cron: "0 0 * * *" # Run daily at midnight
jobs:
@ -10,7 +10,7 @@ jobs:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Install Dependencies
run: |
pip install aiohttp
@ -26,4 +26,4 @@ jobs:
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
--base main
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}

6
.github/workflows/check_duplicate_issues.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
contents: read
steps:
- name: Check for potential duplicates
uses: wow-actions/potential-duplicates@v1
uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
label: potential-duplicate
@ -30,13 +30,13 @@ jobs:
- name: Checkout close script
if: github.event.action == 'opened'
uses: actions/checkout@v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
- name: Set up Python
if: github.event.action == 'opened'
uses: actions/setup-python@v5
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.11"

7
.github/workflows/codeql.yml vendored
View file

@ -15,6 +15,7 @@ concurrency:
jobs:
analyze:
if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
name: Analyze (${{ matrix.language }})
runs-on: ubuntu-latest
timeout-minutes: 30
@ -37,16 +38,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql/codeql-config.yml
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
with:
category: "/language:${{ matrix.language }}"

44
.github/workflows/codspeed.yml vendored Normal file
View file

@ -0,0 +1,44 @@
name: CodSpeed Benchmarks
on:
push:
branches:
- main
pull_request:
branches:
- main
# Allow CodSpeed to trigger backtest performance analysis
# in order to generate initial data
workflow_dispatch:
permissions:
contents: read
id-token: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
benchmarks:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install dependencies
run: |
pip install -e "."
pip install pytest pytest-codspeed==4.3.0
- name: Run benchmarks
uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
with:
mode: simulation
run: pytest tests/benchmarks/ --codspeed

10
.github/workflows/create_daily_staging_branch.yml vendored
View file

@ -2,16 +2,17 @@ name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0,12 * * *' # Runs every 12 hours at midnight and noon UTC
workflow_dispatch: # Allow manual trigger
- cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
@ -43,11 +44,12 @@ jobs:
fi
create-internal-dev-branch:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0

443
.github/workflows/ghcr_deploy.yml vendored
View file

@ -1,443 +0,0 @@
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
name: Build, Publish LiteLLM Docker Image. New Release
on:
workflow_dispatch:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
default: "latest"
commit_hash:
description: "Commit hash"
required: true
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
CHART_NAME: litellm-helm
# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
# print commit hash, tag, and release type
print:
runs-on: ubuntu-latest
steps:
- run: |
echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
echo "Tag: ${{ github.event.inputs.tag }}"
echo "Release type: ${{ github.event.inputs.release_type }}"
docker-hub-deploy:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-database image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./docker/Dockerfile.database
tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-spend-logs image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./litellm-js/spend-logs/Dockerfile
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-non_root image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./docker/Dockerfile.non_root
tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
build-and-push-image:
runs-on: ubuntu-latest
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
- name: Build and push Docker image
uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
with:
context: .
push: true
tags: |
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
labels: ${{ steps.meta.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-ee:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for EE Dockerfile
id: meta-ee
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push EE Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: Dockerfile
push: true
tags: |
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-ee.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-database:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for database Dockerfile
id: meta-database
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: ./docker/Dockerfile.database
push: true
tags: |
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-database.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-non_root:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for non_root Dockerfile
id: meta-non_root
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push non_root Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: ./docker/Dockerfile.non_root
push: true
tags: |
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
labels: ${{ steps.meta-non_root.outputs.labels }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
build-and-push-image-spend-logs:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for spend-logs Dockerfile
id: meta-spend-logs
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
# Configure multi platform Docker builds
- name: Set up QEMU
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: ./litellm-js/spend-logs/Dockerfile
push: true
tags: |
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
${{ (github.event.inputs.release_type == 'stable' || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
run-observatory-tests:
if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
needs: [docker-hub-deploy]
uses: ./.github/workflows/run_observatory_tests.yml
with:
tag: ${{ github.event.inputs.tag }}
commit_hash: ${{ github.event.inputs.commit_hash }}
secrets: inherit
build-and-push-helm-chart:
if: github.event.inputs.release_type != 'dev'
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: lowercase github.repository_owner
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
# This allows users to easily map Helm chart versions to LiteLLM versions
# See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
INPUT_TAG="${{ github.event.inputs.tag }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
# Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
# v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
CHART_VERSION="${INPUT_TAG#v}"
# Add suffix for 'latest' releases (rc already has suffix in tag)
if [ "$RELEASE_TYPE" = "latest" ]; then
CHART_VERSION="${CHART_VERSION}-latest"
fi
# App version = Docker tag (keeps 'v' prefix to match Docker image tags)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}
registry_password: ${{ secrets.GITHUB_TOKEN }}
update_dependencies: true
release:
name: "New LiteLLM Release"
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
runs-on: "ubuntu-latest"
steps:
- name: Display version
run: echo "Current version is ${{ github.event.inputs.tag }}"
- name: "Set Release Tag"
run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
- name: Display release tag
run: echo "RELEASE_TAG is $RELEASE_TAG"
- name: "Create release"
uses: "actions/github-script@v6"
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
script: |
const commitHash = "${{ github.event.inputs.commit_hash}}";
console.log("Commit Hash:", commitHash); // Add this line for debugging
try {
const response = await github.rest.repos.createRelease({
draft: false,
generate_release_notes: true,
target_commitish: commitHash,
name: process.env.RELEASE_TAG,
owner: context.repo.owner,
prerelease: false,
repo: context.repo.repo,
tag_name: process.env.RELEASE_TAG,
});
core.exportVariable('RELEASE_ID', response.data.id);
core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
} catch (error) {
core.setFailed(error.message);
}
- name: Fetch Release Notes
id: release-notes
uses: actions/github-script@v6
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
script: |
try {
const response = await github.rest.repos.getRelease({
owner: context.repo.owner,
repo: context.repo.repo,
release_id: process.env.RELEASE_ID,
});
const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
return formattedBody;
} catch (error) {
core.setFailed(error.message);
}
env:
RELEASE_ID: ${{ env.RELEASE_ID }}
- name: Github Releases To Discord
env:
WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
REALEASE_TAG: ${{ env.RELEASE_TAG }}
RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
run: |
curl -H "Content-Type: application/json" -X POST -d '{
"content": "New LiteLLM release '"${RELEASE_TAG}"'",
"username": "Release Changelog",
"avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
"embeds": [
{
"title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
"description": "'"${RELEASE_NOTES}"'",
"color": 2105893
}
]
}' $WEBHOOK_URL

67
.github/workflows/ghcr_helm_deploy.yml vendored
View file

@ -1,67 +0,0 @@
# Standalone workflow to publish LiteLLM Helm Chart
# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
name: Build, Publish LiteLLM Helm Chart. New Release
on:
workflow_dispatch:
inputs:
tag:
description: "LiteLLM version tag (e.g., v1.81.0)"
required: true
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
REPO_OWNER: ${{github.repository_owner}}
# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
jobs:
build-and-push-helm-chart:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: lowercase github.repository_owner
run: |
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
# Sync Helm chart version with LiteLLM release version (1-1 versioning)
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
INPUT_TAG="${{ github.event.inputs.tag }}"
# Chart version = LiteLLM version without 'v' prefix
# v1.81.0 -> 1.81.0
CHART_VERSION="${INPUT_TAG#v}"
# App version = Docker tag (keeps 'v' prefix)
APP_VERSION="${INPUT_TAG}"
echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- name: Lint helm chart
run: helm lint deploy/charts/litellm-helm
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: litellm-helm
repository: ${{ env.REPO_OWNER }}
tag: ${{ steps.chart_version.outputs.version }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/litellm-helm
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}
registry_password: ${{ secrets.GITHUB_TOKEN }}
update_dependencies: true

19
.github/workflows/helm_unit_test.yml vendored
View file

@ -11,17 +11,26 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Helm 3.11.1
uses: azure/setup-helm@v1
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:
version: '3.11.1'
version: "3.11.1"
- name: Install Helm Unit Test Plugin
run: |
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
- name: Verify Helm Unit Test Plugin integrity
run: |
EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
exit 1
fi
echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
- name: Run unit tests
run:
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm

2
.github/workflows/interpret_load_test.py vendored
View file

@ -123,7 +123,7 @@ if __name__ == "__main__":
+ docker_run_command
+ "\n\n"
+ "### Don't want to maintain your internal proxy? get in touch 🎉"
+ "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
+ "\nHosted Proxy Alpha: https://enterprise.litellm.ai/demo"
+ "\n\n"
+ "## Load Test LiteLLM Proxy Results"
+ "\n\n"

11
.github/workflows/issue-keyword-labeler.yml vendored
View file

@ -2,8 +2,8 @@ name: Issue Keyword Labeler
on:
issues:
types:
- opened
types:
- opened
jobs:
scan-and-label:
@ -13,7 +13,7 @@ jobs:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Scan for provider keywords
id: scan
@ -24,7 +24,7 @@ jobs:
- name: Ensure label exists
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@ -51,7 +51,7 @@ jobs:
- name: Add label to the issue
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@ -61,4 +61,3 @@ jobs:
issue_number: context.issue.number,
labels: ['llm translation']
});

2
.github/workflows/label-component.yml vendored
View file

@ -12,7 +12,7 @@ jobs:
issues: write
steps:
- name: Add component labels
uses: actions/github-script@v7
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |

34
.github/workflows/main.yml vendored
View file

@ -1,34 +0,0 @@
name: Publish Dev Release to PyPI
on:
workflow_dispatch:
jobs:
publish-dev-release:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.8 # Adjust the Python version as needed
- name: Install dependencies
run: pip install toml twine
- name: Read version from pyproject.toml
id: read-version
run: |
version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Check if version exists on PyPI
id: check-version
run: |
set -e
if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."

207
.github/workflows/publish-migrations.yml vendored
View file

@ -1,207 +0,0 @@
name: Publish Prisma Migrations
permissions:
contents: write
pull-requests: write
on:
push:
paths:
- 'schema.prisma' # Check root schema.prisma
branches:
- main
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:
image: postgres:14
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
# Add shadow database service
postgres_shadow:
image: postgres:14
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5433:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
- name: Install Dependencies
run: |
pip install prisma
pip install python-dotenv
- name: Generate Initial Migration if None Exists
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
mkdir -p deploy/migrations
echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
echo "No existing migrations found, creating baseline..."
VERSION=$(date +%Y%m%d%H%M%S)
mkdir -p deploy/migrations/${VERSION}_initial
echo "Generating initial migration..."
# Save raw output for debugging
prisma migrate diff \
--from-empty \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Raw migration file content:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
echo "Cleaning migration file..."
# Clean the file
sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
# Verify the migration file
if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
echo "ERROR: Migration file is empty after cleaning"
echo "Original content was:"
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
exit 1
fi
echo "Final migration file content:"
cat deploy/migrations/${VERSION}_initial/migration.sql
# Verify it starts with SQL
if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
echo "ERROR: Migration file does not start with SQL command or comment"
echo "First line is:"
head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
echo "Full content is:"
cat deploy/migrations/${VERSION}_initial/migration.sql
exit 1
fi
echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
fi
- name: Compare and Generate Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create temporary migration workspace
mkdir -p temp_migrations
# Copy existing migrations (will not fail if directory is empty)
cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
VERSION=$(date +%Y%m%d%H%M%S)
# Generate diff against existing migrations or empty state
prisma migrate diff \
--from-migrations temp_migrations \
--to-schema-datamodel schema.prisma \
--shadow-database-url "${SHADOW_DATABASE_URL}" \
--script > temp_migrations/migration_${VERSION}.sql
# Check if there are actual changes
if [ -s temp_migrations/migration_${VERSION}.sql ]; then
echo "Changes detected, creating new migration"
mkdir -p deploy/migrations/${VERSION}_schema_update
mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
else
echo "No schema changes detected"
exit 0
fi
- name: Verify Migration
if: success()
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
run: |
# Create test database
psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
# Apply all migrations in order to verify
for migration in deploy/migrations/*/migration.sql; do
echo "Applying migration: $migration"
psql "${SHADOW_DATABASE_URL}" -f $migration
done
# Add this step before create-pull-request to debug permissions
- name: Check Token Permissions
run: |
echo "Checking token permissions..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm/collaborators
echo "\nChecking if token can create PRs..."
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
https://api.github.com/repos/BerriAI/litellm
# Add this debug step before git push
- name: Debug Changed Files
run: |
echo "Files staged for commit:"
git diff --name-status --staged
echo "\nAll changed files:"
git status
- name: Create Pull Request
if: success()
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: "chore: update prisma migrations"
title: "Update Prisma Migrations"
body: |
Auto-generated migration based on schema.prisma changes.
Generated files:
- deploy/migrations/${VERSION}_schema_update/migration.sql
- deploy/migrations/${VERSION}_schema_update/README.md
branch: feat/prisma-migration-${{ env.VERSION }}
base: main
delete-branch: true
- name: Generate and Save Migrations
run: |
# Only add migration files
git add deploy/migrations/
git status # Debug what's being committed
git commit -m "chore: update prisma migrations"

94
.github/workflows/publish_enterprise.yml vendored
View file

@ -1,94 +0,0 @@
name: Publish litellm-enterprise to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
pull-requests: write
defaults:
run:
working-directory: enterprise
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump and create PR
id: create-pr
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
git checkout -b "$BRANCH"
git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push origin "$BRANCH" --force
gh pr create \
--title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
--body "Version bump for litellm-enterprise. Merge to update main." \
--head "$BRANCH" \
--base main \
|| true
PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
run: |
gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
run: |
pip install twine
twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

74
.github/workflows/publish_proxy_extras.yml vendored
View file

@ -1,74 +0,0 @@
name: Publish litellm-proxy-extras to PyPI
on:
workflow_dispatch:
inputs:
bump:
description: "Version bump type"
required: true
default: "patch"
type: choice
options:
- patch
- minor
- major
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
defaults:
run:
working-directory: litellm-proxy-extras
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install Poetry
run: pip install poetry
- name: Bump version
id: bump
run: |
OLD=$(poetry version -s)
poetry version ${{ github.event.inputs.bump }}
NEW=$(poetry version -s)
echo "old=$OLD" >> $GITHUB_OUTPUT
echo "new=$NEW" >> $GITHUB_OUTPUT
- name: Update version refs in root pyproject.toml and requirements.txt
run: |
OLD=${{ steps.bump.outputs.old }}
NEW=${{ steps.bump.outputs.new }}
sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
- name: Update poetry.lock
working-directory: .
run: poetry lock
- name: Build
run: poetry build
- name: Commit version bump
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
pip install twine
twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

39
.github/workflows/reset_stable.yml vendored
View file

@ -1,39 +0,0 @@
name: Reset litellm_stable branch
on:
release:
types: [published, created]
jobs:
update-stable-branch:
if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Reset litellm_stable_release_branch branch to the release commit
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Fetch all branches and tags
git fetch --all
# Check if the litellm_stable_release_branch branch exists
if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
echo "litellm_stable_release_branch branch exists."
git checkout litellm_stable_release_branch
else
echo "litellm_stable_release_branch branch does not exist. Creating it."
git checkout -b litellm_stable_release_branch
fi
# Reset litellm_stable_release_branch branch to the release commit
git reset --hard $GITHUB_SHA
# Push the updated litellm_stable_release_branch branch
git push origin litellm_stable_release_branch --force

67
.github/workflows/simple_pypi_publish.yml vendored
View file

@ -1,67 +0,0 @@
name: Simple PyPI Publish
on:
workflow_dispatch:
inputs:
version:
description: 'Version to publish (e.g., 1.74.10)'
required: true
type: string
env:
TWINE_USERNAME: __token__
jobs:
publish:
runs-on: ubuntu-latest
if: github.repository == 'BerriAI/litellm'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install toml build wheel twine
- name: Update version in pyproject.toml
run: |
python -c "
import toml
with open('pyproject.toml', 'r') as f:
data = toml.load(f)
data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
with open('pyproject.toml', 'w') as f:
toml.dump(data, f)
print(f'Updated version to ${{ github.event.inputs.version }}')
"
- name: Copy model prices file
run: |
cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- name: Build package
run: |
rm -rf build dist
python -m build
- name: Publish to PyPI
env:
TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
run: |
twine upload dist/*
- name: Output success
run: |
echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/"

1
.github/workflows/stale.yml vendored
View file

@ -7,6 +7,7 @@ on:
jobs:
stale:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v8

5
.github/workflows/test-linting.yml vendored
View file

@ -28,9 +28,12 @@ jobs:
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Check poetry.lock is up to date
run: |
poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
- name: Install dependencies
run: |
poetry lock
poetry install --with dev
- name: Check Black formatting

12
.pre-commit-config.yaml
View file

@ -14,12 +14,12 @@ repos:
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
# - id: black
# name: black
# entry: poetry run black
# language: system
# types: [python]
# files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- id: black
name: black
entry: poetry run black
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:

18
.semgrep/rules/security/no-claude-directory.yml Normal file
View file

@ -0,0 +1,18 @@
rules:
- id: no-claude-directory-committed
message: >
.claude/ directory must not be committed to the repository.
It contains local Claude Code settings (permissions, worktree paths) that are
developer-machine-specific and may expose internal paths or credentials.
Add .claude/ to .gitignore instead.
severity: ERROR
languages: [generic]
paths:
include:
- "/.claude/**"
- "/.claude/*"
pattern-regex: '[\s\S]+'
metadata:
category: security
tags: [supply-chain, secrets]
confidence: HIGH

13
CLAUDE.md
View file

@ -91,6 +91,10 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Async/await patterns throughout
- Type hints required for all public APIs
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
### Testing Strategy
- Unit tests in `tests/test_litellm/`
@ -98,6 +102,8 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
### UI / Backend Consistency
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
@ -134,6 +140,11 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
### Setup Wizard (`litellm/setup_wizard.py`)
- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
@ -150,4 +161,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
**Fix options:**
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

4
Dockerfile
View file

@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
RUN pip install PyJWT==2.12.0 --no-cache-dir
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.

6
README.md
View file

@ -28,6 +28,9 @@
<a href="https://www.litellm.ai/support">
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
</a>
<a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
<img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
</a>
</h4>
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
@ -399,7 +402,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
# Enterprise
For companies that need better security, user management and professional support
[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
[Talk to founders](https://enterprise.litellm.ai/demo)
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
@ -449,7 +452,6 @@ All these checks must pass before your PR can be merged.
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- [Community Slack 💭](https://www.litellm.ai/support)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
# Why did we build this

15
ci_cd/security_scans.sh
View file

@ -10,13 +10,13 @@ echo "Starting security scans for LiteLLM..."
# Function to install Trivy and required tools
install_trivy() {
echo "Installing Trivy and required tools..."
TRIVY_VERSION="0.35.0"
sudo apt-get update
sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
sudo apt-get update
sudo apt-get install trivy
echo "Trivy and required tools installed successfully"
sudo apt-get install -y wget jq curl bsdmainutils
wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb"
sudo dpkg -i trivy.deb
rm trivy.deb
echo "Trivy ${TRIVY_VERSION} installed successfully"
}
# Function to install Grype
@ -163,6 +163,9 @@ run_grype_scans() {
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
"CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
"CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
)
# Build JSON array of allowlisted CVE IDs for jq

2
cookbook/benchmark/readme.md
View file

@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
```
## Support
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

1
cookbook/codellama-server/README.MD
View file

@ -143,7 +143,6 @@ All responses from the server are returned in the following format (for all LLM
- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
## Roadmap

1
cookbook/litellm_proxy_server/readme.md
View file

@ -164,7 +164,6 @@ All responses from the server are returned in the following format (for all LLM
- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
## Roadmap

2
docker/Dockerfile.custom_ui
View file

@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

4
docker/Dockerfile.database
View file

@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -112,7 +112,7 @@ RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_au
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
RUN pip install PyJWT==2.12.0 --no-cache-dir
# Build Admin UI (runtime stage)
# Convert Windows line endings to Unix and make executable

4
docker/Dockerfile.dev
View file

@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
# Fix JWT dependency conflicts early
RUN pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install PyJWT==2.9.0 --no-cache-dir
pip install PyJWT==2.12.0 --no-cache-dir
# Copy only necessary files for build
COPY pyproject.toml README.md schema.prisma poetry.lock ./
@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

6
docker/Dockerfile.non_root
View file

@ -32,7 +32,7 @@ RUN for i in 1 2 3; do \
# Cache Python dependencies
COPY requirements.txt .
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
&& pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
&& pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
# Copy source after dependency layers
COPY . .
@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -198,7 +198,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
rm -rf /wheels && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \

17
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
View file

@ -3,18 +3,9 @@ slug: anthropic_advanced_features
title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)"
date: 2025-11-25T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
hide_table_of_contents: false
@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter.
{/* truncate */}
---
| Feature | Supported Models |

15
docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md
View file

@ -3,18 +3,9 @@ slug: anthropic-wildcard-model-access-incident
title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
date: 2026-02-23T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
tags: [incident-report, proxy, auth, model-access]
hide_table_of_contents: false
---

24
docs/my-website/blog/authors.yml
View file

@ -4,6 +4,12 @@ litellm:
url: https://github.com/BerriAI/litellm
image_url: https://github.com/BerriAI.png
sameer:
name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
krrish:
name: Krrish Dholakia
title: CEO, LiteLLM
@ -22,3 +28,21 @@ ishaan-alt:
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
ryan:
name: Ryan Crabbe
title: Performance Engineer, LiteLLM
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
alexsander:
name: Alexsander Hamir
title: Performance Engineer, LiteLLM
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
yuneng:
name: Yuneng Jiang
title: SWE @ LiteLLM (Full Stack)
url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
image_url: https://avatars.githubusercontent.com/u/171294688?v=4

17
docs/my-website/blog/claude_code_beta_headers/index.md
View file

@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident
title: "Incident Report: Invalid beta headers with Claude Code"
date: 2026-02-16T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- sameer
- ishaan-alt
- krrish
tags: [incident-report, anthropic, stability]
hide_table_of_contents: false
---
@ -173,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
## Related documentation
- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide
- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

17
docs/my-website/blog/claude_opus_4_6/index.md
View file

@ -3,18 +3,9 @@ slug: claude_opus_4_6
title: "Day 0 Support: Claude Opus 4.6"
date: 2026-02-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- sameer
- ishaan-alt
- krrish
description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, opus 4.6]
hide_table_of_contents: false
@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
{/* truncate */}
## Docker Image
```bash

12
docs/my-website/blog/claude_sonnet_4_6/index.md
View file

@ -3,14 +3,8 @@ slug: claude_sonnet_4_6
title: "Day 0 Support: Claude Sonnet 4.6"
date: 2026-02-17T10:00:00
authors:
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- ishaan-alt
- krrish
description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, sonnet 4.6]
hide_table_of_contents: false
@ -21,6 +15,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
{/* truncate */}
## Docker Image
```bash

15
docs/my-website/blog/fastapi_middleware_performance/index.mdx
View file

@ -3,18 +3,9 @@ slug: fastapi-middleware-performance
title: "Your Middleware Could Be a Bottleneck"
date: 2026-02-07T10:00:00
authors:
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Ryan Crabbe
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
- krrish
- ishaan-alt
- ryan
description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
tags: [performance, fastapi, middleware]
hide_table_of_contents: false

20
docs/my-website/blog/gemin_3.1/index.md
View file

@ -3,18 +3,9 @@ slug: gemini_3_1_pro
title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
date: 2026-02-19T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
@ -28,6 +19,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
{/* truncate */}
## Deploy this version
<Tabs>
@ -67,7 +60,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
@ -147,4 +140,3 @@ curl -X POST http://localhost:4000/v1/chat/completions \
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |

26
docs/my-website/blog/gemini_3/index.md
View file

@ -3,18 +3,9 @@ slug: gemini_3
title: "DAY 0 Support: Gemini 3 on LiteLLM"
date: 2025-11-19T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr
:::
{/* truncate */}
## Quick Start
<Tabs>
@ -976,8 +969,7 @@ messages.append(response.choices[0].message) # ✅ Includes thought signatures
## Additional Resources
- [Gemini Provider Documentation](../gemini.md)
- [Thought Signatures Guide](../gemini.md#thought-signatures)
- [Reasoning Content Documentation](../../reasoning_content.md)
- [Function Calling Guide](../../function_calling.md)
- [Gemini Provider Documentation](../../docs/providers/gemini)
- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures)
- [Reasoning Content Documentation](../../docs/reasoning_content)
- [Function Calling Guide](../../docs/completion/function_call)

21
docs/my-website/blog/gemini_3_1_flash_lite/index.md
View file

@ -3,18 +3,9 @@ slug: gemini_3_1_flash_lite_preview
title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
date: 2026-03-03T08:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms, supernova]
hide_table_of_contents: false
@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
:::
{/* truncate */}
## Deploy this version
<Tabs>
@ -150,7 +143,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview o
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
@ -172,4 +165,4 @@ LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `th
| `medium` | `medium` | Balanced reasoning for moderate complexity |
| `high` | `high` | Maximum reasoning depth, complex problems |
| `disable` | `minimal` | Disable extended reasoning |
| `none` | `minimal` | No extended reasoning |
| `none` | `minimal` | No extended reasoning |

20
docs/my-website/blog/gemini_3_flash/index.md
View file

@ -3,18 +3,9 @@ slug: gemini_3_flash
title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
date: 2025-12-17T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
:::
{/* truncate */}
## Deploy this version
<Tabs>
@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |

7
docs/my-website/blog/gemini_embedding_2_multimodal/index.md
View file

@ -3,10 +3,7 @@ slug: gemini_embedding_2_multimodal
title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
date: 2025-03-11T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- sameer
description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
tags: [gemini, embeddings, multimodal, vertex ai]
hide_table_of_contents: false
@ -19,6 +16,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
{/* truncate */}
## Supported Input Types
| Modality | Supported Formats |

17
docs/my-website/blog/gpt_5_3_codex/index.md
View file

@ -3,18 +3,9 @@ slug: gpt_5_3_codex
title: "Day 0 Support: GPT-5.3-Codex"
date: 2026-02-24T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
tags: [openai, gpt-5.3-codex, codex, day 0 support]
hide_table_of_contents: false
@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
{/* truncate */}
## Why `phase` matters for GPT-5.3-Codex
`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.

17
docs/my-website/blog/gpt_5_4/index.md
View file

@ -3,18 +3,9 @@ slug: gpt_5_4
title: "Day 0 Support: GPT-5.4"
date: 2026-03-05T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "GPT-5.4 model support in LiteLLM"
tags: [openai, gpt-5.4, completion]
hide_table_of_contents: false
@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports fully GPT-5.4!
{/* truncate */}
## Docker Image
```bash

106
docs/my-website/blog/gpt_5_4_mini_nano/index.md Normal file
View file

@ -0,0 +1,106 @@
---
slug: gpt_5_4_mini_nano
title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
date: 2026-03-17T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
:::note
If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
:::
## Usage
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-5.4-mini
litellm_params:
model: openai/gpt-5.4-mini
api_key: os.environ/OPENAI_API_KEY
- model_name: gpt-5.4-nano
litellm_params:
model: openai/gpt-5.4-nano
api_key: os.environ/OPENAI_API_KEY
```
**2. Start the proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Test it**
```bash
# GPT-5.4-mini
curl -X POST "http://localhost:4000/v1/chat/completions" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.4-mini",
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
# GPT-5.4-nano
curl -X POST "http://localhost:4000/v1/chat/completions" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-5.4-nano",
"messages": [{"role": "user", "content": "What is 2 + 2?"}]
}'
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
from litellm import completion
# GPT-5.4-mini
response = completion(
model="openai/gpt-5.4-mini",
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
print(response.choices[0].message.content)
# GPT-5.4-nano
response = completion(
model="openai/gpt-5.4-nano",
messages=[{"role": "user", "content": "What is 2 + 2?"}],
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## Notes
- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.

78
docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md Normal file
View file

@ -0,0 +1,78 @@
---
slug: guardrail-logging-secret-exposure-incident
title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces"
date: 2026-03-18T10:00:00
authors:
- litellm
tags: [incident-report, security, guardrails]
hide_table_of_contents: false
---
**Date:** March 18, 2026
**Duration:** Unknown
**Severity:** High
**Status:** Resolved
## Summary
When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials.
This information could then propagate to logging and observability surfaces that consume guardrail metadata, including:
- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data
- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend
LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths.
{/* truncate */}
---
## Background
LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry.
When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems.
```mermaid
flowchart TD
inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"]
storeSecrets --> guardrailRuns["3. Custom guardrail runs"]
guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"]
fullDataReturn --> loggingBuild["5. Build guardrail log payload"]
loggingBuild --> spendLogs["6a. Persist to spend logs / UI"]
loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"]
```
---
## Root Cause
The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged.
---
## Impact
This issue required all of the following:
1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`.
2. LiteLLM logged that guardrail response through the standard guardrail logging path.
3. An operator, admin, or telemetry consumer had access to the resulting logs or traces.
When those conditions were met, sensitive values could become visible through:
- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI.
- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans.
- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values.
This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems.
---
## Guidance For Users
- Upgrade to LiteLLM 1.82.3+.
- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period.
- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems.
- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata.

16
docs/my-website/blog/httpx_cache_eviction_incident/index.md
View file

@ -3,17 +3,9 @@ slug: httpx-cache-eviction-incident
title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
date: 2026-02-27T10:00:00
authors:
- name: Ryan Crabbe
title: Performance Engineer, LiteLLM
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- ryan
- ishaan-alt
- krrish
tags: [incident-report, caching, stability]
hide_table_of_contents: false
---
@ -31,6 +23,8 @@ A change to improve Redis connection pool cleanup introduced a regression that c
**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
{/* truncate */}
---
## Background

18
docs/my-website/blog/litellm_observatory/index.md
View file

@ -3,18 +3,9 @@ slug: litellm-observatory
title: "Improve release stability with 24 hour load tests"
date: 2026-02-06T10:00:00
authors:
- name: Alexsander Hamir
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- alexsander
- krrish
- ishaan-alt
description: "How we built a long-running, release-validation system to catch regressions before they reach users."
tags: [testing, observability, reliability, releases]
hide_table_of_contents: false
@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa
This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
{/* truncate */}
---
## Why We Built the Observatory
@ -133,4 +126,3 @@ Reliability is an ongoing investment.
LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
We’ll continue to share those improvements openly as we go.

17
docs/my-website/blog/minimax_m2_5/index.md
View file

@ -3,18 +3,9 @@ slug: minimax_m2_5
title: "Day 0 Support: MiniMax-M2.5"
date: 2026-02-12T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
tags: [minimax, M2.5, llm]
hide_table_of_contents: false
@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
{/* truncate */}
## Supported Models
LiteLLM supports the following MiniMax models:

5
docs/my-website/blog/model_cost_map_incident/index.md
View file

@ -3,10 +3,7 @@ slug: model-cost-map-incident
title: "Incident Report: Invalid model cost map on main"
date: 2026-02-10T10:00:00
authors:
- name: Ishaan Jaffer
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/ishaanjaffer/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- ishaan
tags: [incident-report, stability]
hide_table_of_contents: false
---

18
docs/my-website/blog/realtime_webrtc_http_endpoints/index.md
View file

@ -3,18 +3,9 @@ slug: realtime_webrtc_http_endpoints
title: "Realtime WebRTC HTTP Endpoints"
date: 2026-03-12T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
tags: [realtime, webrtc, proxy, openai]
hide_table_of_contents: false
@ -24,6 +15,8 @@ import WebRTCTester from '@site/src/components/WebRTCTester';
Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
{/* truncate */}
## How it works
![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
@ -116,4 +109,3 @@ A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERS
**Q: What if I get no audio?**
A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.

15
docs/my-website/blog/responses_api_encrypted_content_incident/index.md
View file

@ -3,18 +3,9 @@ slug: responses-api-encrypted-content-incident
title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
date: 2026-02-24T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
tags: [incident-report, proxy, responses-api, load-balancing]
hide_table_of_contents: false
---

709
docs/my-website/blog/security_update_march_2026/index.md Normal file
View file

@ -0,0 +1,709 @@
---
slug: security-update-march-2026
title: "Security Update: Suspected Supply Chain Incident"
date: 2026-03-24T14:00:00
authors:
- krrish
- ishaan-alt
description: "As of 2:00 PM ET on March 24, 2026"
tags: [security, incident-report]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
> **Status:** Active investigation
> **Last updated:** March 25, 2026
> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts.
## TLDR;
- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages have now been removed from PyPI.
- We believe that the compromise originated from the Trivy dependency used in our CI/CD security scanning workflow.
- Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
- We are pausing new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.
## Overview
LiteLLM AI Gateway is investigating a suspected supply chain attack involving unauthorized PyPI package publishes. Current evidence suggests a maintainer's PyPI account may have been compromised and used to distribute malicious code.
At this time, we believe this incident may be linked to the broader [Trivy security compromise](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/), in which stolen credentials were reportedly used to gain unauthorized access to the LiteLLM publishing pipeline.
This investigation is ongoing. Details below may change as we confirm additional findings.
## Confirmed affected versions
The following LiteLLM versions published to PyPI were impacted:
- **v1.82.7**: contained a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
- **v1.82.8**: contained `litellm_init.pth` and a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
If you installed or ran either of these versions, review the recommendations below immediately.
Note: These versions have already been removed from PyPI.
## What happened
Initial evidence suggests the attacker bypassed official CI/CD workflows and uploaded malicious packages directly to PyPI.
These compromised versions appear to have included a credential stealer designed to:
- Harvest secrets by scanning for:
- environment variables
- SSH keys
- cloud provider credentials (AWS, GCP, Azure)
- Kubernetes tokens
- database passwords
- Encrypt and exfiltrate data via a `POST` request to `models.litellm.cloud`, which is **not** an official BerriAI / LiteLLM domain
## Who is affected
You may be affected if **any** of the following are true:
- You installed or upgraded LiteLLM via `pip` on **March 24, 2026**, between **10:39 UTC and 16:00 UTC**
- You ran `pip install litellm` without pinning a version and received **v1.82.7** or **v1.82.8**
- You built a Docker image during this window that included `pip install litellm` without a pinned version
- A dependency in your project pulled in LiteLLM as a transitive, unpinned dependency
(for example through AI agent frameworks, MCP servers, or LLM orchestration tools)
You are **not** affected if any of the following are true:
**LiteLLM AI Gateway/Proxy users:** Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
- You are using **LiteLLM Cloud**
- You are using the official LiteLLM AI Gateway Docker image: `ghcr.io/berriai/litellm`
- You are on **v1.82.6 or earlier** and did not upgrade during the affected window
- You installed LiteLLM from source via the GitHub repository, which was **not** compromised
### How to check if you are affected
<Tabs>
<TabItem value="sdk" label="SDK">
```bash
pip show litellm
```
</TabItem>
<TabItem value="proxy" label="PROXY">
Go to the proxy base url, and check the version of the installed LiteLLM.
![Proxy version check](../../img/security_update_march_2026/proxy_version.png)
</TabItem>
<TabItem value="github" label="GitHub Actions">
Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions.
**Requirements:** Python 3 and `requests` (`pip install requests`).
**Setup:**
```bash
export GITHUB_TOKEN="your-github-pat"
```
**Run:**
```bash
python find_litellm_github.py
```
Set the `ORG` variable in the script to your GitHub organization name.
Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
<details>
<summary>View full script (find_litellm_github.py)</summary>
```python
#!/usr/bin/env python3
"""
Scan all GitHub Actions jobs in a GitHub org that ran between
0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
"""
import io
import os
import re
import sys
import zipfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
import requests
GITHUB_URL = "https://api.github.com"
ORG = "your-org" # <-- set to your GitHub organization
TOKEN = os.environ.get("GITHUB_TOKEN", "")
TODAY = datetime.now(timezone.utc).date()
WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc)
WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
TARGET_VERSIONS = {"1.82.7", "1.82.8"}
VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
SESSION = requests.Session()
SESSION.headers.update({
"Authorization": f"Bearer {TOKEN}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
})
def get_paginated(url, params=None):
params = dict(params or {})
params.setdefault("per_page", 100)
page = 1
while True:
params["page"] = page
resp = SESSION.get(url, params=params, timeout=30)
if resp.status_code == 404:
return
resp.raise_for_status()
data = resp.json()
if isinstance(data, dict):
items = next((v for v in data.values() if isinstance(v, list)), [])
else:
items = data
if not items:
break
yield from items
if len(items) < params["per_page"]:
break
page += 1
def parse_ts(ts_str):
if not ts_str:
return None
return datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
def get_repos():
repos = []
for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}):
repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]})
return repos
def get_runs_in_window(repo_full_name):
created_filter = (
f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}"
f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}"
)
url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs"
runs = []
for run in get_paginated(url, {"created": created_filter, "per_page": 100}):
ts = parse_ts(run.get("run_started_at") or run.get("created_at"))
if ts and WINDOW_START <= ts <= WINDOW_END:
runs.append(run)
return runs
def get_jobs_for_run(repo_full_name, run_id):
url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs"
jobs = []
for job in get_paginated(url, {"filter": "all"}):
ts = parse_ts(job.get("started_at"))
if ts and WINDOW_START <= ts <= WINDOW_END:
jobs.append(job)
return jobs
def fetch_job_log(repo_full_name, job_id):
url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs"
resp = SESSION.get(url, timeout=60, allow_redirects=True)
if resp.status_code in (403, 404, 410):
return ""
resp.raise_for_status()
content_type = resp.headers.get("Content-Type", "")
if "zip" in content_type or resp.content[:2] == b"PK":
try:
with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
parts = []
for name in sorted(zf.namelist()):
with zf.open(name) as f:
parts.append(f.read().decode("utf-8", errors="replace"))
return "\n".join(parts)
except zipfile.BadZipFile:
pass
return resp.text
def check_job(repo_full_name, job):
job_id = job["id"]
job_name = job["name"]
run_id = job["run_id"]
started = job.get("started_at", "")
log_text = fetch_job_log(repo_full_name, job_id)
if not log_text:
return None
found_versions = set()
context_lines = []
for line in log_text.splitlines():
m = VERSION_PATTERN.search(line)
if m:
ver = m.group(1)
if ver in TARGET_VERSIONS:
found_versions.add(ver)
context_lines.append(line.strip())
if not found_versions:
return None
return {
"repo": repo_full_name,
"run_id": run_id,
"job_id": job_id,
"job_name": job_name,
"started_at": started,
"versions": sorted(found_versions),
"context": context_lines[:10],
"job_url": job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"),
}
def main():
if not TOKEN:
print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr)
sys.exit(1)
print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
print()
print(f"Fetching repositories for org '{ORG}'...")
repos = get_repos()
print(f" Found {len(repos)} repositories")
print()
jobs_to_check = []
print("Scanning workflow runs for time window...")
for repo in repos:
full_name = repo["full_name"]
try:
runs = get_runs_in_window(full_name)
except requests.HTTPError as e:
print(f" WARN: {full_name} - {e}", file=sys.stderr)
continue
if not runs:
continue
print(f" {full_name}: {len(runs)} run(s) in window")
for run in runs:
try:
jobs = get_jobs_for_run(full_name, run["id"])
except requests.HTTPError as e:
print(f" WARN: run {run['id']} - {e}", file=sys.stderr)
continue
for job in jobs:
jobs_to_check.append((full_name, job))
total = len(jobs_to_check)
print(f"\nFetching logs for {total} job(s)...")
print()
hits = []
with ThreadPoolExecutor(max_workers=8) as pool:
futures = {
pool.submit(check_job, full_name, job): (full_name, job["id"])
for full_name, job in jobs_to_check
}
done = 0
for future in as_completed(futures):
done += 1
full_name, jid = futures[future]
try:
result = future.result()
except Exception as e:
print(f" ERROR {full_name} job {jid}: {e}", file=sys.stderr)
continue
if result:
hits.append(result)
print(
f" [{done}/{total}] {full_name} job {jid}" +
(f" *** HIT: litellm {result['versions']} ***" if result else ""),
flush=True,
)
print()
print("=" * 72)
print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
print("=" * 72)
if not hits:
print("No matches found.")
return
for h in sorted(hits, key=lambda x: x["started_at"]):
print()
print(f" Repo : {h['repo']}")
print(f" Job : {h['job_name']} (#{h['job_id']})")
print(f" Run ID : {h['run_id']}")
print(f" Started : {h['started_at']}")
print(f" Versions : litellm {', '.join(h['versions'])}")
print(f" URL : {h['job_url']}")
print(f" Log lines :")
for line in h["context"]:
print(f" {line}")
if __name__ == "__main__":
main()
```
</details>
</TabItem>
<TabItem value="gitlab" label="GitLab CI">
Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions.
**Requirements:** Python 3 and `requests` (`pip install requests`).
**Setup:**
```bash
export GITLAB_TOKEN="your-gitlab-pat"
```
**Run:**
```bash
python find_litellm_jobs.py
```
Set the `GROUP_NAME` variable in the script to your GitLab group name.
Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
<details>
<summary>View full script (find_litellm_jobs.py)</summary>
```python
#!/usr/bin/env python3
"""
Scan all GitLab CI/CD jobs in a GitLab group that ran between
0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
"""
import os
import re
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
import requests
GITLAB_URL = "https://gitlab.com"
GROUP_NAME = "YourGroup" # <-- set to your GitLab group name
TOKEN = os.environ.get("GITLAB_TOKEN", "")
TODAY = datetime.now(timezone.utc).date()
WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc)
WINDOW_END = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
TARGET_VERSIONS = {"1.82.7", "1.82.8"}
VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
HEADERS = {"PRIVATE-TOKEN": TOKEN}
SESSION = requests.Session()
SESSION.headers.update(HEADERS)
def get_paginated(url, params=None):
params = dict(params or {})
params.setdefault("per_page", 100)
page = 1
while True:
params["page"] = page
resp = SESSION.get(url, params=params, timeout=30)
resp.raise_for_status()
data = resp.json()
if not data:
break
yield from data
if len(data) < params["per_page"]:
break
page += 1
def get_group_id(group_name):
resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30)
resp.raise_for_status()
return resp.json()["id"]
def get_all_projects(group_id):
projects = []
for p in get_paginated(
f"{GITLAB_URL}/api/v4/groups/{group_id}/projects",
{"include_subgroups": "true", "archived": "false"},
):
projects.append({"id": p["id"], "name": p["path_with_namespace"]})
return projects
def parse_ts(ts_str):
if not ts_str:
return None
ts_str = ts_str.replace("Z", "+00:00")
return datetime.fromisoformat(ts_str)
def jobs_in_window(project_id):
matching = []
url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs"
params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]}
page = 1
while True:
params["page"] = page
resp = SESSION.get(url, params=params, timeout=30)
if resp.status_code == 403:
return matching
resp.raise_for_status()
jobs = resp.json()
if not jobs:
break
stop_early = False
for job in jobs:
ts = parse_ts(job.get("started_at") or job.get("created_at"))
if ts is None:
continue
if ts > WINDOW_END:
continue
if ts < WINDOW_START:
stop_early = True
continue
matching.append(job)
if stop_early or len(jobs) < 100:
break
page += 1
return matching
def fetch_trace(project_id, job_id):
url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace"
resp = SESSION.get(url, timeout=60)
if resp.status_code in (403, 404):
return ""
resp.raise_for_status()
return resp.text
def check_job(project_name, project_id, job):
job_id = job["id"]
job_name = job["name"]
ref = job.get("ref", "")
started = job.get("started_at", job.get("created_at", ""))
trace = fetch_trace(project_id, job_id)
if not trace:
return None
found_versions = set()
for match in VERSION_PATTERN.finditer(trace):
ver = match.group(1)
if ver in TARGET_VERSIONS:
found_versions.add(ver)
if not found_versions:
return None
context_lines = []
for line in trace.splitlines():
if VERSION_PATTERN.search(line):
ver_match = VERSION_PATTERN.search(line)
if ver_match and ver_match.group(1) in TARGET_VERSIONS:
context_lines.append(line.strip())
return {
"project": project_name,
"project_id": project_id,
"job_id": job_id,
"job_name": job_name,
"ref": ref,
"started_at": started,
"versions": sorted(found_versions),
"context": context_lines[:10],
"job_url": f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}",
}
def main():
if not TOKEN:
print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr)
sys.exit(1)
print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
print()
print(f"Resolving group '{GROUP_NAME}'...")
group_id = get_group_id(GROUP_NAME)
print("Fetching projects...")
projects = get_all_projects(group_id)
print(f" Found {len(projects)} projects")
print()
all_jobs_to_check = []
print("Scanning job listings for time window...")
for proj in projects:
try:
jobs = jobs_in_window(proj["id"])
except requests.HTTPError as e:
print(f" WARN: {proj['name']} - {e}", file=sys.stderr)
continue
if jobs:
print(f" {proj['name']}: {len(jobs)} job(s) in window")
for j in jobs:
all_jobs_to_check.append((proj["name"], proj["id"], j))
total = len(all_jobs_to_check)
print(f"\nFetching traces for {total} job(s)...")
print()
hits = []
with ThreadPoolExecutor(max_workers=10) as pool:
futures = {
pool.submit(check_job, pname, pid, job): (pname, job["id"])
for pname, pid, job in all_jobs_to_check
}
done = 0
for future in as_completed(futures):
done += 1
pname, jid = futures[future]
try:
result = future.result()
except Exception as e:
print(f" ERROR checking {pname} job {jid}: {e}", file=sys.stderr)
continue
if result:
hits.append(result)
print(f" [{done}/{total}] checked {pname} job {jid}" +
(f" *** HIT: litellm {result['versions']} ***" if result else ""),
flush=True)
print()
print("=" * 72)
print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
print("=" * 72)
if not hits:
print("No matches found.")
return
for h in sorted(hits, key=lambda x: x["started_at"]):
print()
print(f" Project : {h['project']}")
print(f" Job : {h['job_name']} (#{h['job_id']})")
print(f" Branch/tag: {h['ref']}")
print(f" Started : {h['started_at']}")
print(f" Versions : litellm {', '.join(h['versions'])}")
print(f" URL : {h['job_url']}")
print(f" Log lines :")
for line in h["context"]:
print(f" {line}")
if __name__ == "__main__":
main()
```
</details>
</TabItem>
</Tabs>
*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.*
## Indicators of compromise (IoCs)
Review affected systems for the following indicators:
- `litellm_init.pth` present in your `site-packages`
- Outbound traffic or requests to `models.litellm[.]cloud`
This domain is **not** affiliated with LiteLLM
## Immediate actions for affected users
If you installed or ran **v1.82.7** or **v1.82.8**, take the following actions immediately.
### 1. Rotate all secrets
Treat any credentials present on the affected systems as compromised, including:
- API keys
- Cloud access keys
- Database passwords
- SSH keys
- Kubernetes tokens
- Any secrets stored in environment variables or configuration files
### 2. Inspect your filesystem
Check your `site-packages` directory for a file named `litellm_init.pth`:
```bash
find /usr/lib/python3.13/site-packages/ -name "litellm_init.pth"
```
If present:
- remove it immediately
- investigate the host for further compromise
- preserve relevant artifacts if your security team is performing forensics
### 3. Audit version history
Review your:
- Local environments
- CI/CD pipelines
- Docker builds
- Deployment logs
Confirm whether **v1.82.7** or **v1.82.8** was installed anywhere.
Pin LiteLLM to a known safe version such as **v1.82.6 or earlier**, or to a later verified release once announced.
## Response and remediation
The LiteLLM AI Gateway team has already taken the following steps:
- Removed compromised packages from PyPI
- Rotated maintainer credentials and established new authorized maintainers
- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain
## Questions and support
If you believe your systems may be affected, contact us immediately:
- **Security:** `security@berri.ai`
- **Support:** `support@berri.ai`
- **Slack:** Reach out to the LiteLLM team directly
For real-time updates, follow [LiteLLM (YC W23) on X](https://x.com/LiteLLM).

14
docs/my-website/blog/server_root_path/index.md
View file

@ -3,17 +3,9 @@ slug: server-root-path-incident
title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
date: 2026-02-21T10:00:00
authors:
- name: Yuneng Jiang
title: SWE @ LiteLLM (Full Stack)
url: https://www.linkedin.com/in/yunengjiang/
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- yuneng
- ishaan-alt
- krrish
tags: [incident-report, ui, stability]
hide_table_of_contents: false
---

17
docs/my-website/blog/sub_millisecond_proxy_overhead/index.md
View file

@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead
title: "Achieving Sub-Millisecond Proxy Overhead"
date: 2026-02-02T10:00:00
authors:
- name: Alexsander Hamir
title: "Performance Engineer, LiteLLM"
url: https://www.linkedin.com/in/alexsander-baptista/
image_url: https://github.com/AlexsanderHamir.png
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- alexsander
- krrish
- ishaan-alt
description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
tags: [performance, architecture]
hide_table_of_contents: false
@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o
To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
{/* truncate */}
---
## Where We're Coming From

121
docs/my-website/blog/video_characters_litellm/index.md Normal file
View file

@ -0,0 +1,121 @@
---
slug: video_characters_api
title: "New Video Characters, Edit and Extension API support"
date: 2026-03-16T10:00:00
authors:
- sameer
- krrish
- ishaan-alt
description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
tags: [videos, characters, proxy, routing]
hide_table_of_contents: false
---
LiteLLM now supoports videos character, edit and extension apis.
{/* truncate */}
## What's New
Four new endpoints for video character operations:
- **Create character** - Upload a video to create a reusable asset
- **Get character** - Retrieve character metadata
- **Edit video** - Modify generated videos
- **Extend video** - Continue clips with character consistency
**Available from:** LiteLLM v1.83.0+
## Quick Example
```python
import litellm
# Create character from video
character = litellm.avideo_create_character(
name="Luna",
video=open("luna.mp4", "rb"),
custom_llm_provider="openai",
model="sora-2"
)
print(f"Character: {character.id}")
# Use in generation
video = litellm.avideo(
model="sora-2",
prompt="Luna dances through a magical forest.",
characters=[{"id": character.id}],
seconds="8"
)
# Get character info
fetched = litellm.avideo_get_character(
character_id=character.id,
custom_llm_provider="openai"
)
# Edit with character preserved
edited = litellm.avideo_edit(
video_id=video.id,
prompt="Add warm golden lighting"
)
# Extend sequence
extended = litellm.avideo_extension(
video_id=video.id,
prompt="Luna waves goodbye",
seconds="5"
)
```
## Via Proxy
```bash
# Create character
curl -X POST "http://localhost:4000/v1/videos/characters" \
-H "Authorization: Bearer sk-litellm-key" \
-F "video=@luna.mp4" \
-F "name=Luna"
# Get character
curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
-H "Authorization: Bearer sk-litellm-key"
# Edit video
curl -X POST "http://localhost:4000/v1/videos/edits" \
-H "Authorization: Bearer sk-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"video": {"id": "video_xyz789"},
"prompt": "Add warm golden lighting and enhance colors"
}'
# Extend video
curl -X POST "http://localhost:4000/v1/videos/extensions" \
-H "Authorization: Bearer sk-litellm-key" \
-H "Content-Type: application/json" \
-d '{
"video": {"id": "video_xyz789"},
"prompt": "Luna waves goodbye and walks into the sunset",
"seconds": "5"
}'
```
## Managed Character IDs
LiteLLM automatically encodes provider and model metadata into character IDs:
**What happens:**
```
Upload character "Luna" with model "sora-2" on OpenAI
↓
LiteLLM creates: char_abc123def456 (contains provider + model_id)
↓
When you reference it later, LiteLLM decodes automatically
↓
Router knows exactly which deployment to use
```
**Behind the scenes:**
- Character ID format: `character_<base64_encoded_metadata>`
- Metadata includes: provider, model_id, original_character_id
- Transparent to you - just use the ID, LiteLLM handles routing

15
docs/my-website/blog/vllm_embeddings_incident/index.md
View file

@ -3,18 +3,9 @@ slug: vllm-embeddings-incident
title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
date: 2026-02-18T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
- sameer
- krrish
- ishaan-alt
tags: [incident-report, embeddings, vllm]
hide_table_of_contents: false
---

9
docs/my-website/docs/anthropic_unified/index.md
View file

@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the
A system prompt providing context or specific instructions to the model.
- **temperature** (number):
Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
- **thinking** (object):
- **thinking** (object):
Configuration for enabling extended thinking. If enabled, it includes:
- **budget_tokens** (integer):
- **budget_tokens** (integer):
Minimum of 1024 tokens (and less than `max_tokens`).
- **type** (enum):
- **type** (enum):
E.g., `"enabled"`.
- **summary** (string, optional):
Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`.
When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API.
- **tool_choice** (object):
Instructs how the model should utilize any provided tools.
- **tools** (array of objects):

206
docs/my-website/docs/completion/prompt_caching.md
View file

@ -6,6 +6,8 @@ import TabItem from '@theme/TabItem';
Supported Providers:
- OpenAI (`openai/`)
- Anthropic API (`anthropic/`)
- Google AI Studio (`gemini/`)
- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
- Deepseek API (`deepseek/`)
@ -257,7 +259,7 @@ Anthropic charges for cache writes.
Specify the content to cache with `"cache_control": {"type": "ephemeral"}`.
If you pass that in for any other llm provider, it will be ignored.
This same format also works for [Gemini / Vertex AI](#google-ai-studio--vertex-ai-gemini-example). For other providers, it will be ignored.
<Tabs>
<TabItem value="sdk" label="SDK">
@ -356,6 +358,208 @@ print(response.usage)
</TabItem>
</Tabs>
### Google AI Studio / Vertex AI (Gemini) Example
Use the same Anthropic-style `cache_control` format — LiteLLM automatically translates it to Google's [context caching API](https://ai.google.dev/api/caching).
**How it works under the hood:**
1. Messages with `cache_control` are separated and sent to Google's `cachedContents` API
2. The cached content ID is then passed as `cachedContent` in the Gemini request body
3. Works across all three providers: `gemini/` (Google AI Studio), `vertex_ai/`, and `vertex_ai_beta/`
4. Requires a minimum of **1024 tokens** in the cached content — below that, caching is silently skipped
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = ""
response = completion(
model="gemini/gemini-2.5-flash",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are an AI assistant tasked with analyzing legal documents.",
},
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 400,
"cache_control": {"type": "ephemeral"},
},
],
},
{
"role": "user",
"content": "what are the key terms and conditions in this agreement?",
},
],
)
print(response.usage)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python
from openai import OpenAI
client = OpenAI(
api_key="LITELLM_PROXY_KEY", # sk-1234
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
)
response = client.chat.completions.create(
model="gemini-2.5-flash",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are an AI assistant tasked with analyzing legal documents.",
},
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 400,
"cache_control": {"type": "ephemeral"},
},
],
},
{
"role": "user",
"content": "what are the key terms and conditions in this agreement?",
},
],
)
print(response.usage)
```
</TabItem>
</Tabs>
#### Vertex AI
For Vertex AI, use `vertex_ai/` prefix:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-2.5-flash",
vertex_project="my-gcp-project",
vertex_location="us-central1",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are an AI assistant tasked with analyzing legal documents.",
},
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 400,
"cache_control": {"type": "ephemeral"},
},
],
},
{
"role": "user",
"content": "what are the key terms and conditions in this agreement?",
},
],
)
print(response.usage)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gemini-2.5-flash
litellm_params:
model: vertex_ai/gemini-2.5-flash
vertex_project: my-gcp-project
vertex_location: us-central1
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python
from openai import OpenAI
client = OpenAI(
api_key="LITELLM_PROXY_KEY", # sk-1234
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
)
response = client.chat.completions.create(
model="gemini-2.5-flash",
messages=[
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are an AI assistant tasked with analyzing legal documents.",
},
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 400,
"cache_control": {"type": "ephemeral"},
},
],
},
{
"role": "user",
"content": "what are the key terms and conditions in this agreement?",
},
],
)
print(response.usage)
```
</TabItem>
</Tabs>
### Deepeek Example
Works the same as OpenAI.

2
docs/my-website/docs/data_security.md
View file

@ -128,8 +128,6 @@ We'll review all reports promptly. Note that we don't currently offer a bug boun
Legal Entity Name: Berrie AI Incorporated
Company Phone Number: 7708783106
Point of contact email address for security incidents: krrish@berri.ai
Point of contact email address for general security-related questions: krrish@berri.ai

2
docs/my-website/docs/debugging/local_debugging.md
View file

@ -67,6 +67,6 @@ response = completion("command-nightly", messages, logger_fn=my_custom_logging_f
## Still Seeing Issues?
Text us @ +17708783106 or Join the [Discord](https://discord.com/invite/wuPM9dRgDw).
Join the [Discord](https://discord.com/invite/wuPM9dRgDw).
We promise to help you in `lite`ning speed ❤️

6
docs/my-website/docs/enterprise.md
View file

@ -4,7 +4,7 @@ import Image from '@theme/IdealImage';
:::info
- ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs.
- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://enterprise.litellm.ai/demo) to discuss your needs.
:::
For companies that need SSO, user management and professional support for LiteLLM Proxy
@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o
### What’s the cost of the Self-Managed Enterprise edition?
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://enterprise.litellm.ai/demo)
### How does deployment with Enterprise License work?
@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr
Pricing is based on usage. We can figure out a price that works for your team, on the call.
[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
[**Contact Us to learn more**](https://enterprise.litellm.ai/demo)

2
docs/my-website/docs/fine_tuning.md
View file

@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
:::info
This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
This is an Enterprise only endpoint [Get Started with Enterprise here](https://enterprise.litellm.ai/demo)
:::

78
docs/my-website/docs/guides/index.md Normal file
View file

@ -0,0 +1,78 @@
---
title: Guides
sidebar_label: Overview
---
import NavigationCards from '@site/src/components/NavigationCards';
**Guides** are focused references organized by the job you are trying to do with LiteLLM: make requests, use tools, handle media, manage context, or operate the gateway safely.
> New to LiteLLM or not sure whether you need the SDK or Gateway path first? Start at [Learn →](/docs/learn)
---
## Build With LiteLLM
<NavigationCards
columns={3}
items={[
{
icon: "⚡",
title: "Core Requests",
description: "Streaming, batching, structured outputs, and reasoning behavior.",
to: "/docs/guides/core_request_response_patterns",
},
{
icon: "🛠️",
title: "Tool Calling",
description: "Function calling, web tools, interception patterns, computer use, code interpreter, and tool-call hygiene.",
to: "/docs/guides/tools_integrations",
},
{
icon: "🖼️",
title: "Multimodal I/O",
description: "Vision, audio, PDFs, image generation, and video generation.",
to: "/docs/guides/multimodal_io",
},
{
icon: "📚",
title: "Retrieval & Knowledge",
description: "Vector stores, file search, citations, and knowledge-base routing.",
to: "/docs/guides/retrieval_knowledge",
},
{
icon: "🧠",
title: "Prompts & Context",
description: "Prompt caching, trimming, formatting, assistant prefill, and predicted outputs.",
to: "/docs/guides/prompts_context",
},
]}
/>
---
## Operate & Extend
<NavigationCards
columns={3}
items={[
{
icon: "🎛️",
title: "Compatibility & Extensibility",
description: "Provider-specific params, model aliases, fine-tuned models, and adapters.",
to: "/docs/guides/compatibility_extensibility",
},
{
icon: "🧪",
title: "Reliability, Testing & Spend",
description: "Retries, fallbacks, mock responses, and budget controls.",
to: "/docs/guides/reliability_testing_spend",
},
{
icon: "🔒",
title: "Security & Network",
description: "SSL, custom CA bundles, HTTP proxy settings, and per-service verification.",
to: "/docs/guides/security_network",
},
]}
/>

1051
docs/my-website/docs/index.md
View file

File diff suppressed because it is too large Load diff

340
docs/my-website/docs/integrations/index.md
View file

@ -1,18 +1,336 @@
# Integrations
---
title: Integrations
sidebar_label: Overview
---
import NavigationCards from '@site/src/components/NavigationCards';
This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
## AI Agent Frameworks
- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
---
## Development Tools
- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
## Observability
## Observability & Monitoring
- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
- **[Datadog](../observability/datadog.md)**
Track, debug, and analyze LLM calls with observability platforms.
<NavigationCards
columns={3}
items={[
{
icon: "🪢",
title: "Langfuse",
description: "LLM observability and analytics.",
to: "/docs/observability/langfuse_integration",
},
{
icon: "🐶",
title: "Datadog",
description: "Metrics, traces, and dashboards.",
to: "/docs/observability/datadog",
},
{
icon: "📡",
title: "OpenTelemetry",
description: "Vendor-neutral tracing.",
to: "/docs/observability/opentelemetry_integration",
},
{
icon: "🔗",
title: "LangSmith",
description: "LLM debugging and evaluation.",
to: "/docs/observability/langsmith_integration",
},
{
icon: "🔥",
title: "Arize / Phoenix",
description: "ML observability and evaluation.",
to: "/docs/observability/arize_integration",
},
{
icon: "🌀",
title: "Helicone",
description: "LLM request logging and analytics.",
to: "/docs/observability/helicone_integration",
},
{
icon: "📊",
title: "MLflow",
description: "Experiment tracking.",
to: "/docs/observability/mlflow",
},
{
icon: "🏋️",
title: "Weights & Biases",
description: "ML experiment tracking.",
to: "/docs/observability/wandb_integration",
},
{
icon: "📉",
title: "PostHog",
description: "Product analytics.",
to: "/docs/observability/posthog_integration",
},
]}
/>
Click into each section to learn more about the integrations.
[View all observability integrations →](/docs/integrations/observability_integrations)
---
## Alerting & Monitoring
Set up alerts, metrics collection, and infrastructure monitoring.
<NavigationCards
columns={2}
items={[
{
icon: "📈",
title: "Prometheus",
description: "Metrics collection and monitoring.",
to: "../proxy/prometheus",
},
{
icon: "🚨",
title: "PagerDuty",
description: "Incident response and alerting.",
to: "../proxy/pagerduty",
},
{
icon: "🔔",
title: "Alerting",
description: "Slack, Teams, and webhook alerts.",
to: "../proxy/alerting",
},
{
icon: "🔍",
title: "Pyroscope",
description: "Continuous profiling.",
to: "../proxy/pyroscope_profiling",
},
]}
/>
---
## Guardrail Providers
Add safety and content filtering to LLM calls.
<NavigationCards
columns={3}
items={[
{
icon: "🛡️",
title: "Lakera AI",
description: "Prompt injection detection.",
to: "/docs/proxy/guardrails/lakera_ai",
},
{
icon: "☁️",
title: "Azure Content Safety",
description: "Content moderation.",
to: "/docs/proxy/guardrails/azure_content_guardrail",
},
{
icon: "🛏️",
title: "Bedrock Guardrails",
description: "AWS Bedrock safety.",
to: "/docs/proxy/guardrails/bedrock",
},
{
icon: "🤖",
title: "OpenAI Moderation",
description: "OpenAI content policy.",
to: "/docs/proxy/guardrails/openai_moderation",
},
{
icon: "🔐",
title: "Secret Detection",
description: "Prevent credential leaks.",
to: "/docs/proxy/guardrails/secret_detection",
},
{
icon: "🕵️",
title: "PII Masking",
description: "Mask sensitive data.",
to: "/docs/proxy/guardrails/pii_masking_v2",
},
]}
/>
[View all guardrail providers →](/docs/guardrail_providers)
---
## Policies
Define and enforce usage policies across your LLM deployment.
<NavigationCards
columns={3}
items={[
{
icon: "📋",
title: "Guardrail Policies",
description: "Policy-based guardrail rules.",
to: "../proxy/guardrails/guardrail_policies",
},
{
icon: "🔀",
title: "Policy Flow Builder",
description: "Visual policy configuration.",
to: "../proxy/guardrails/policy_flow_builder",
},
{
icon: "📄",
title: "Policy Templates",
description: "Pre-built policy templates.",
to: "../proxy/guardrails/policy_templates",
},
]}
/>
---
## AI Tools
Connect LiteLLM to AI-powered coding and productivity tools.
<NavigationCards
columns={3}
items={[
{
icon: "💬",
title: "OpenWebUI",
description: "Self-hosted ChatGPT-style interface.",
to: "../tutorials/openweb_ui",
},
{
icon: "🤖",
title: "Claude Code",
description: "Use LiteLLM with Claude Code.",
to: "../tutorials/claude_responses_api",
},
{
icon: "🖱️",
title: "Cursor",
description: "AI code editor integration.",
to: "../tutorials/cursor_integration",
},
{
icon: "🐙",
title: "GitHub Copilot",
description: "GitHub Copilot integration.",
to: "../tutorials/github_copilot_integration",
},
{
icon: "💻",
title: "OpenCode",
description: "Open source coding assistant.",
to: "../tutorials/opencode_integration",
},
{
icon: "🔧",
title: "Retool Assist",
description: "Retool AI assistant.",
to: "../tutorials/retool_assist",
},
]}
/>
---
## Agent SDKs
Use LiteLLM with agent frameworks and SDKs.
<NavigationCards
columns={3}
items={[
{
icon: "🤖",
title: "OpenAI Agents SDK",
description: "Build agents with OpenAI's SDK.",
to: "../tutorials/openai_agents_sdk",
},
{
icon: "🧠",
title: "Claude Agent SDK",
description: "Build agents with Anthropic's SDK.",
to: "../tutorials/claude_agent_sdk",
},
{
icon: "🌐",
title: "Google ADK",
description: "Google Agent Development Kit.",
to: "../tutorials/google_adk",
},
{
icon: "🚀",
title: "CopilotKit",
description: "In-app AI copilots.",
to: "../tutorials/copilotkit_sdk",
},
{
icon: "🧬",
title: "Letta",
description: "Build stateful LLM agents with persistent memory.",
to: "./letta",
},
{
icon: "🎙️",
title: "LiveKit",
description: "Real-time voice and video AI agents.",
to: "../tutorials/livekit_xai_realtime",
},
]}
/>
---
## Prompt Management
Manage, version, and deploy prompts.
<NavigationCards
columns={3}
items={[
{
icon: "📝",
title: "LiteLLM Prompt Management",
description: "Built-in prompt management.",
to: "../proxy/litellm_prompt_management",
},
{
icon: "🔌",
title: "Custom Prompt Management",
description: "Bring your own prompt store.",
to: "../proxy/custom_prompt_management",
},
{
icon: "🔥",
title: "Arize Phoenix Prompts",
description: "Prompt management with Phoenix.",
to: "../proxy/arize_phoenix_prompts",
},
]}
/>
---
## Manage with AI Agents
Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language.
<NavigationCards
columns={1}
items={[
{
icon: "🤖",
title: "LiteLLM Skills",
description: "Manage LiteLLM via Claude Code — create keys, teams, models, and more using natural language commands.",
to: "../tutorials/claude_code_skills",
},
]}
/>

12
docs/my-website/docs/integrations/letta.md
View file

@ -920,9 +920,9 @@ for model in models:
## Resources
- [Letta Documentation](https://docs.letta.ai/)
- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
- [LiteLLM SDK Documentation](../completion/input.md)
- [Function Calling Guide](../completion/function_call.md)
- [Observability Setup](../observability/langfuse_integration.md)
- [Router Configuration](../routing.md)
- [Letta Documentation](https://docs.letta.com/)
- [LiteLLM Proxy Documentation](/docs/simple_proxy)
- [LiteLLM SDK Documentation](/docs/#litellm-python-sdk)
- [Function Calling Guide](/docs/completion/function_call)
- [Observability Setup](/docs/integrations/observability_integrations)
- [Router Configuration](/docs/routing)

28
docs/my-website/docs/integrations/observability_index.md Normal file
View file

@ -0,0 +1,28 @@
---
title: Observability
sidebar_label: Overview
slug: observability_integrations
---
Track, debug, and analyze LLM calls with observability platforms.
import NavigationCards from '@site/src/components/NavigationCards';
## Observability Integrations
<NavigationCards
columns={3}
items={[
{ icon: "🪢", title: "Langfuse", description: "LLM observability and analytics.", to: "/docs/observability/langfuse_integration" },
{ icon: "🐶", title: "Datadog", description: "Metrics, traces, and dashboards.", to: "/docs/observability/datadog" },
{ icon: "📡", title: "OpenTelemetry", description: "Vendor-neutral tracing.", to: "/docs/observability/opentelemetry_integration" },
{ icon: "🔗", title: "LangSmith", description: "LLM debugging and evaluation.", to: "/docs/observability/langsmith_integration" },
{ icon: "🔥", title: "Arize / Phoenix", description: "ML observability and evaluation.", to: "/docs/observability/arize_integration" },
{ icon: "🌀", title: "Helicone", description: "LLM request logging and analytics.", to: "/docs/observability/helicone_integration" },
{ icon: "📊", title: "MLflow", description: "Experiment tracking.", to: "/docs/observability/mlflow" },
{ icon: "🏋️", title: "Weights & Biases", description: "ML experiment tracking.", to: "/docs/observability/wandb_integration" },
{ icon: "📉", title: "PostHog", description: "Product analytics.", to: "/docs/observability/posthog_integration" },
]}
/>
[View all observability integrations →](/docs/observability/callbacks)

2
docs/my-website/docs/integrations/websearch_interception.md
View file

@ -375,7 +375,7 @@ search_tools:
- [Search Providers](../search/index.md) - Detailed search provider setup
- [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code
- [Tool Calling](../completion/function_call.md) - General tool calling documentation
- [Callbacks](./custom_callback.md) - Custom callback documentation
- [Callbacks](../observability/custom_callback.md) - Custom callback documentation
## Technical Details

174
docs/my-website/docs/learn/gateway_quickstart.md Normal file
View file

@ -0,0 +1,174 @@
---
title: Gateway Quickstart
sidebar_label: Gateway Quickstart
description: Start LiteLLM Gateway, add models and keys, then connect applications and SDKs to one shared endpoint.
---
import NavigationCards from '@site/src/components/NavigationCards';
Use this path if you need one shared OpenAI-compatible endpoint for a team or platform.
If you need a Docker or database-first setup, use the [Docker + Database tutorial](/docs/proxy/docker_quick_start). Otherwise, use the steps below to get to a working request fast.
## 1. Install The Gateway
```bash
pip install 'litellm[proxy]'
```
## 2. Set One Provider Key
```bash
export OPENAI_API_KEY="your-api-key"
```
## 3. Create `config.yaml`
```yaml
model_list:
- model_name: gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234
```
## 4. Start The Gateway
```bash
litellm --config config.yaml
```
You should see the proxy start on `http://0.0.0.0:4000`.
## 5. Send Your First Request
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "Hello from LiteLLM Gateway"}
]
}'
```
## 6. Check The Response
If the request succeeds, the proxy returns `200 OK` with an OpenAI-style response.
The assistant text will be in:
```json
choices[0].message.content
```
If your gateway is routing to OpenAI, a real response can look like this:
```json
{
"id": "chatcmpl-abc123",
"created": 1677858242,
"model": "gpt-4o-mini-2024-07-18",
"object": "chat.completion",
"system_fingerprint": "fp_406d6473f8",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "Hello! How can I assist you today?",
"tool_calls": null,
"function_call": null,
"annotations": []
}
}
],
"usage": {
"completion_tokens": 9,
"prompt_tokens": 13,
"total_tokens": 22,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
}
},
"service_tier": "default"
}
```
`id`, `created`, the resolved model version, token counts, and message text will vary by request. Other providers may return a smaller or slightly different set of fields, but `choices[0].message.content` is the main field to read.
## 7. Add Keys And The UI
If you need virtual keys, spend tracking, or the admin UI, add a database next.
- Add `database_url` under `general_settings`
- Use [Virtual keys](/docs/proxy/virtual_keys) for key creation and budgets
- Use [Admin UI](/docs/proxy/ui) to manage models and keys
- Use the [Docker + Database tutorial](/docs/proxy/docker_quick_start) if you want a fuller setup
## 8. Pick Your Next Step
<NavigationCards
columns={3}
items={[
{
icon: "🖥️",
title: "Make LLM Requests",
description: "Point LiteLLM or OpenAI-compatible clients to the gateway.",
to: "/docs/proxy/user_keys",
},
{
icon: "🎛️",
title: "Model Config",
description: "Add more models and gateway settings.",
to: "/docs/proxy/configs",
},
{
icon: "🔑",
title: "Virtual Keys",
description: "Create keys, budgets, and access controls.",
to: "/docs/proxy/virtual_keys",
},
{
icon: "📈",
title: "Add Logging",
description: "Capture logs, spend, and traces.",
to: "/docs/proxy/logging",
},
{
icon: "🔀",
title: "Load Balance",
description: "Route across deployments, regions, or providers.",
to: "/docs/proxy/load_balancing",
},
{
icon: "🛡️",
title: "Add Guardrails",
description: "Add safety checks and policy enforcement.",
to: "/docs/proxy/guardrails/quick_start",
},
{
icon: "📊",
title: "Reliability",
description: "Configure retries, fallbacks, and timeouts.",
to: "/docs/proxy/reliability",
},
]}
/>
## When To Use The SDK Path Instead
If you only need to call models from one application and do not need centralized auth or shared infrastructure, start with the [SDK Quickstart](/docs/learn/sdk_quickstart) instead.

117
docs/my-website/docs/learn/index.md Normal file
View file

@ -0,0 +1,117 @@
---
title: Learn LiteLLM
sidebar_label: Learn
slug: /learn
---
import NavigationCards from '@site/src/components/NavigationCards';
LiteLLM gives you one OpenAI-compatible interface for 100+ LLM providers. Start with the path that matches your setup.
---
## Start Here
Pick one path first.
<NavigationCards
columns={2}
items={[
{
icon: "🐍",
title: "SDK Quickstart",
description: "Use LiteLLM directly in application code.",
listDescription: [
"Install",
"First request",
"Next SDK features",
],
to: "/docs/learn/sdk_quickstart",
},
{
icon: "🖥️",
title: "Gateway Quickstart",
description: "Run LiteLLM as a shared gateway.",
listDescription: [
"Start proxy",
"Add models and keys",
"Connect clients",
],
to: "/docs/learn/gateway_quickstart",
},
]}
/>
---
## Common Tasks
Jump to a specific task.
<NavigationCards
columns={3}
items={[
{
icon: "⚡",
title: "Stream Responses",
description: "Return tokens as they are generated.",
to: "/docs/guides/core_request_response_patterns",
},
{
icon: "🧰",
title: "Use Tools",
description: "Add function calling to your app.",
to: "/docs/guides/tools_integrations",
},
{
icon: "🔀",
title: "Add Routing",
description: "Retries, fallbacks, and load balancing.",
to: "/docs/routing-load-balancing",
},
{
icon: "🔑",
title: "Set Up Keys",
description: "Gateway auth, virtual keys, and access control.",
to: "/docs/proxy/virtual_keys",
},
{
icon: "📈",
title: "Add Logging",
description: "Capture request logs and spend data.",
to: "/docs/proxy/logging",
},
{
icon: "🌐",
title: "Choose A Provider",
description: "Find provider-specific auth and params.",
to: "/docs/providers",
},
]}
/>
---
## Docs Map
Use these when you already know the type of doc you want.
<NavigationCards
columns={2}
items={[
{
icon: "📚",
title: "Guides",
description: "Feature reference.",
to: "/docs/guides",
},
{
icon: "🛠️",
title: "Tutorials",
description: "Step-by-step integrations.",
to: "/docs/tutorials",
},
]}
/>
Not sure where to start? Use [SDK Quickstart](/docs/learn/sdk_quickstart) for app code or [Gateway Quickstart](/docs/learn/gateway_quickstart) for shared infrastructure.

174
docs/my-website/docs/learn/sdk_quickstart.md Normal file
View file

@ -0,0 +1,174 @@
---
title: SDK Quickstart
sidebar_label: SDK Quickstart
description: Make your first LiteLLM SDK call, then jump to the right docs for the next feature you need.
---
import NavigationCards from '@site/src/components/NavigationCards';
Use this path if you are integrating LiteLLM directly into application code.
## 1. Install LiteLLM
```bash
pip install litellm
```
## 2. Set Provider Credentials
Start with one provider and set its environment variables.
- OpenAI: `OPENAI_API_KEY`
- Anthropic: `ANTHROPIC_API_KEY`
- Azure OpenAI: `AZURE_API_KEY`, `AZURE_API_BASE`, `AZURE_API_VERSION`
- Bedrock: standard AWS credentials
- Vertex AI: `VERTEXAI_PROJECT`, `VERTEXAI_LOCATION`
If you have not picked a provider yet, browse [all supported providers](/docs/providers).
## 3. Make Your First Call
```python
from litellm import completion
import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
response = completion(
model="openai/gpt-4o",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
print(response.choices[0].message.content)
```
## 4. Check The Response
The line below:
```python
print(response.choices[0].message.content)
```
prints the assistant text, for example:
```text
Hello! I'm doing well, thanks for asking.
```
If you print the full object with:
```python
print(response)
```
you will see a Python `ModelResponse(...)` object. For an OpenAI-backed model, it can look like this:
```python
ModelResponse(
id='chatcmpl-abc123',
created=1773782130,
model='gpt-4o-2024-08-06',
object='chat.completion',
system_fingerprint='fp_4ff89bf575',
choices=[
Choices(
finish_reason='stop',
index=0,
message=Message(
content="Hello! I'm just a program, but I'm here to help you. How can I assist you today?",
role='assistant',
tool_calls=None,
function_call=None,
provider_specific_fields={'refusal': None},
annotations=[]
),
provider_specific_fields={}
)
],
usage=Usage(
completion_tokens=21,
prompt_tokens=13,
total_tokens=34,
completion_tokens_details=CompletionTokensDetailsWrapper(...),
prompt_tokens_details=PromptTokensDetailsWrapper(...)
),
service_tier='default'
)
```
The same response follows an OpenAI-style shape. Conceptually, it looks like this:
```json
{
"id": "chatcmpl-abc123",
"object": "chat.completion",
"created": 1677858242,
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hello! I'm doing well, thanks for asking."
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 13,
"completion_tokens": 12,
"total_tokens": 25
}
}
```
`id`, `created`, token counts, and message text will vary by request.
If you call an OpenAI-backed model, you may also see extra fields such as `system_fingerprint`, `service_tier`, `tool_calls`, `function_call`, `annotations`, `provider_specific_fields`, and detailed token usage. For the full output reference, see [completion output](/docs/completion/output).
Need more provider examples? See the main [Getting Started](/docs/#quick-start) page.
## 5. Pick Your Next Step
<NavigationCards
columns={3}
items={[
{
icon: "⚡",
title: "Stream Responses",
description: "Receive tokens incrementally with stream=True.",
to: "/docs/completion/stream",
},
{
icon: "🧰",
title: "Use Tools",
description: "Add function calling in a provider-agnostic way.",
to: "/docs/completion/function_call",
},
{
icon: "📦",
title: "Return JSON",
description: "Constrain responses to structured JSON output.",
to: "/docs/completion/json_mode",
},
{
icon: "🔀",
title: "Add Routing",
description: "Use retries, fallbacks, and load balancing in app code.",
to: "/docs/routing",
},
{
icon: "🌐",
title: "Choose A Provider",
description: "Find provider-specific auth, model naming, and params.",
to: "/docs/providers",
},
]}
/>
## When To Use Gateway Instead
Use LiteLLM Gateway if you need centralized auth, virtual keys, spend tracking, shared logging, or one OpenAI-compatible endpoint for multiple apps.
[Go to Gateway Quickstart →](/docs/learn/gateway_quickstart)

4
docs/my-website/docs/load_test_advanced.md
View file

@ -11,7 +11,7 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
- [Github releases](https://github.com/BerriAI/litellm/releases)
- [litellm docker containers](https://github.com/BerriAI/litellm/pkgs/container/litellm)
- [litellm database docker container](https://github.com/BerriAI/litellm/pkgs/container/litellm-database)
- [ ] Ensure you're following **ALL** [best practices for production](./proxy/production_setup.md)
- [ ] Ensure you're following **ALL** [best practices for production](./proxy/prod.md)
- [ ] Locust - Ensure you're Locust instance can create 1K+ requests per second
- 👉 You can use our **[maintained locust instance here](https://locust-load-tester-production.up.railway.app/)**
- If you're self hosting locust
@ -222,4 +222,4 @@ class MyUser(HttpUser):
def on_start(self):
self.api_key = os.getenv('API_KEY', 'sk-1234')
self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
```
```

294
docs/my-website/docs/mcp_zero_trust.md Normal file
View file

@ -0,0 +1,294 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MCP Zero Trust Auth (JWT Signer)
![Zero Trust MCP Gateway](/img/mcp_zero_trust_gateway.png)
MCP servers have no built-in way to verify that a request actually came through LiteLLM. Without this guardrail, any client that can reach your MCP server directly can call tools — bypassing your access controls entirely.
`MCPJWTSigner` fixes this. It signs every outbound tool call with a short-lived RS256 JWT. Your MCP server verifies the signature against LiteLLM's public key. Requests that didn't go through LiteLLM have no valid signature and are rejected.
---
## Basic setup
Add the guardrail to your config and point your MCP server at LiteLLM's JWKS endpoint. Every tool call gets a signed JWT automatically — no changes needed on the client side.
```yaml title="config.yaml"
mcp_servers:
- server_name: weather
url: http://localhost:8000/mcp
transport: http
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
issuer: "https://my-litellm.example.com" # defaults to request base URL
audience: "mcp" # default: "mcp"
ttl_seconds: 300 # default: 300
```
**Bring your own signing key** — recommended for production. Auto-generated keys are lost on restart.
```bash
export MCP_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\n..."
# or point to a file
export MCP_JWT_SIGNING_KEY="file:///secrets/mcp-signing-key.pem"
```
**Build a verified MCP server with [FastMCP](https://gofastmcp.com):**
```python title="weather_server.py"
from fastmcp import FastMCP, Context
from fastmcp.server.auth.providers.jwt import JWTVerifier
auth = JWTVerifier(
jwks_uri="https://my-litellm.example.com/.well-known/jwks.json",
issuer="https://my-litellm.example.com",
audience="mcp",
algorithm="RS256",
)
mcp = FastMCP("weather-server", auth=auth)
@mcp.tool()
async def get_weather(city: str, ctx: Context) -> str:
caller = ctx.client_id # JWT `sub` — the verified user identity
return f"Weather in {city}: sunny, 72°F (requested by {caller})"
if __name__ == "__main__":
mcp.run(transport="http", host="0.0.0.0", port=8000)
```
FastMCP fetches the JWKS automatically and re-fetches when the signing key changes.
LiteLLM publishes OIDC discovery so MCP servers find the key without any manual configuration:
```
GET /.well-known/openid-configuration → { "jwks_uri": "https://<litellm>/.well-known/jwks.json" }
GET /.well-known/jwks.json → { "keys": [{ "kty": "RSA", "alg": "RS256", ... }] }
```
> **Read further only if you need to:** thread a corporate IdP identity into the JWT, enforce specific claims on callers, add custom metadata, use AWS Bedrock AgentCore Gateway, or debug JWT rejections.
---
## Thread IdP identity into MCP JWTs
By default the outbound JWT `sub` is LiteLLM's internal `user_id`. If your users authenticate with Okta, Azure AD, or another IdP, the MCP server sees a LiteLLM-internal ID — not the user's email or employee ID.
With verify+re-sign, LiteLLM validates the incoming IdP token first, then builds the outbound JWT using the real identity claims from that token. The MCP server gets the user's actual identity without ever having to trust the original IdP directly.
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
issuer: "https://my-litellm.example.com"
# Validate the incoming Bearer token against the IdP
access_token_discovery_uri: "https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration"
verify_issuer: "https://login.microsoftonline.com/{tenant}/v2.0"
verify_audience: "api://my-app"
# Which claim to use for `sub` in the outbound JWT — first non-empty value wins
end_user_claim_sources:
- "token:sub" # from the verified incoming JWT
- "token:email" # fallback to email
- "litellm:user_id" # last resort: LiteLLM's internal user_id
```
If the incoming token is **opaque** (not a JWT — some IdPs issue these), add an introspection endpoint. LiteLLM will POST the token to it (RFC 7662) and use the returned claims:
```yaml
token_introspection_endpoint: "https://idp.example.com/oauth2/introspect"
```
**Supported `end_user_claim_sources` values:**
| Source | Resolves to |
|--------|-------------|
| `token:<claim>` | Any claim from the verified incoming JWT (e.g. `token:sub`, `token:email`, `token:oid`) |
| `litellm:user_id` | LiteLLM's internal user ID |
| `litellm:email` | User email from LiteLLM auth context |
| `litellm:end_user_id` | End-user ID if set separately |
| `litellm:team_id` | Team ID from LiteLLM auth context |
---
## Block callers missing required attributes
Some MCP servers expose sensitive operations that should only be reachable by verified employees — not service accounts, not external API keys. You can enforce this at the LiteLLM layer so the MCP server never receives the request at all.
`required_claims` rejects with `403` if the incoming token is missing any listed claim. `optional_claims` forwards claims that are useful but not mandatory.
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
access_token_discovery_uri: "https://idp.example.com/.well-known/openid-configuration"
# Service accounts without `employee_id` are blocked before the tool runs
required_claims:
- "sub"
- "employee_id"
# Forward these into the outbound JWT when present — skipped silently if absent
optional_claims:
- "groups"
- "department"
```
**What the client sees when blocked:**
```json
HTTP 403
{ "error": "MCPJWTSigner: incoming token is missing required claims: ['employee_id']. Configure the IdP to include these claims." }
```
---
## Add custom metadata to every JWT
Your MCP server may need context that LiteLLM doesn't carry natively — which deployment sent the request, a tenant ID, an environment tag. Use claim operations to inject, override, or strip claims from the outbound JWT.
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
# add: insert only when the key is not already in the JWT
add_claims:
deployment_id: "prod-us-east-1"
tenant_id: "acme-corp"
# set: always override — even if the claim came from the incoming token
set_claims:
env: "production"
# remove: strip claims the MCP server shouldn't see
remove_claims:
- "nbf" # some validators reject nbf; remove it if yours does
```
Operations run in order — `add_claims` → `set_claims` → `remove_claims`. `set_claims` always wins over `add_claims`; `remove_claims` beats both.
---
## AWS Bedrock AgentCore Gateway
Bedrock AgentCore Gateway uses two separate JWTs: one to authenticate the transport connection and another to authorize tool calls. They need different `aud` values and TTLs — a single JWT won't work for both.
LiteLLM can issue both in one hook and inject them into separate headers:
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
issuer: "https://my-litellm.example.com"
audience: "mcp-resource" # for the MCP resource layer
ttl_seconds: 300
# Second JWT for the transport channel — same sub/act/scope, different aud + TTL
channel_token_audience: "bedrock-agentcore-gateway"
channel_token_ttl: 60 # transport tokens should be short-lived
```
LiteLLM injects two headers on every tool call:
- `Authorization: Bearer <resource-token>` — audience `mcp-resource`, TTL 300s
- `x-mcp-channel-token: Bearer <channel-token>` — audience `bedrock-agentcore-gateway`, TTL 60s
Both tokens are signed with the same LiteLLM key, so your MCP server only needs to trust one JWKS endpoint.
---
## Control which scopes go into the JWT
By default LiteLLM generates least-privilege scopes per request:
- Tool call → `mcp:tools/call mcp:tools/{name}:call`
- List tools → `mcp:tools/call mcp:tools/list`
If your MCP server does its own scope enforcement and needs a specific format, set `allowed_scopes` to replace auto-generation entirely:
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
allowed_scopes:
- "mcp:tools/call"
- "mcp:tools/list"
- "mcp:admin"
```
Every JWT carries exactly those scopes regardless of which tool is being called.
---
## Debug JWT rejections
Your MCP server is returning 401 and you're not sure what's in the JWT. Enable `debug_headers` and LiteLLM adds a `x-litellm-mcp-debug` response header with the key claims that were signed:
```yaml title="config.yaml"
guardrails:
- guardrail_name: mcp-jwt-signer
litellm_params:
guardrail: mcp_jwt_signer
mode: pre_mcp_call
default_on: true
debug_headers: true
```
Response header:
```
x-litellm-mcp-debug: v=1; kid=a3f1b2c4d5e6f708; sub=alice@corp.com; iss=https://my-litellm.example.com; exp=1712345678; scope=mcp:tools/call mcp:tools/get_weather:call
```
Check that `kid` matches what the MCP server fetched from JWKS, `iss`/`aud` match your server's expected values, and `exp` hasn't passed. Disable in production — the header leaks claim metadata.
---
## JWT claims reference
| Claim | Value |
|-------|-------|
| `iss` | `issuer` config value (or request base URL) |
| `aud` | `audience` config value (default: `"mcp"`) |
| `sub` | Resolved via `end_user_claim_sources` (default: `user_id` → api-key hash → `"litellm-proxy"`) |
| `act.sub` | `team_id` → `org_id` → `"litellm-proxy"` (RFC 8693 delegation) |
| `email` | `user_email` from LiteLLM auth context (when available) |
| `scope` | Auto-generated per tool call, or `allowed_scopes` when set |
| `iat`, `exp`, `nbf` | Standard timing claims (RFC 7519) |
---
## Limitations
- **OpenAPI-backed MCP servers** (`spec_path` set) do not support JWT injection. LiteLLM logs a warning and skips the header. Use SSE/HTTP transport servers to get full JWT injection.
- The keypair is **in-memory by default** and rotated on each restart unless `MCP_JWT_SIGNING_KEY` is set. FastMCP's `JWTVerifier` handles key rotation transparently via JWKS key ID matching.
---
## Related
- [MCP Guardrails](./mcp_guardrail) — PII masking and blocking for MCP calls
- [MCP OAuth](./mcp_oauth) — upstream OAuth2 for MCP server access
- [MCP AWS SigV4](./mcp_aws_sigv4) — AWS-signed requests to MCP servers

3
docs/my-website/docs/migration.md
View file

@ -31,5 +31,4 @@ When we have breaking changes (i.e. going from 1.x.x to 2.x.x), we will document
**How can we communicate changes better?**
Tell us
- [Discord](https://discord.com/invite/wuPM9dRgDw)
- Email (krrish@berri.ai/ishaan@berri.ai)
- Text us (+17708783106)
- Email (support@berri.ai)

1
docs/my-website/docs/observability/arize_integration.md
View file

@ -194,5 +194,4 @@ print(response)
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

3
docs/my-website/docs/observability/gcs_bucket_integration.md
View file

@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
:::info
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
✨ This is an Enterprise only feature [Get Started with Enterprise here](https://enterprise.litellm.ai/demo)
:::
@ -79,5 +79,4 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/langfuse_integration.md
View file

@ -342,5 +342,4 @@ Be aware that if you are continuing an existing trace, and you set `update_trace
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

6
docs/my-website/docs/observability/langfuse_otel_integration.md
View file

@ -83,6 +83,9 @@ os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# Or use self-hosted instance
# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers
# os.environ["OTEL_IGNORE_CONTEXT_PROPAGATION"] = "true"
litellm.callbacks = ["langfuse_otel"]
```
@ -124,6 +127,9 @@ export LANGFUSE_PUBLIC_KEY="pk-lf-..."
export LANGFUSE_SECRET_KEY="sk-lf-..."
export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region
# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint
# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers
# export OTEL_IGNORE_CONTEXT_PROPAGATION="true"
```
2. Setup config.yaml

1
docs/my-website/docs/observability/langsmith_integration.md
View file

@ -225,5 +225,4 @@ environment_variables:
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/logfire_integration.md
View file

@ -63,5 +63,4 @@ response = litellm.completion(
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/lunary_integration.md
View file

@ -176,5 +176,4 @@ You can find more details about the different ways of making requests to the Lit
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/opik_integration.md
View file

@ -261,5 +261,4 @@ All requests made with this key will automatically be tracked in the "TestProjec
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/phoenix_integration.md
View file

@ -127,5 +127,4 @@ Depending on which Phoenix Cloud version or deployment you are using, you should
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/promptlayer_integration.md
View file

@ -84,5 +84,4 @@ Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https:
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/slack_integration.md
View file

@ -101,5 +101,4 @@ response = litellm.completion(
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/sumologic_integration.md
View file

@ -328,5 +328,4 @@ If you get authentication errors, regenerate the HTTP Source URL in Sumo Logic:
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

1
docs/my-website/docs/observability/supabase_integration.md
View file

@ -105,5 +105,4 @@ litellm.modify_integration("supabase",{"table_name": "litellm_logs"})
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

148
docs/my-website/docs/observability/vantage.md Normal file
View file

@ -0,0 +1,148 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vantage Integration
LiteLLM can export proxy spend data to [Vantage](https://vantage.sh) as [FOCUS 1.2](https://focus.finops.org/) formatted cost reports. This lets you visualize LLM spend alongside your cloud infrastructure costs in the Vantage dashboard.
## Overview
| Property | Details |
|----------|---------|
| Destination | Export LiteLLM usage data to Vantage Custom Provider |
| Data format | FOCUS CSV (automatically transformed from LiteLLM spend data) |
| Supported operations | Manual export, automatic scheduled export (hourly/daily/interval) |
| Authentication | Vantage API key + Custom Provider token |
## Prerequisites
You need two credentials from the [Vantage console](https://console.vantage.sh):
1. **API Key** — Go to **Settings → API Access Tokens** → Create a token with **Write** scope. The token looks like `vntg_tkn_...`.
2. **Custom Provider Token** — Go to **Settings → Integrations** → Create a **Custom Provider** integration → Copy the Provider ID (looks like `accss_crdntl_...`).
## Setup via API
The recommended setup uses the proxy admin endpoints. No config file changes needed.
### 1. Initialize credentials
```bash
curl -X POST http://localhost:4000/vantage/init \
-H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
-H "Content-Type: application/json" \
-d '{
"api_key": "vntg_tkn_YOUR_VANTAGE_API_KEY",
"integration_token": "accss_crdntl_YOUR_PROVIDER_TOKEN"
}'
```
Credentials are encrypted and stored in the proxy database.
### 2. Preview data (dry run)
```bash
curl -X POST http://localhost:4000/vantage/dry-run \
-H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
-H "Content-Type: application/json" \
-d '{"limit": 10}'
```
This returns FOCUS-transformed data without sending anything to Vantage. Use it to verify the pipeline works and inspect the data mapping.
### 3. Export to Vantage
```bash
curl -X POST http://localhost:4000/vantage/export \
-H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
-H "Content-Type: application/json" \
-d '{}'
```
Optional parameters:
- `limit` — Max number of records to export
- `start_time_utc` / `end_time_utc` — Filter by time range (must be provided together)
### 4. Verify in Vantage
Go to **Settings → Integrations → your Custom Provider → Import Costs** tab to see uploaded CSVs. Once the status changes from "Importing and Processing" to "Stable", costs appear in **Cost Reporting → All Resources**.
## Setup via Environment Variables
For automatic scheduled exports, configure via environment variables and proxy config:
### Environment variables
| Variable | Required | Description |
|----------|----------|-------------|
| `VANTAGE_API_KEY` | Yes | Vantage API access token |
| `VANTAGE_INTEGRATION_TOKEN` | Yes | Custom Provider token from Vantage dashboard |
| `VANTAGE_BASE_URL` | No | API URL override (default: `https://api.vantage.sh`) |
| `VANTAGE_EXPORT_FREQUENCY` | No | `hourly` (default), `daily`, or `interval` |
| `VANTAGE_EXPORT_INTERVAL_SECONDS` | No | Seconds between exports when frequency is `interval` |
### Proxy config
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-your-key
litellm_settings:
callbacks: ["vantage"]
```
```bash
export VANTAGE_API_KEY="vntg_tkn_..."
export VANTAGE_INTEGRATION_TOKEN="accss_crdntl_..."
litellm --config /path/to/config.yaml
```
The proxy registers a background job that exports data on the configured schedule.
## API Endpoints
All endpoints require admin authentication.
| Method | Endpoint | Description |
|--------|----------|-------------|
| `POST` | `/vantage/init` | Store Vantage credentials (encrypted) |
| `GET` | `/vantage/settings` | View current config (credentials masked) |
| `PUT` | `/vantage/settings` | Update credentials or base URL |
| `POST` | `/vantage/dry-run` | Preview FOCUS data without uploading |
| `POST` | `/vantage/export` | Upload cost data to Vantage |
| `DELETE` | `/vantage/delete` | Remove credentials and stop scheduled exports |
## FOCUS Field Mapping
LiteLLM spend data is transformed into the FOCUS 1.2 schema:
| LiteLLM Field | FOCUS Column | Description |
|---------------|-------------|-------------|
| `spend` | BilledCost, EffectiveCost | Cost of the usage |
| `model` | ChargeDescription, ResourceId | Model identifier |
| `model_group` | ServiceName | Model group / deployment |
| `custom_llm_provider` | ProviderName, PublisherName | Provider (openai, anthropic, etc.) |
| `api_key` | BillingAccountId | Hashed API key |
| `api_key_alias` | BillingAccountName | Human-readable key alias |
| `team_id` | SubAccountId | Team identifier |
| `team_alias` | SubAccountName | Team name |
Additional metadata (user_id, model_group, etc.) is included in the `Tags` column as JSON.
## Upload Limits
Vantage enforces per-upload limits. LiteLLM handles these automatically:
- **10,000 rows** per upload — large exports are split into batches
- **2 MB** per upload — oversized batches are further split by size
- **Unsupported columns** are stripped before upload
## Related Links
- [Vantage](https://vantage.sh)
- [Vantage Custom Providers](https://docs.vantage.sh/connecting_custom_providers)
- [FOCUS Specification](https://focus.finops.org/)
- [Focus Export (S3/Parquet)](./focus.md)

1
docs/my-website/docs/observability/wandb_integration.md
View file

@ -57,5 +57,4 @@ response = litellm.completion(
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

48
docs/my-website/docs/prompt_management.md Normal file
View file

@ -0,0 +1,48 @@
---
title: Prompt Management with Responses API
---
# Prompt Management with Responses API
Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`.
## Basic Usage
```bash
curl -X POST "http://localhost:4000/v1/responses" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"prompt_id": "my-responses-prompt",
"prompt_variables": {"topic": "large language models"},
"input": []
}'
```
## Multi-turn Follow-up in `input`
To send follow-up turns in one request, pass message history in `input`.
```bash
curl -X POST "http://localhost:4000/v1/responses" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o",
"prompt_id": "my-responses-prompt",
"prompt_variables": {"topic": "large language models"},
"input": [
{"role": "user", "content": "Topic is LLMs. Start short."},
{"role": "assistant", "content": "Sure, go ahead."},
{"role": "user", "content": "Now give me 3 bullets and include pricing caveat."}
]
}'
```
## Notes
- Prompt template messages are merged with your `input` messages.
- Prompt variable substitution applies to prompt message content.
- Tool call payload fields are not substituted by prompt variables.
- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn.

95
docs/my-website/docs/providers/aws_sagemaker.md
View file

@ -526,3 +526,98 @@ print(f"response: {response}")
```
## Nova Models on SageMaker
LiteLLM supports Amazon Nova models (Nova Micro, Nova Lite, Nova 2 Lite) deployed on SageMaker Inference real-time endpoints. These custom/fine-tuned Nova models use an OpenAI-compatible API format.
**Reference:** [AWS Blog - Amazon SageMaker Inference for Custom Amazon Nova Models](https://aws.amazon.com/blogs/aws/announcing-amazon-sagemaker-inference-for-custom-amazon-nova-models/)
### Usage
Use the `sagemaker_nova/` prefix with your SageMaker endpoint name:
```python
import litellm
import os
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = "us-east-1"
# Basic chat completion
response = litellm.completion(
model="sagemaker_nova/my-nova-endpoint",
messages=[{"role": "user", "content": "Hello, how are you?"}],
temperature=0.7,
max_tokens=512,
)
print(response.choices[0].message.content)
```
### Streaming
```python
response = litellm.completion(
model="sagemaker_nova/my-nova-endpoint",
messages=[{"role": "user", "content": "Write a short poem"}],
stream=True,
stream_options={"include_usage": True},
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
### Multimodal (Images)
Nova models on SageMaker support image inputs using base64 data URIs:
```python
response = litellm.completion(
model="sagemaker_nova/my-nova-endpoint",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
]
}
],
)
```
### Proxy Config
```yaml
model_list:
- model_name: nova-micro
litellm_params:
model: sagemaker_nova/my-nova-micro-endpoint
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
### Supported Parameters
All standard OpenAI parameters are supported, plus these Nova-specific parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `top_k` | integer | Limits token selection to top K most likely tokens |
| `reasoning_effort` | `"low"` \| `"high"` | Reasoning effort level (Nova 2 Lite custom models only) |
| `allowed_token_ids` | array[int] | Restrict output to specified token IDs |
| `truncate_prompt_tokens` | integer | Truncate prompt to N tokens if it exceeds limit |
```python
response = litellm.completion(
model="sagemaker_nova/my-nova-endpoint",
messages=[{"role": "user", "content": "Think step by step: what is 2+2?"}],
top_k=40,
reasoning_effort="low",
logprobs=True,
top_logprobs=2,
)
```

2
docs/my-website/docs/providers/azure/azure.md
View file

@ -1032,7 +1032,7 @@ print("list_batches_response=", list_batches_response)
</TabItem>
</Tabs>
### [Health Check Azure Batch models](./proxy/health.md#batch-models-azure-only)
### [Health Check Azure Batch models](../../proxy/health.md#batch-models-azure-only)
### [BETA] Loadbalance Multiple Azure Deployments

5
docs/my-website/docs/providers/azure/azure_anthropic.md
View file

@ -372,7 +372,6 @@ response = completion(
## Related Documentation
- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage
- [Anthropic Provider Documentation](../anthropic.md) - For standard Anthropic API usage
- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models
- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup
- [Azure Authentication Guide](../../secret_managers/azure_key_vault.md) - For Azure AD token setup

Some files were not shown because too many files have changed in this diff Show more