scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Merge remote-tracking branch 'origin' into litellm_email_budget_alerts_2

Browse source
This commit is contained in:
yuneng-jiang 2025-12-17 11:37:49 -08:00
commit 087dfbf648
233 changed files with 16399 additions and 1085 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -23,13 +23,15 @@ body:
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
- type: dropdown
id: ml-ops-team
id: component
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: What part of LiteLLM is this about?
options:
- "No"
- "Yes"
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: input

12
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -22,6 +22,18 @@ body:
description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
validations:
required: true
- type: dropdown
id: component
attributes:
label: What part of LiteLLM is this about?
options:
- "SDK (litellm Python package)"
- "Proxy"
- "UI Dashboard"
- "Docs"
- "Other"
validations:
required: true
- type: dropdown
id: hiring-interest
attributes:

5
.github/pull_request_template.md vendored
View file

@ -1,7 +1,3 @@
## Title
<!-- e.g. "Implement user authentication feature" -->
## Relevant issues
<!-- e.g. "Fixes #000" -->
@ -11,7 +7,6 @@
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] I have added a screenshot of my new test passing locally
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem

43
.github/workflows/create_daily_staging_branch.yml vendored Normal file
View file

@ -0,0 +1,43 @@
name: Create Daily Staging Branch
on:
schedule:
- cron: '0 0 * * *' # Runs daily at midnight UTC
workflow_dispatch: # Allow manual trigger
jobs:
create-staging-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Create daily staging branch
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Configure Git user
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Generate branch name with MM_DD_YYYY format
BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')"
echo "Creating branch: $BRANCH_NAME"
# Fetch all branches
git fetch --all
# Check if the branch already exists
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists. Skipping creation."
else
echo "Creating new branch: $BRANCH_NAME"
# Create the new branch from main
git checkout -b $BRANCH_NAME origin/main
# Push the new branch
git push origin $BRANCH_NAME
echo "Successfully created and pushed branch: $BRANCH_NAME"
fi

2
.github/workflows/issue-keyword-labeler.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
id: scan
env:
PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
run: python3 .github/scripts/scan_keywords.py
- name: Ensure label exists

144
.github/workflows/label-component.yml vendored Normal file
View file

@ -0,0 +1,144 @@
name: Label Component Issues
on:
issues:
types:
- opened
jobs:
add-component-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Add SDK label
if: contains(github.event.issue.body, 'SDK (litellm Python package)')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'sdk';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add Proxy label
if: contains(github.event.issue.body, 'Proxy')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'proxy';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add UI Dashboard label
if: contains(github.event.issue.body, 'UI Dashboard')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'ui-dashboard';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
- name: Add Docs label
if: contains(github.event.issue.body, 'Docs')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'docs';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});

17
.github/workflows/label-mlops.yml vendored
View file

@ -1,17 +0,0 @@
name: Label ML Ops Team Issues
on:
issues:
types:
- opened
jobs:
add-mlops-label:
runs-on: ubuntu-latest
steps:
- name: Check if ML Ops Team is selected
uses: actions-ecosystem/action-add-labels@v1
if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
labels: "mlops user request"

2
deploy/charts/litellm-helm/README.md
View file

@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |

46
docker-compose.hardened.yml Normal file
View file

@ -0,0 +1,46 @@
services:
# Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
# Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
litellm:
build:
context: .
dockerfile: docker/Dockerfile.non_root
target: runtime
args:
PROXY_EXTRAS_SOURCE: "local"
depends_on:
- squid
user: "101:101"
group_add:
- "2345"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
volumes:
- ./proxy_server_config.yaml:/app/config.yaml:ro
environment:
LITELLM_NON_ROOT: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db"
command:
- "--port"
- "4000"
- "--config"
- "/app/config.yaml"
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
ports:
- "3128:3128"
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m

2
docker-compose.yml
View file

@ -4,7 +4,7 @@ services:
context: .
args:
target: runtime
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:

4
docker/Dockerfile.alpine
View file

@ -34,8 +34,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime
# Update dependencies and clean up
RUN apk upgrade --no-cache
# Update dependencies and clean up, install libsndfile for audio processing
RUN apk upgrade --no-cache && apk add --no-cache libsndfile
WORKDIR /app

255
docker/Dockerfile.non_root
View file

@ -1,154 +1,183 @@
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG PROXY_EXTRAS_SOURCE=published
# -----------------
# Builder Stage
# -----------------
FROM $LITELLM_BUILD_IMAGE AS builder
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app
# Install build dependencies including Node.js for UI build
USER root
# Install build dependencies with retry logic (includes node for UI build)
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm && break || sleep 5; \
done \
apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm && break || sleep 5; \
done \
&& pip install --no-cache-dir --upgrade pip build
# Copy project files
# Cache Python dependencies
COPY requirements.txt .
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
&& pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
# Copy source after dependency layers
COPY . .
# Set LITELLM_NON_ROOT flag for build time
# Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true
# Build Admin UI
RUN mkdir -p /tmp/litellm_ui
# Build Admin UI using the upstream command order while keeping a single RUN layer
RUN mkdir -p /tmp/litellm_ui && \
npm install -g npm@latest && npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
mkdir -p /tmp/litellm_assets && \
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
( cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done ) && \
cd /app/ui/litellm-dashboard && rm -rf ./out
RUN npm install -g npm@latest && npm cache clean --force
RUN cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi
RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps
RUN cd /app/ui/litellm-dashboard && npm run build
RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg
RUN cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done
RUN cd /app/ui/litellm-dashboard && rm -rf ./out
# Build package and wheel dependencies
# Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
RUN rm -rf dist/* && python -m build && \
pip install dist/*.whl && \
pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
rm -f /wheels/litellm-*.whl && \
cp dist/*.whl /wheels/
# Optionally build local litellm-proxy-extras wheel
RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \
cp dist/*.whl /wheels/; \
fi
# Pre-cache Prisma binaries in the builder stage
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
XDG_CACHE_HOME=/app/.cache \
PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \
&& mkdir -p /app/.cache/npm
RUN NPM_CONFIG_CACHE=/app/.cache/npm \
python -c "import prisma.cli.prisma as p; p.ensure_cached()"
RUN prisma generate && \
prisma --version && \
prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
# -----------------
# Runtime Stage
# -----------------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG PROXY_EXTRAS_SOURCE
WORKDIR /app
# Install runtime dependencies
USER root
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy only necessary artifacts from builder stage for runtime
COPY . .
# Install runtime dependencies with retry
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /app/schema.prisma /app/
COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
COPY --from=builder /app/.cache /app/.cache
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
COPY --from=builder \
/usr/lib/python3.13/site-packages/nodejs* \
/usr/lib/python3.13/site-packages/prisma* \
/usr/lib/python3.13/site-packages/tomlkit* \
/usr/lib/python3.13/site-packages/nodeenv* \
/usr/lib/python3.13/site-packages/
COPY --from=builder /usr/bin/prisma /usr/bin/prisma
# Install package from wheel and dependencies
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Final runtime environment configuration
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
HOME=/app \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# Install packages from wheels and optional extras without network
RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \
pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \
pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \
if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \
pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \
else \
echo "litellm_proxy_extras wheel not found; skipping local install"; \
fi; \
fi
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Permissions, cleanup, and Prisma prep
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
rm -rf /wheels && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \
mkdir -p /tmp/.npm /nonexistent /.npm && \
prisma generate
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
pip uninstall PyJWT -y && \
pip install PyJWT==2.9.0 --no-cache-dir
# Set Prisma cache directories
ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
ENV NPM_CONFIG_CACHE=/.npm
# Install prisma and make entrypoints executable
RUN pip install --no-cache-dir prisma && \
chmod +x docker/entrypoint.sh && \
chmod +x docker/prod_entrypoint.sh
# Create directories and set permissions for non-root user
RUN mkdir -p /nonexistent /.npm /tmp/litellm_assets && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# OpenShift compatibility
RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
# Switch to non-root user
# Switch to non-root user for runtime
USER nobody
# Set HOME for prisma generate to have a writable directory
ENV HOME=/app
# Set LITELLM_NON_ROOT flag for runtime
ENV LITELLM_NON_ROOT=true
RUN prisma generate
# Prisma runtime knobs for offline containers
ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
NPM_CONFIG_CACHE=/app/.cache/npm \
NPM_CONFIG_PREFER_OFFLINE=true \
PRISMA_OFFLINE_MODE=true
EXPOSE 4000/tcp
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]

24
docker/README.md
View file

@ -59,6 +59,30 @@ To stop the running containers, use the following command:
docker compose down
```
## Hardened / Offline Testing
To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:
```bash
docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache
docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
```
This setup:
- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
- Runs the proxy as a non-root user with a read-only rootfs and only two writable tmpfs mounts:
- `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
- `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
You should also verify offline Prisma behaviour with:
```bash
docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version
```
This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access.
## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.

2
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md
View file

@ -6,7 +6,7 @@ authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/

2
docs/my-website/blog/gemini_3/index.md
View file

@ -6,7 +6,7 @@ authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/

222
docs/my-website/blog/gemini_3_flash/index.md Normal file
View file

@ -0,0 +1,222 @@
---
slug: gemini_3_flash
title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
date: 2025-12-17T10:00:00
authors:
- name: Sameer Kankute
title: SWE @ LiteLLM (LLM Translation)
url: https://www.linkedin.com/in/sameer-kankute/
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini 3 Flash Day 0 Support
LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
## What's New
### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
Gemini 3 Flash introduces granular thinking control with `thinkingLevel` instead of `thinkingBudget`.
- **MINIMAL**: Ultra-lightweight thinking for fast responses
- **MEDIUM**: Balanced thinking for complex reasoning
- **HIGH**: Maximum reasoning depth
LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
### 2. Thought Signatures
Like `gemini-3-pro`, this model also includes thought signatures for tool calls. LiteLLM handles signature extraction and embedding internally. [Learn more about thought signatures](../gemini_3/index.md#thought-signatures).
**Edge Case Handling**: If thought signatures are missing in the request, LiteLLM adds a dummy signature ensuring the API call doesn't break
---
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint
All endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
- Converstion of provider specific thinking related param to thinkingLevel
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**Basic Usage with MEDIUM thinking (NEW)**
```python
from litellm import completion
# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
reasoning_effort="medium", # NEW: MEDIUM thinking level
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gemini-3-flash
litellm_params:
model: gemini/gemini-3-flash-preview
api_key: os.environ/GEMINI_API_KEY
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**3. Call with MEDIUM thinking**
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-3-flash",
"messages": [{"role": "user", "content": "Complex reasoning task"}],
"reasoning_effort": "medium"
}'
``'
</TabItem>
</Tabs>
---
## All `reasoning_effort` Levels
<Tabs>
<TabItem value="minimal" label="MINIMAL">
**Ultra-fast, minimal reasoning**
```python
from litellm import completion
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "What's 2+2?"}],
reasoning_effort="minimal",
)
```
</TabItem>
<TabItem value="low" label="LOW">
**Simple instruction following**
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Write a haiku about coding"}],
reasoning_effort="low",
)
```
</TabItem>
<TabItem value="medium" label="MEDIUM (NEW)">
**Balanced reasoning for complex tasks** ✨
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Analyze this dataset and find patterns"}],
reasoning_effort="medium", # NEW!
)
```
</TabItem>
<TabItem value="high" label="HIGH">
**Maximum reasoning depth**
```python
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Prove this mathematical theorem"}],
reasoning_effort="high",
)
```
</TabItem>
</Tabs>
---
## Key Features
✅ **Thinking Levels**: MINIMAL, LOW, MEDIUM, HIGH
✅ **Thought Signatures**: Track reasoning with unique identifiers
✅ **Seamless Integration**: Works with existing OpenAI-compatible client
✅ **Backward Compatible**: Gemini 2.5 models continue using `thinkingBudget`
---
## Installation
```bash
pip install litellm --upgrade
```
```python
import litellm
from litellm import completion
response = completion(
model="gemini/gemini-3-flash-preview",
messages=[{"role": "user", "content": "Your question here"}],
reasoning_effort="medium", # Use MEDIUM thinking
)
print(response)
```
## `reasoning_effort` Mapping for Gemini 3+
| reasoning_effort | thinking_level |
|------------------|----------------|
| `minimal` | `minimal` |
| `low` | `low` |
| `medium` | `medium` |
| `high` | `high` |
| `disable` | `minimal` |
| `none` | `minimal` |

15
docs/my-website/docs/a2a.md
View file

@ -16,7 +16,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
| Feature | Supported |
|---------|-----------|
| Supported Agent Providers | A2A, LangGraph, Azure AI Foundry, Bedrock AgentCore |
| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI |
| Logging | ✅ |
| Load Balancing | ✅ |
| Streaming | ✅ |
@ -45,17 +45,26 @@ You can add A2A-compatible agents through the LiteLLM Admin UI.
The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`).
### Add Azure AI Foundry Agents
Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway)
### Add Vertex AI Agent Engine
Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine)
### Add Bedrock AgentCore Agents
Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway)
### Add LangGraph Agents
Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway)
### Add Bedrock AgentCore Agents
### Add Pydantic AI Agents
Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway)
Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway)
## Invoking your Agents

2
docs/my-website/docs/benchmarks.md
View file

@ -172,7 +172,7 @@ class MyUser(HttpUser):
## Logging Callbacks
### [GCS Bucket Logging](https://docs.litellm.ai/docs/proxy/bucket)
### [GCS Bucket Logging](https://docs.litellm.ai/docs/observability/gcs_bucket_integration)
Using GCS Bucket has **no impact on latency, RPS compared to Basic Litellm Proxy**

2
docs/my-website/docs/index.md
View file

@ -657,7 +657,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

214
docs/my-website/docs/interactions.md Normal file
View file

@ -0,0 +1,214 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# /interactions
| Feature | Supported | Notes |
|---------|-----------|-------|
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) |
## **LiteLLM Python SDK Usage**
### Quick Start
```python showLineNumbers title="Create Interaction"
from litellm import create_interaction
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = create_interaction(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
```
### Async Usage
```python showLineNumbers title="Async Create Interaction"
from litellm import acreate_interaction
import os
import asyncio
os.environ["GEMINI_API_KEY"] = "your-api-key"
async def main():
response = await acreate_interaction(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
asyncio.run(main())
```
### Streaming
```python showLineNumbers title="Streaming Interaction"
from litellm import create_interaction
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = create_interaction(
model="gemini/gemini-2.5-flash",
input="Write a 3 paragraph story about a robot.",
stream=True
)
for chunk in response:
print(chunk)
```
## **LiteLLM AI Gateway (Proxy) Usage**
### Setup
Add this to your litellm proxy config.yaml:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
```
Start litellm:
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### Test Request
<Tabs>
<TabItem value="curl" label="curl">
```bash showLineNumbers title="Create Interaction"
curl -X POST "http://localhost:4000/v1beta/interactions" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini/gemini-2.5-flash",
"input": "Tell me a short joke about programming."
}'
```
**Streaming:**
```bash showLineNumbers title="Streaming Interaction"
curl -N -X POST "http://localhost:4000/v1beta/interactions" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini/gemini-2.5-flash",
"input": "Write a 3 paragraph story about a robot.",
"stream": true
}'
```
**Get Interaction:**
```bash showLineNumbers title="Get Interaction by ID"
curl "http://localhost:4000/v1beta/interactions/{interaction_id}" \
-H "Authorization: Bearer sk-1234"
```
</TabItem>
<TabItem value="google-sdk" label="Google GenAI SDK">
Point the Google GenAI SDK to LiteLLM Proxy:
```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
from google import genai
import os
# Point SDK to LiteLLM Proxy
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234" # Your LiteLLM API key
client = genai.Client()
# Create an interaction
interaction = client.interactions.create(
model="gemini/gemini-2.5-flash",
input="Tell me a short joke about programming."
)
print(interaction.outputs[-1].text)
```
**Streaming:**
```python showLineNumbers title="Google GenAI SDK Streaming"
from google import genai
import os
os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
os.environ["GEMINI_API_KEY"] = "sk-1234"
client = genai.Client()
for chunk in client.interactions.create_stream(
model="gemini/gemini-2.5-flash",
input="Write a story about space exploration.",
):
print(chunk)
```
</TabItem>
</Tabs>
## **Request/Response Format**
### Request Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | Model to use (e.g., `gemini/gemini-2.5-flash`) |
| `input` | string | Yes | The input text for the interaction |
| `stream` | boolean | No | Enable streaming responses |
| `tools` | array | No | Tools available to the model |
| `system_instruction` | string | No | System instructions for the model |
| `generation_config` | object | No | Generation configuration |
| `previous_interaction_id` | string | No | ID of previous interaction for context |
### Response Format
```json
{
"id": "interaction_abc123",
"object": "interaction",
"model": "gemini-2.5-flash",
"status": "completed",
"created": "2025-01-15T10:30:00Z",
"updated": "2025-01-15T10:30:05Z",
"role": "model",
"outputs": [
{
"type": "text",
"text": "Why do programmers prefer dark mode? Because light attracts bugs!"
}
],
"usage": {
"total_input_tokens": 10,
"total_output_tokens": 15,
"total_tokens": 25
}
}
```
## **Supported Providers**
| Provider | Link to Usage |
|----------|---------------|
| Google AI Studio | [Usage](#quick-start) |

2
docs/my-website/docs/observability/datadog.md
View file

@ -181,7 +181,7 @@ docker run \
-e USE_DDTRACE=true \
-e USE_DDPROFILER=true \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

84
docs/my-website/docs/providers/anthropic.md
View file

@ -1936,3 +1936,87 @@ curl http://0.0.0.0:4000/v1/chat/completions \
</TabItem>
</Tabs>
## Usage - Agent Skills
LiteLLM supports using Agent Skills with the API
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = completion(
model="claude-sonnet-4-5-20250929",
messages=messages,
tools= [
{
"type": "code_execution_20250825",
"name": "code_execution"
}
],
container= {
"skills": [
{
"type": "anthropic",
"skill_id": "pptx",
"version": "latest"
}
]
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: claude-sonnet-4-5-20250929
litellm_params:
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
```
2. Start Proxy
```
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl --location 'http://localhost:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer <YOUR-LITELLM-KEY>' \
--data '{
"model": "claude-sonnet-4-5-20250929",
"messages": [
{
"role": "user",
"content": "Hi"
}
],
"tools": [
{
"type": "code_execution_20250825",
"name": "code_execution"
}
],
"container": {
"skills": [
{
"type": "anthropic",
"skill_id": "pptx",
"version": "latest"
}
]
}
}'
```
</TabItem>
</Tabs>
The container and its "id" will be present in "provider_specific_fields" in streaming/non-streaming response

108
docs/my-website/docs/providers/custom_llm_server.md
View file

@ -17,6 +17,7 @@ Supported Routes:
- `/v1/completions` -> `litellm.atext_completion`
- `/v1/embeddings` -> `litellm.aembedding`
- `/v1/images/generations` -> `litellm.aimage_generation`
- `/v1/images/edits` -> `litellm.aimage_edit`
- `/v1/messages` -> `litellm.acompletion`
@ -263,6 +264,83 @@ Expected Response
}
```
## Image Edit
1. Setup your `custom_handler.py` file
```python
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
import time
class MyCustomLLM(CustomLLM):
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
# Your custom image edit logic here
# e.g., call Stability AI, Black Forest Labs, etc.
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/edited-image.png")],
)
my_custom_llm = MyCustomLLM()
```
2. Add to `config.yaml`
In the config below, we pass
python_filename: `custom_handler.py`
custom_handler_instance_name: `my_custom_llm`. This is defined in Step 1
custom_handler: `custom_handler.my_custom_llm`
```yaml
model_list:
- model_name: "my-custom-image-edit-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
```
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/v1/images/edits' \
-H 'Authorization: Bearer sk-1234' \
-F 'model=my-custom-image-edit-model' \
-F 'image=@/path/to/image.png' \
-F 'prompt=Make the sky blue'
```
Expected Response
```
{
"created": 1721955063,
"data": [{"url": "https://example.com/edited-image.png"}],
}
```
## Anthropic `/v1/messages`
- Write the integration for .acompletion
@ -517,4 +595,34 @@ class CustomLLM(BaseLLM):
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def image_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
```

121
docs/my-website/docs/providers/pydantic_ai_agent.md Normal file
View file

@ -0,0 +1,121 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Pydantic AI Agents
Call Pydantic AI Agents via LiteLLM's A2A Gateway.
| Property | Details |
|----------|---------|
| Description | Pydantic AI agents with native A2A support via the `to_a2a()` method. LiteLLM provides fake streaming support for agents that don't natively stream. |
| Provider Route on LiteLLM | A2A Gateway |
| Supported Endpoints | `/v1/a2a/message/send` |
| Provider Doc | [Pydantic AI Agents ↗](https://ai.pydantic.dev/agents/) |
## LiteLLM A2A Gateway
All Pydantic AI agents need to be exposed as A2A agents using the `to_a2a()` method. Once your agent server is running, you can add it to the LiteLLM Gateway.
### 1. Setup Pydantic AI Agent Server
LiteLLM requires Pydantic AI agents to follow the [A2A (Agent-to-Agent) protocol](https://github.com/google/A2A). Pydantic AI has native A2A support via the `to_a2a()` method, which exposes your agent as an A2A-compliant server.
#### Install Dependencies
```bash
pip install pydantic-ai fasta2a uvicorn
```
#### Create Agent
```python title="agent.py"
from pydantic_ai import Agent
agent = Agent('openai:gpt-4o-mini', instructions='Be helpful!')
@agent.tool_plain
def get_weather(city: str) -> str:
"""Get weather for a city."""
return f"Weather in {city}: Sunny, 72°F"
@agent.tool_plain
def calculator(expression: str) -> str:
"""Evaluate a math expression."""
return str(eval(expression))
# Native A2A server - Pydantic AI handles it automatically
app = agent.to_a2a()
```
#### Run Server
```bash
uvicorn agent:app --host 0.0.0.0 --port 9999
```
Server runs at `http://localhost:9999`
### 2. Navigate to Agents
From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent".
### 3. Select Pydantic AI Agent Type
Click "A2A Standard" to see available agent types, then select "Pydantic AI".
![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/1055acb1-064b-4465-8e6a-8278291bc661/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=395,147)
![Select Pydantic AI](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/0998e38c-8534-40f1-931a-be96c2cae0ad/ascreenshot.jpeg?tl_px=0,52&br_px=2201,1283&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=421,277)
### 4. Configure the Agent
Fill in the following fields:
- **Agent Name** - A unique identifier for your agent (e.g., `test-pydantic-agent`)
- **Agent URL** - The URL where your Pydantic AI agent is running. We use `http://localhost:9999` because that's where we started our Pydantic AI agent server in the previous step.
![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/8cf3fbde-05f3-48d1-81b6-6f857bd6d360/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=443,225)
![Configure Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb555808-4761-4c49-a415-200ac1bdb525/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
![Enter Agent URL](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/303eae61-4352-4fb0-a537-806839c234ba/ascreenshot.jpeg?tl_px=0,212&br_px=2201,1443&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=456,277)
### 5. Create Agent
Click "Create Agent" to save your configuration.
![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/914f3367-df7d-4244-bd4d-e99ce0a6193a/ascreenshot.jpeg?tl_px=416,438&br_px=2618,1669&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=690,277)
### 6. Test in Playground
Go to "Playground" in the sidebar to test your agent.
![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/c73c9f3b-22af-4105-aafa-2d34c4986ef3/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=44,97)
### 7. Select A2A Endpoint
Click the endpoint dropdown and search for "a2a", then select `/v1/a2a/message/send`.
![Click Endpoint Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/196d97ac-bcba-47f0-9880-97b80250e00c/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=261,230)
![Search for A2A](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/26b68f21-29f9-4c4c-b8b5-d2e11cbfd14a/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0)
![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/41576fb1-d385-4fb2-84e9-142dd7fe5181/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=307,270)
### 8. Select Your Agent and Send a Message
Pick your Pydantic AI agent from the dropdown and send a test message.
![Click Agent Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a96d7967-3d54-4cbf-bd3e-b38f1be9df76/ascreenshot.jpeg?tl_px=0,54&br_px=2201,1285&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=274,277)
![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/e05a5a6e-d044-4480-b94e-7c03cfb92ac5/ascreenshot.jpeg?tl_px=0,113&br_px=2201,1344&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=290,277)
![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/29162702-968a-401a-aac1-c844bfc5f4a3/ascreenshot.jpeg?tl_px=91,653&br_px=2292,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,436)
## Further Reading
- [Pydantic AI Documentation](https://ai.pydantic.dev/)
- [Pydantic AI Agents](https://ai.pydantic.dev/agents/)
- [A2A Agent Gateway](../a2a.md)
- [A2A Cost Tracking](../a2a_cost_tracking.md)

65
docs/my-website/docs/providers/sap.md
View file

@ -5,12 +5,12 @@ import TabItem from '@theme/TabItem';
LiteLLM supports SAP Generative AI Hub's Orchestration Service.
| Property | Details |
|-------|-------|
| Description | SAP's Generative AI Hub provides access to foundation models through the AI Core orchestration service. |
| Provider Route on LiteLLM | `sap/` |
| Supported Endpoints | `/chat/completions` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
| Property | Details |
|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
| Description | SAP's Generative AI Hub provides access to OpenAI, Anthropic, Gemini, Mistral, NVIDIA, Amazon, and SAP LLMs through the AI Core orchestration service. |
| Provider Route on LiteLLM | `sap/` |
| Supported Endpoints | `/chat/completions`, `/embeddings` |
| API Reference | [SAP AI Core Documentation](https://help.sap.com/docs/sap-ai-core) |
## Authentication
@ -23,7 +23,14 @@ SAP Generative AI Hub uses service key authentication. You can provide credentia
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
## Usage - LiteLLM Python SDK
```python showLineNumbers title="SAP Chat Completion"
@ -55,16 +62,33 @@ for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
```python showLineNumbers title="SAP Embedding"
from litellm import embedding
import os
os.environ["AICORE_SERVICE_KEY"] = '{"clientid": "...", "clientsecret": "...", ...}'
result = embedding(
model="sap/text-embedding-3-small",
input="Answer to the ultimate question of life, the universe, and everything is 42")
print(result.data[0])
```
## Usage - LiteLLM Proxy
Add to your LiteLLM Proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: sap-gpt4
- model_name: "sap/*"
litellm_params:
model: sap/gpt-4
api_key: os.environ/AICORE_SERVICE_KEY
model: "sap/*"
general_settings:
master_key: your-proxy-api-key
environment_variables:
AICORE_SERVICE_KEY: '{"clientid": "...", "clientsecret": "...", ...}'
```
Start the proxy:
@ -81,7 +105,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "sap-gpt4",
"model": "sap/gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
@ -98,12 +122,29 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="sap-gpt4",
model="sap/gpt-4",
messages=[{"role": "user", "content": "Hello"}]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="LiteLLM SDK"
import os
import litellm
os.environ["LITELLM_PROXY_API_KEY"] = "your-proxy-api-key"
litellm.use_litellm_proxy = True # it is important to set this parameter
response = litellm.completion(
model="sap/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="http://your-proxy-api-base"
)
print(response)
```
</TabItem>
</Tabs>

216
docs/my-website/docs/providers/vertex_ai_agent_engine.md Normal file
View file

@ -0,0 +1,216 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Agent Engine
Call Vertex AI Agent Engine (Reasoning Engines) in the OpenAI Request/Response format.
| Property | Details |
|----------|---------|
| Description | Vertex AI Agent Engine provides hosted agent runtimes that can execute agentic workflows with foundation models, tools, and custom logic. |
| Provider Route on LiteLLM | `vertex_ai/agent_engine/{RESOURCE_NAME}` |
| Supported Endpoints | `/chat/completions`, `/v1/messages`, `/v1/responses`, `/v1/a2a/message/send` |
| Provider Doc | [Vertex AI Agent Engine ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) |
## Quick Start
### Model Format
```shell showLineNumbers title="Model Format"
vertex_ai/agent_engine/{RESOURCE_NAME}
```
**Example:**
- `vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`
### LiteLLM Python SDK
```python showLineNumbers title="Basic Agent Completion"
import litellm
response = litellm.completion(
model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888",
messages=[
{"role": "user", "content": "Explain machine learning in simple terms"}
],
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Streaming Agent Responses"
import litellm
response = await litellm.acompletion(
model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888",
messages=[
{"role": "user", "content": "What are the key principles of software architecture?"}
],
stream=True,
)
async for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
### LiteLLM Proxy
#### 1. Configure your model in config.yaml
<Tabs>
<TabItem value="config-yaml" label="config.yaml">
```yaml showLineNumbers title="LiteLLM Proxy Configuration"
model_list:
- model_name: vertex-agent-1
litellm_params:
model: vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888
vertex_project: your-project-id
vertex_location: us-central1
```
</TabItem>
</Tabs>
#### 2. Start the LiteLLM Proxy
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
```
#### 3. Make requests to your Vertex AI Agent Engine
<Tabs>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Basic Agent Request"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "vertex-agent-1",
"messages": [
{"role": "user", "content": "Summarize the main benefits of cloud computing"}
]
}'
```
</TabItem>
<TabItem value="openai-sdk" label="OpenAI Python SDK">
```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
response = client.chat.completions.create(
model="vertex-agent-1",
messages=[
{"role": "user", "content": "What are best practices for API design?"}
]
)
print(response.choices[0].message.content)
```
</TabItem>
</Tabs>
## LiteLLM A2A Gateway
You can also connect to Vertex AI Agent Engine through LiteLLM's A2A (Agent-to-Agent) Gateway UI. This provides a visual way to register and test agents without writing code.
### 1. Navigate to Agents
From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent".
![Click Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9a979927-ce6b-4168-9fba-e53e28f1c2c4/ascreenshot.jpeg?tl_px=0,14&br_px=1376,783&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=17,277)
![Add New Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a311750c-2e85-4589-99cb-2ce7e4021e77/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=195,257)
### 2. Select Vertex AI Agent Engine Type
Click "A2A Standard" to see available agent types, then select "Vertex AI Agent Engine".
![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/5b1acc4c-dc3f-4639-b4a0-e64b35c228fd/ascreenshot.jpeg?tl_px=52,0&br_px=1428,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,271)
![Select Vertex AI Agent Engine](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/2f3bab61-3e02-4db7-84f0-82200a0f4136/ascreenshot.jpeg?tl_px=0,244&br_px=1376,1013&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=477,277)
### 3. Configure the Agent
Fill in the following fields:
- **Agent Name** - A friendly name for your agent (e.g., `my-vertex-agent`)
- **Reasoning Engine Resource ID** - The full resource path from Google Cloud Console (e.g., `projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`)
- **Vertex Project** - Your Google Cloud project ID
- **Vertex Location** - The region where your agent is deployed (e.g., `us-central1`)
![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/695b84c7-9511-4337-bf19-f4505ab2b72b/ascreenshot.jpeg?tl_px=0,90&br_px=1376,859&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=480,276)
![Enter Resource ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/ddce64df-b3a3-4519-ab62-f137887bcea2/ascreenshot.jpeg?tl_px=0,294&br_px=1376,1063&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=440,277)
You can find the Resource ID in Google Cloud Console under Vertex AI > Agent Engine:
![Copy Resource ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/185d7f17-cbaa-45de-948d-49d2091805ea/ascreenshot.jpeg?tl_px=0,165&br_px=1376,934&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=493,276)
![Enter Vertex Project](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a64da441-3e61-4811-a1e3-9f0b12c949ff/ascreenshot.jpeg?tl_px=0,233&br_px=1376,1002&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=501,277)
You can find the Project ID in Google Cloud Console:
![Copy Project ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9ecad3bb-a534-42d6-9604-33906014fad6/user_cropped_screenshot.webp?tl_px=0,0&br_px=1728,1028&force_format=jpeg&q=100&width=1120.0)
![Enter Vertex Location](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/316d1f38-4fb7-4377-86b6-c0fe7ac24383/ascreenshot.jpeg?tl_px=0,330&br_px=1376,1099&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=423,277)
### 4. Create Agent
Click "Create Agent" to save your configuration.
![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb04b95d-793f-4eed-acf4-d1b3b5fa65e9/ascreenshot.jpeg?tl_px=352,347&br_px=1728,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=623,498)
### 5. Test in Playground
Go to "Playground" in the sidebar to test your agent.
![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9e01369b-6102-4fe3-96a7-90082cadfd6e/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=55,226)
### 6. Select A2A Endpoint
Click the endpoint dropdown and select `/v1/a2a/message/send`.
![Select Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/d5aeac35-531b-4cf0-af2d-88f0a71fd736/ascreenshot.jpeg?tl_px=0,146&br_px=1376,915&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=299,277)
### 7. Select Your Agent and Send a Message
Pick your Vertex AI Agent Engine from the dropdown and send a test message.
![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/353431f3-a0ba-4436-865d-ae11595e9cc4/ascreenshot.jpeg?tl_px=0,263&br_px=1376,1032&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=270,277)
![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fbfce72e-f50b-43e1-b6e5-0d41192d8e2d/ascreenshot.jpeg?tl_px=95,347&br_px=1471,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,474)
![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/892dd826-fbf9-4530-8d82-95270889274a/ascreenshot.jpeg?tl_px=0,82&br_px=1376,851&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=485,277)
## Environment Variables
| Variable | Description |
|----------|-------------|
| `GOOGLE_APPLICATION_CREDENTIALS` | Path to service account JSON key file |
| `VERTEXAI_PROJECT` | Google Cloud project ID |
| `VERTEXAI_LOCATION` | Google Cloud region (default: `us-central1`) |
```bash
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
export VERTEXAI_PROJECT="your-project-id"
export VERTEXAI_LOCATION="us-central1"
```
## Further Reading
- [Vertex AI Agent Engine Documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview)
- [Create a Reasoning Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/create)
- [A2A Agent Gateway](../a2a.md)
- [Vertex AI Provider](./vertex.md)

4
docs/my-website/docs/proxy/configs.md
View file

@ -655,7 +655,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-e LITELLM_CONFIG_BUCKET_TYPE="gcs" \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-latest --detailed_debug
docker.litellm.ai/berriai/litellm-database:main-latest --detailed_debug
```
</TabItem>
@ -676,7 +676,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_NAME=<bucket_name> \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-latest
docker.litellm.ai/berriai/litellm-database:main-latest
```
</TabItem>
</Tabs>

82
docs/my-website/docs/proxy/deploy.md
View file

@ -10,10 +10,38 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber
## Quick Start
:::info
Facing issues with pulling the docker image? Email us at support@berri.ai.
:::
To start using Litellm, run the following commands in a shell:
<Tabs>
<TabItem value="docker" label="Docker">
```
docker pull docker.litellm.ai/berriai/litellm:main-latest
```
[**See all docker images**](https://github.com/orgs/BerriAI/packages)
</TabItem>
<TabItem value="pip" label="LiteLLM CLI (pip package)">
```shell
$ pip install 'litellm[proxy]'
```
</TabItem>
<TabItem value="docker-compose" label="Docker Compose (Proxy + DB)">
Use this docker compose to spin up the proxy with a postgres database running locally.
```bash
# Get the code
# Get the docker compose file
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
@ -30,6 +58,8 @@ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
docker compose up
```
</TabItem>
</Tabs>
### Docker Run
@ -57,7 +87,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-stable \
docker.litellm.ai/berriai/litellm:main-stable \
--config /app/config.yaml --detailed_debug
```
@ -87,12 +117,12 @@ See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli):
Here's how you can run the docker image and pass your config to `litellm`
```shell
docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml
```
Here's how you can run the docker image and start litellm on port 8002 with `num_workers=8`
```shell
docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8
docker run docker.litellm.ai/berriai/litellm:main-stable --port 8002 --num_workers 8
```
@ -100,7 +130,7 @@ docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-stable
FROM docker.litellm.ai/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@ -242,7 +272,7 @@ spec:
spec:
containers:
- name: litellm
image: ghcr.io/berriai/litellm:main-stable # it is recommended to fix a version generally
image: docker.litellm.ai/berriai/litellm:main-stable # it is recommended to fix a version generally
args:
- "--config"
- "/app/proxy_server_config.yaml"
@ -279,9 +309,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart
#### Step 1. Pull the litellm helm chart
```bash
helm pull oci://ghcr.io/berriai/litellm-helm
helm pull oci://docker.litellm.ai/berriai/litellm-helm
# Pulled: ghcr.io/berriai/litellm-helm:0.1.2
# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2
# Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a
```
@ -340,7 +370,7 @@ Requirements:
We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
```shell
docker pull ghcr.io/berriai/litellm-database:main-stable
docker pull docker.litellm.ai/berriai/litellm-database:main-stable
```
```shell
@ -351,7 +381,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable \
docker.litellm.ai/berriai/litellm-database:main-stable \
--config /app/config.yaml --detailed_debug
```
@ -379,7 +409,7 @@ spec:
spec:
containers:
- name: litellm-container
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
imagePullPolicy: Always
env:
- name: AZURE_API_KEY
@ -516,9 +546,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart
#### Step 1. Pull the litellm helm chart
```bash
helm pull oci://ghcr.io/berriai/litellm-helm
helm pull oci://docker.litellm.ai/berriai/litellm-helm
# Pulled: ghcr.io/berriai/litellm-helm:0.1.2
# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2
# Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a
```
@ -575,7 +605,7 @@ router_settings:
Start docker container with config
```shell
docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml
docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml
```
### Deploy with Database + Redis
@ -610,7 +640,7 @@ Start `litellm-database`docker container with config
docker run --name litellm-proxy \
-e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable --config your_config.yaml
docker.litellm.ai/berriai/litellm-database:main-stable --config your_config.yaml
```
### (Non Root) - without Internet Connection
@ -620,7 +650,7 @@ By default `prisma generate` downloads [prisma's engine binaries](https://www.pr
Use this docker image to deploy litellm with pre-generated prisma binaries.
```bash
docker pull ghcr.io/berriai/litellm-non_root:main-stable
docker pull docker.litellm.ai/berriai/litellm-non_root:main-stable
```
[Published Docker Image link](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root)
@ -639,7 +669,7 @@ Use this, If you need to set ssl certificates for your on prem litellm proxy
Pass `ssl_keyfile_path` (Path to the SSL keyfile) and `ssl_certfile_path` (Path to the SSL certfile) when starting litellm proxy
```shell
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--ssl_keyfile_path ssl_test/keyfile.key \
--ssl_certfile_path ssl_test/certfile.crt
```
@ -654,7 +684,7 @@ Step 1. Build your custom docker image with hypercorn
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-stable
FROM docker.litellm.ai/berriai/litellm:main-stable
# Set the working directory to /app
WORKDIR /app
@ -702,7 +732,7 @@ Usage Example:
In this example, we set the keepalive timeout to 75 seconds.
```shell showLineNumbers title="docker run"
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--keepalive_timeout 75
```
@ -711,7 +741,7 @@ In this example, we set the keepalive timeout to 75 seconds.
```shell showLineNumbers title="Environment Variable"
export KEEPALIVE_TIMEOUT=75
docker run ghcr.io/berriai/litellm:main-stable
docker run docker.litellm.ai/berriai/litellm:main-stable
```
@ -722,7 +752,7 @@ Use this to mitigate memory growth by recycling workers after a fixed number of
Usage Examples:
```shell showLineNumbers title="docker run (CLI flag)"
docker run ghcr.io/berriai/litellm:main-stable \
docker run docker.litellm.ai/berriai/litellm:main-stable \
--max_requests_before_restart 10000
```
@ -730,7 +760,7 @@ Or set via environment variable:
```shell showLineNumbers title="Environment Variable"
export MAX_REQUESTS_BEFORE_RESTART=10000
docker run ghcr.io/berriai/litellm:main-stable
docker run docker.litellm.ai/berriai/litellm:main-stable
```
@ -759,7 +789,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-e LITELLM_CONFIG_BUCKET_TYPE="gcs" \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable --detailed_debug
docker.litellm.ai/berriai/litellm-database:main-stable --detailed_debug
```
</TabItem>
@ -780,7 +810,7 @@ docker run --name litellm-proxy \
-e LITELLM_CONFIG_BUCKET_NAME=<bucket_name> \
-e LITELLM_CONFIG_BUCKET_OBJECT_KEY="<object_key>> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable
docker.litellm.ai/berriai/litellm-database:main-stable
```
</TabItem>
</Tabs>
@ -907,7 +937,7 @@ Run the following command, replacing `<database_url>` with the value you copied
docker run --name litellm-proxy \
-e DATABASE_URL=<database_url> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-stable
docker.litellm.ai/berriai/litellm-database:main-stable
```
#### 4. Access the Application:
@ -986,7 +1016,7 @@ services:
context: .
args:
target: runtime
image: ghcr.io/berriai/litellm:main-stable
image: docker.litellm.ai/berriai/litellm:main-stable
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
volumes:

6
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -20,7 +20,7 @@ End-to-End tutorial for LiteLLM Proxy to:
<TabItem value="docker" label="Docker">
```
docker pull ghcr.io/berriai/litellm:main-latest
docker pull docker.litellm.ai/berriai/litellm:main-latest
```
[**See all docker images**](https://github.com/orgs/BerriAI/packages)
@ -119,7 +119,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
# RUNNING on http://0.0.0.0:4000
@ -302,7 +302,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

2
docs/my-website/docs/proxy/enterprise.md
View file

@ -29,7 +29,7 @@ Features:
- **Spend Tracking & Data Exports**
- ✅ [Set USD Budgets Spend for Custom Tags](./provider_budget_routing#-tag-budgets)
- ✅ [Set Model budgets for Virtual Keys](./users#-virtual-key-model-specific)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](./proxy/bucket#🪣-logging-gcs-s3-buckets)
- ✅ [Exporting LLM Logs to GCS Bucket, Azure Blob Storage](../observability/gcs_bucket_integration)
- ✅ [`/spend/report` API endpoint](cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend)
- **Control Guardrails per API Key/Team**
- **Custom Branding**

2
docs/my-website/docs/proxy/guardrails/pangea.md
View file

@ -67,7 +67,7 @@ docker run --rm \
-e PANGEA_AI_GUARD_TOKEN=$PANGEA_AI_GUARD_TOKEN \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml
```

100
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

@ -72,13 +72,15 @@ litellm --config config.yaml --port 4000
### Overview
Pillar Security supports three execution modes for comprehensive protection:
Pillar Security supports five execution modes for comprehensive protection:
| Mode | When It Runs | What It Protects | Use Case
|------|-------------|------------------|----------
| **`pre_call`** | Before LLM call | User input only | Block malicious prompts, prevent prompt injection
| **`during_call`** | Parallel with LLM call | User input only | Input monitoring with lower latency
| **`post_call`** | After LLM response | Full conversation context | Output filtering, PII detection in responses
| **`pre_mcp_call`** | Before MCP tool call | MCP tool inputs | Validate and sanitize MCP tool call arguments
| **`during_mcp_call`** | During MCP tool call | MCP tool inputs | Real-time monitoring of MCP tool calls
### Why Dual Mode is Recommended
@ -198,6 +200,85 @@ litellm_settings:
set_verbose: true # Enable detailed logging
```
</TabItem>
<TabItem value="masking" label="Masking Mode - Auto-Sanitize PII">
**Best for:**
- 🔒 **PII Protection**: Automatically sanitize sensitive data before sending to LLM
- ✅ **Continue Workflows**: Allow requests to proceed with masked content
- 🛡️ **Zero Trust**: Never expose sensitive data to LLM models
- 📊 **Compliance**: Meet data privacy requirements without blocking legitimate requests
```yaml
model_list:
- model_name: gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "pillar-masking"
litellm_params:
guardrail: pillar
mode: "pre_call" # Scan input before LLM call
api_key: os.environ/PILLAR_API_KEY # Your Pillar API key
api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint
on_flagged_action: "mask" # Mask sensitive content instead of blocking
persist_session: true # Keep records for investigation
include_scanners: true # Understand which scanners triggered
include_evidence: true # Capture evidence for analysis
default_on: true # Enable for all requests
general_settings:
master_key: "YOUR_LITELLM_PROXY_MASTER_KEY"
litellm_settings:
set_verbose: true
```
**How it works:**
1. User sends request with sensitive data: `"My email is john@example.com"`
2. Pillar detects PII and returns masked version: `"My email is [MASKED_EMAIL]"`
3. LiteLLM replaces original messages with masked messages
4. Request proceeds to LLM with sanitized content
5. User receives response without exposing sensitive data
</TabItem>
<TabItem value="mcp" label="MCP Call Protection">
**Best for:**
- 🤖 **Agent Workflows**: Protect MCP (Model Context Protocol) tool calls
- 🔒 **Tool Input Validation**: Scan arguments passed to MCP tools
- 🛡️ **Comprehensive Coverage**: Extend security to all LLM endpoints
```yaml
model_list:
- model_name: gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "pillar-mcp-guard"
litellm_params:
guardrail: pillar
mode: "pre_mcp_call" # Scan MCP tool call inputs
api_key: os.environ/PILLAR_API_KEY # Your Pillar API key
api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint
on_flagged_action: "block" # Block malicious MCP calls
default_on: true # Enable for all MCP calls
general_settings:
master_key: "YOUR_LITELLM_PROXY_MASTER_KEY"
litellm_settings:
set_verbose: true
```
**MCP Modes:**
- `pre_mcp_call`: Scan MCP tool call inputs before execution
- `during_mcp_call`: Monitor MCP tool calls in real-time
</TabItem>
</Tabs>
@ -251,6 +332,15 @@ Logs the violation but allows the request to proceed:
on_flagged_action: "monitor"
```
#### Mask
Automatically sanitizes sensitive content (PII, secrets, etc.) in your messages before sending them to the LLM:
```yaml
on_flagged_action: "mask"
```
When masking is enabled, sensitive information is automatically replaced with masked versions, allowing requests to proceed safely without exposing sensitive data to the LLM.
**Response Headers:**
You can opt in to receiving detection details in response headers by configuring `include_scanners: true` and/or `include_evidence: true`. When enabled, these headers are included for **every request**—not just flagged ones—enabling comprehensive metrics, false positive analysis, and threat investigation.
@ -383,7 +473,8 @@ export PILLAR_TIMEOUT="5.0"
**Quick takeaways**
- Every request still runs *all* Pillar scanners; these options only change what comes back.
- Choose richer responses when you need audit trails, lighter responses when latency or cost matters.
- Blocking is controlled by LiteLLM’s `on_flagged_action` configuration—Pillar headers do not change block/monitor behaviour.
- Actions (block/monitor/mask) are controlled by LiteLLM's `on_flagged_action` configuration—Pillar headers are automatically set based on your config.
- When blocking (`on_flagged_action: "block"`), the `include_scanners` and `include_evidence` settings control what details are included in the exception response.
Pillar Security executes the full scanner suite on each call. The settings below tune the Protect response headers LiteLLM sends, letting you balance fidelity, retention, and latency.
@ -415,9 +506,10 @@ include_evidence: true # → plr_evidence (default true in LiteLLM)
```
Use when you only care about whether Pillar detected a threat.
> **📝 Note:** `flagged: true` means Pillar’s scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration (no Pillar header controls it):
> - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error
> **📝 Note:** `flagged: true` means Pillar's scanners recommend blocking. Pillar only reports this verdict—LiteLLM enforces your policy via the `on_flagged_action` configuration:
> - `on_flagged_action: "block"` → LiteLLM raises a 400 guardrail error (exception includes scanners/evidence based on `include_scanners`/`include_evidence` settings)
> - `on_flagged_action: "monitor"` → LiteLLM logs the threat but still returns the LLM response
> - `on_flagged_action: "mask"` → LiteLLM replaces messages with masked versions and allows the request to proceed
- **Scanner breakdown** (`include_scanners=true`)
```json

25
docs/my-website/docs/proxy/load_balancing.md
View file

@ -29,6 +29,10 @@ LiteLLM automatically distributes requests across multiple deployments of the sa
| **latency-based-routing** | Routes to fastest responding deployment | Latency-critical applications |
| **cost-based-routing** | Routes to deployment with lowest cost | Cost-sensitive applications |
:::tip Deployment Priority
Use the `order` parameter to prioritize specific deployments. [See Deployment Ordering](#deployment-ordering-priority) for details.
:::
## Quick Start - Load Balancing
#### Step 1 - Set deployments on config
@ -243,6 +247,27 @@ class RouterModelGroupAliasItem(TypedDict):
hidden: bool # if 'True', don't return on `/v1/models`, `/v1/model/info`, `/v1/model_group/info`
```
## Deployment Ordering (Priority)
Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them.
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-primary
api_key: os.environ/AZURE_API_KEY
order: 1 # 👈 Highest priority - always tried first
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
```
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
### When You'll See Load Balancing in Action
**Immediate Effects:**

2
docs/my-website/docs/proxy/shared_health_check.md
View file

@ -269,7 +269,7 @@ spec:
spec:
containers:
- name: litellm-proxy
image: ghcr.io/berriai/litellm:latest
image: docker.litellm.ai/berriai/litellm:latest
env:
- name: USE_SHARED_HEALTH_CHECK
value: "true"

53
docs/my-website/docs/routing.md
View file

@ -832,6 +832,59 @@ asyncio.run(router_acompletion())
## Basic Reliability
### Deployment Ordering (Priority)
Set `order` in `litellm_params` to prioritize deployments. Lower values = higher priority. When multiple deployments share the same `order`, the routing strategy picks among them.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import Router
model_list = [
{
"model_name": "gpt-4",
"litellm_params": {
"model": "azure/gpt-4-primary",
"api_key": os.getenv("AZURE_API_KEY"),
"order": 1, # 👈 Highest priority
},
},
{
"model_name": "gpt-4",
"litellm_params": {
"model": "azure/gpt-4-fallback",
"api_key": os.getenv("AZURE_API_KEY_2"),
"order": 2, # 👈 Used when order=1 is unavailable
},
},
]
router = Router(model_list=model_list)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-primary
api_key: os.environ/AZURE_API_KEY
order: 1 # 👈 Highest priority
- model_name: gpt-4
litellm_params:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
```
</TabItem>
</Tabs>
### Weighted Deployments
Set `weight` on a deployment to pick one deployment more often than others.

2
docs/my-website/docs/secret_managers/custom_secret_manager.md
View file

@ -76,7 +76,7 @@ docker run -d \
--name litellm-proxy \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/my_secret_manager.py:/app/my_secret_manager.py \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug

9
docs/my-website/docs/secret_managers/hashicorp_vault.md
View file

@ -47,6 +47,8 @@ HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****"
# OPTIONAL
HCP_VAULT_REFRESH_INTERVAL="86400" # defaults to 86400, frequency of cache refresh for Hashicorp Vault
HCP_VAULT_MOUNT_NAME="secret" # OPTIONAL. defaults to "secret", set this if your KV engine is mounted elsewhere
HCP_VAULT_PATH_PREFIX="litellm" # OPTIONAL. defaults to None, set this if your secrets live under a custom prefix like secret/data/litellm/OPENAI_API_KEY
```
**Step 2.** Add to proxy config.yaml
@ -151,18 +153,20 @@ export HCP_VAULT_TOKEN="hvs.CAESIG52gL6ljBSdmq*****"
LiteLLM reads secrets from Hashicorp Vault's KV v2 engine using the following URL format:
```
{VAULT_ADDR}/v1/{NAMESPACE}/secret/data/{SECRET_NAME}
{VAULT_ADDR}/v1/{NAMESPACE}/{MOUNT_NAME}/data/{PATH_PREFIX}/{SECRET_NAME}
```
For example, if you have:
- `HCP_VAULT_ADDR="https://vault.example.com:8200"`
- `HCP_VAULT_NAMESPACE="admin"`
- `HCP_VAULT_MOUNT_NAME="secret"`
- `HCP_VAULT_PATH_PREFIX="litellm"`
- Secret name: `AZURE_API_KEY`
LiteLLM will look up:
```
https://vault.example.com:8200/v1/admin/secret/data/AZURE_API_KEY
https://vault.example.com:8200/v1/admin/secret/data/litellm/AZURE_API_KEY
```
### Expected Secret Format
@ -193,4 +197,3 @@ When a Virtual Key is Created / Deleted on LiteLLM, LiteLLM will automatically c
LiteLLM stores secret under the `prefix_for_stored_virtual_keys` path (default: `litellm/`)
<Image img={require('../../img/hcorp_virtual_key.png')} />

2
docs/my-website/docs/tutorials/elasticsearch_logging.md
View file

@ -221,7 +221,7 @@ services:
- elasticsearch
litellm:
image: ghcr.io/berriai/litellm:main-latest
image: docker.litellm.ai/berriai/litellm:main-latest
ports:
- "4000:4000"
environment:

2
docs/my-website/docs/tutorials/openai_codex.md
View file

@ -53,7 +53,7 @@ yarn global add @openai/codex
docker run \
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml
```

2
docs/my-website/release_notes/v1.55.8-stable/index.md
View file

@ -53,7 +53,7 @@ Send LLM usage (spend, tokens) data to [Azure Data Lake](https://learn.microsoft
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable
docker.litellm.ai/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable
```
## Get Daily Updates

2
docs/my-website/release_notes/v1.57.3/index.md
View file

@ -39,7 +39,7 @@ Instead of `apt-get` use `apk`, the base litellm image will no longer have `apt-
**You are only impacted if you use `apt-get` in your Dockerfile**
```shell
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest
FROM docker.litellm.ai/berriai/litellm:main-latest
# Set the working directory
WORKDIR /app

2
docs/my-website/release_notes/v1.63.11-stable/index.md
View file

@ -36,7 +36,7 @@ This release is primarily focused on:
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.63.11-stable
docker.litellm.ai/berriai/litellm:main-v1.63.11-stable
```
## Demo Instance

2
docs/my-website/release_notes/v1.63.14/index.md
View file

@ -32,7 +32,7 @@ This release brings:
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.63.14-stable.patch1
docker.litellm.ai/berriai/litellm:main-v1.63.14-stable.patch1
```
## Demo Instance

2
docs/my-website/release_notes/v1.65.4-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.65.4-stable
docker.litellm.ai/berriai/litellm:main-v1.65.4-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.66.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.66.0-stable
docker.litellm.ai/berriai/litellm:main-v1.66.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.67.4-stable/index.md
View file

@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.67.4-stable
docker.litellm.ai/berriai/litellm:main-v1.67.4-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.68.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.68.0-stable
docker.litellm.ai/berriai/litellm:main-v1.68.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.69.0-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.69.0-stable
docker.litellm.ai/berriai/litellm:main-v1.69.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.70.1-stable/index.md
View file

@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.70.1-stable
docker.litellm.ai/berriai/litellm:main-v1.70.1-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.71.1-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.71.1-stable
docker.litellm.ai/berriai/litellm:main-v1.71.1-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.0-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.0-stable
docker.litellm.ai/berriai/litellm:main-v1.72.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.2-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.2-stable
docker.litellm.ai/berriai/litellm:main-v1.72.2-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.72.6-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run
-e STORE_MODEL_IN_DB=True
-p 4000:4000
ghcr.io/berriai/litellm:main-v1.72.6-stable
docker.litellm.ai/berriai/litellm:main-v1.72.6-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.73.0-stable/index.md
View file

@ -37,7 +37,7 @@ The `non-root` docker image has a known issue around the UI not loading. If you
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.73.0-stable
docker.litellm.ai/berriai/litellm:v1.73.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.73.6-stable/index.md
View file

@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.73.6-stable.patch.1
docker.litellm.ai/berriai/litellm:v1.73.6-stable.patch.1
```
</TabItem>

2
docs/my-website/release_notes/v1.74.0-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.74.0-stable
docker.litellm.ai/berriai/litellm:v1.74.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.74.15-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.74.15-stable
docker.litellm.ai/berriai/litellm:v1.74.15-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.74.3-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.74.3-stable
docker.litellm.ai/berriai/litellm:v1.74.3-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.74.7/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.74.7-stable.patch.1
docker.litellm.ai/berriai/litellm:v1.74.7-stable.patch.1
```
</TabItem>

2
docs/my-website/release_notes/v1.74.9-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.74.9-stable.patch.1
docker.litellm.ai/berriai/litellm:v1.74.9-stable.patch.1
```
</TabItem>

2
docs/my-website/release_notes/v1.75.5-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.5-stable
docker.litellm.ai/berriai/litellm:v1.75.5-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.75.8/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8-stable
docker.litellm.ai/berriai/litellm:v1.75.8-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.76.1-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.76.1
docker.litellm.ai/berriai/litellm:v1.76.1
```
</TabItem>

2
docs/my-website/release_notes/v1.76.3-stable/index.md
View file

@ -35,7 +35,7 @@ This release has a known issue where startup is leading to Out of Memory errors
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.76.3
docker.litellm.ai/berriai/litellm:v1.76.3
```
</TabItem>

2
docs/my-website/release_notes/v1.77.2-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.2-stable
docker.litellm.ai/berriai/litellm:main-v1.77.2-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.77.3-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.77.3-stable
docker.litellm.ai/berriai/litellm:v1.77.3-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.77.5-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.77.5-stable
docker.litellm.ai/berriai/litellm:v1.77.5-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.77.7-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.77.7.rc.1
docker.litellm.ai/berriai/litellm:v1.77.7.rc.1
```
</TabItem>

2
docs/my-website/release_notes/v1.78.0-stable/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.78.0-stable
docker.litellm.ai/berriai/litellm:v1.78.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.78.5-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.78.5-stable
docker.litellm.ai/berriai/litellm:v1.78.5-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.79.0-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.79.0-stable
docker.litellm.ai/berriai/litellm:v1.79.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.79.1-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.79.1-stable
docker.litellm.ai/berriai/litellm:v1.79.1-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.79.3-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.79.3-stable
docker.litellm.ai/berriai/litellm:v1.79.3-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.80.0-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.0-stable
docker.litellm.ai/berriai/litellm:v1.80.0-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.80.10-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.10.rc.1
docker.litellm.ai/berriai/litellm:v1.80.10.rc.1
```
</TabItem>

2
docs/my-website/release_notes/v1.80.5-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.5-stable
docker.litellm.ai/berriai/litellm:v1.80.5-stable
```
</TabItem>

2
docs/my-website/release_notes/v1.80.8-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.8-stable
docker.litellm.ai/berriai/litellm:v1.80.8-stable
```
</TabItem>

3
docs/my-website/sidebars.js
View file

@ -472,6 +472,7 @@ const sidebars = {
"generateContent",
"apply_guardrail",
"bedrock_invoke",
"interactions",
{
type: "category",
label: "/images",
@ -632,6 +633,7 @@ const sidebars = {
"providers/vertex_speech",
"providers/vertex_batch",
"providers/vertex_ocr",
"providers/vertex_ai_agent_engine",
]
},
{
@ -738,6 +740,7 @@ const sidebars = {
"providers/petals",
"providers/publicai",
"providers/predibase",
"providers/pydantic_ai_agent",
"providers/ragflow",
"providers/recraft",
"providers/replicate",

2
docs/my-website/src/pages/index.md
View file

@ -604,7 +604,7 @@ docker run \
-e AZURE_API_KEY=d6*********** \
-e AZURE_API_BASE=https://openai-***********/ \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-latest \
docker.litellm.ai/berriai/litellm:main-latest \
--config /app/config.yaml --detailed_debug
```

22
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -750,9 +750,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_id=model_id,
model_name=model_name,
)
await self.store_unified_file_id( # need to store otherwise any retrieve call will fail
# Fetch the actual file object for the output file
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_output_file_id
)
verbose_logger.debug(
f"Successfully retrieved file object for output_file_id={original_output_file_id}"
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=response.output_file_id,
file_object=None,
file_object=file_object,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: original_output_file_id},
user_api_key_dict=user_api_key_dict,

80
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -18,6 +18,45 @@ def str_to_bool(value: Optional[str]) -> bool:
return value.lower() in ("true", "1", "t", "y", "yes")
def _get_prisma_env() -> dict:
"""Get environment variables for Prisma, handling offline mode if configured."""
prisma_env = os.environ.copy()
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# These env vars prevent Prisma from attempting downloads
prisma_env["NPM_CONFIG_PREFER_OFFLINE"] = "true"
prisma_env["NPM_CONFIG_CACHE"] = os.getenv("NPM_CONFIG_CACHE", "/app/.cache/npm")
return prisma_env
def _get_prisma_command() -> str:
"""Get the Prisma command to use, bypassing Python wrapper in offline mode."""
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
# Primary location where Prisma Python package installs the CLI
default_cli_path = "/app/.cache/prisma-python/binaries/node_modules/.bin/prisma"
# Check if custom path is provided (for flexibility)
custom_cli_path = os.getenv("PRISMA_CLI_PATH")
if custom_cli_path and os.path.exists(custom_cli_path):
logger.info(f"Using custom Prisma CLI at {custom_cli_path}")
return custom_cli_path
# Check the default location
if os.path.exists(default_cli_path):
logger.info(f"Using cached Prisma CLI at {default_cli_path}")
return default_cli_path
# If not found, log warning and fall back
logger.warning(
f"Prisma CLI not found at {default_cli_path}. "
"Falling back to Python wrapper (may attempt downloads)"
)
# Fall back to the Python wrapper (will work in online mode)
return "prisma"
class ProxyExtrasDBManager:
@staticmethod
def _get_prisma_dir() -> str:
@ -57,6 +96,11 @@ class ProxyExtrasDBManager:
init_dir.mkdir(parents=True, exist_ok=True)
database_url = os.getenv("DATABASE_URL")
if not database_url:
logger.error("DATABASE_URL not set")
return False
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
try:
# 1. Generate migration SQL file by comparing empty state to current db state
@ -64,7 +108,7 @@ class ProxyExtrasDBManager:
migration_file = init_dir / "migration.sql"
subprocess.run(
[
"prisma",
_get_prisma_command(),
"migrate",
"diff",
"--from-empty",
@ -75,13 +119,14 @@ class ProxyExtrasDBManager:
stdout=open(migration_file, "w"),
check=True,
timeout=30,
env=prisma_env
)
# 3. Mark the migration as applied since it represents current state
logger.info("Marking baseline migration as applied...")
subprocess.run(
[
"prisma",
_get_prisma_command(),
"migrate",
"resolve",
"--applied",
@ -89,6 +134,7 @@ class ProxyExtrasDBManager:
],
check=True,
timeout=30,
env=prisma_env
)
return True
@ -113,21 +159,26 @@ class ProxyExtrasDBManager:
@staticmethod
def _roll_back_migration(migration_name: str):
"""Mark a specific migration as rolled back"""
# Set up environment for offline mode if configured
prisma_env = _get_prisma_env()
subprocess.run(
["prisma", "migrate", "resolve", "--rolled-back", migration_name],
[_get_prisma_command(), "migrate", "resolve", "--rolled-back", migration_name],
timeout=60,
check=True,
capture_output=True,
env=prisma_env
)
@staticmethod
def _resolve_specific_migration(migration_name: str):
"""Mark a specific migration as applied"""
prisma_env = _get_prisma_env()
subprocess.run(
["prisma", "migrate", "resolve", "--applied", migration_name],
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60,
check=True,
capture_output=True,
env=prisma_env
)
@staticmethod
@ -194,6 +245,10 @@ class ProxyExtrasDBManager:
3. Mark all existing migrations as applied.
"""
database_url = os.getenv("DATABASE_URL")
if not database_url:
logger.error("DATABASE_URL not set")
return
diff_dir = (
Path(migrations_dir)
/ "migrations"
@ -216,7 +271,7 @@ class ProxyExtrasDBManager:
with open(diff_sql_path, "w") as f:
subprocess.run(
[
"prisma",
_get_prisma_command(),
"migrate",
"diff",
"--from-url",
@ -228,6 +283,7 @@ class ProxyExtrasDBManager:
check=True,
timeout=60,
stdout=f,
env=_get_prisma_env()
)
except subprocess.CalledProcessError as e:
logger.warning(f"Failed to generate migration diff: {e.stderr}")
@ -245,7 +301,7 @@ class ProxyExtrasDBManager:
logger.info("Running prisma db execute to apply the migration diff...")
result = subprocess.run(
[
"prisma",
_get_prisma_command(),
"db",
"execute",
"--file",
@ -257,6 +313,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.info(f"prisma db execute stdout: {result.stdout}")
logger.info("✅ Migration diff applied successfully")
@ -274,11 +331,12 @@ class ProxyExtrasDBManager:
try:
logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
["prisma", "migrate", "resolve", "--applied", migration_name],
[_get_prisma_command(), "migrate", "resolve", "--applied", migration_name],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e:
@ -312,11 +370,12 @@ class ProxyExtrasDBManager:
try:
# Set migrations directory for Prisma
result = subprocess.run(
["prisma", "migrate", "deploy"],
[_get_prisma_command(), "migrate", "deploy"],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.info(f"prisma migrate deploy stdout: {result.stdout}")
@ -344,7 +403,7 @@ class ProxyExtrasDBManager:
# Mark the failed migration as rolled back
subprocess.run(
[
"prisma",
_get_prisma_command(),
"migrate",
"resolve",
"--rolled-back",
@ -354,6 +413,7 @@ class ProxyExtrasDBManager:
check=True,
capture_output=True,
text=True,
env=_get_prisma_env()
)
logger.info(
f"✅ Migration {failed_migration} marked as rolled back... retrying"
@ -450,7 +510,7 @@ class ProxyExtrasDBManager:
else:
# Use prisma db push with increased timeout
subprocess.run(
["prisma", "db", "push", "--accept-data-loss"],
[_get_prisma_command(), "db", "push", "--accept-data-loss"],
timeout=60,
check=True,
)

178
litellm/__init__.py
View file

@ -1,4 +1,6 @@
### Hide pydantic namespace conflict warnings globally ###
from __future__ import annotations
import warnings
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
@ -26,18 +28,6 @@ from typing import (
)
from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams
from litellm.types.integrations.datadog import DatadogInitParams
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.caching.caching import Cache, DualCache, RedisCache, InMemoryCache
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES
from litellm.types.utils import (
ImageObject,
BudgetConfig,
all_litellm_params,
all_litellm_params as _litellm_completion_params,
CredentialItem,
PriorityReservationDict,
) # maintain backwards compatibility for root param.
from litellm._logging import (
set_verbose,
_turn_on_debug,
@ -84,12 +74,6 @@ from litellm.constants import (
DEFAULT_SOFT_BUDGET,
DEFAULT_ALLOWED_FAILS,
)
from litellm.integrations.dotprompt import (
global_prompt_manager,
global_prompt_directory,
set_global_prompt_directory,
)
from litellm.types.guardrails import GuardrailItem
from litellm.types.secret_managers.main import (
KeyManagementSystem,
KeyManagementSettings,
@ -98,11 +82,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
DefaultTeamSSOParams,
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.types.utils import (
StandardKeyGenerationConfig,
LlmProviders,
SearchProviders,
)
from litellm.types.utils import LlmProviders
from litellm.types.utils import PriorityReservationSettings
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
@ -287,7 +267,7 @@ disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: LLMClientCache = LLMClientCache()
in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
### DEFAULT AZURE API VERSION ###
@ -295,9 +275,9 @@ AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the lates
### DEFAULT WATSONX API VERSION ###
WATSONX_DEFAULT_API_VERSION = "2024-03-13"
### COHERE EMBEDDINGS DEFAULT TYPE ###
COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: COHERE_EMBEDDING_INPUT_TYPES = "search_document"
COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: "COHERE_EMBEDDING_INPUT_TYPES" = "search_document"
### CREDENTIALS ###
credential_list: List[CredentialItem] = []
credential_list: List["CredentialItem"] = []
### GUARDRAILS ###
llamaguard_model_name: Optional[str] = None
openai_moderations_model_name: Optional[str] = None
@ -333,7 +313,7 @@ caching: bool = (
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional[Cache] = (
cache: Optional["Cache"] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
default_in_memory_ttl: Optional[float] = None
@ -372,7 +352,7 @@ aws_sqs_callback_params: Optional[Dict] = None
generic_logger_headers: Optional[Dict] = None
default_key_generate_params: Optional[Dict] = None
upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None
key_generation_settings: Optional[StandardKeyGenerationConfig] = None
key_generation_settings: Optional["StandardKeyGenerationConfig"] = None
default_internal_user_params: Optional[Dict] = None
default_team_params: Optional[Union[DefaultTeamSSOParams, Dict]] = None
default_team_settings: Optional[List] = None
@ -381,7 +361,7 @@ default_max_internal_user_budget: Optional[float] = None
max_internal_user_budget: Optional[float] = None
max_ui_session_budget: Optional[float] = 10 # $10 USD budgets for UI Chat sessions
internal_user_budget_duration: Optional[str] = None
tag_budget_config: Optional[Dict[str, BudgetConfig]] = None
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
max_end_user_budget: Optional[float] = None
max_end_user_budget_id: Optional[str] = None
disable_end_user_cost_tracking: Optional[bool] = None
@ -404,7 +384,9 @@ public_agent_groups: Optional[List[str]] = None
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, PriorityReservationDict]]] = None
priority_reservation: Optional[
Dict[str, Union[float, "PriorityReservationDict"]]
] = None
priority_reservation_settings: "PriorityReservationSettings" = (
PriorityReservationSettings()
)
@ -422,10 +404,6 @@ disable_aiohttp_trust_env: bool = (
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
module_level_aclient = AsyncHTTPHandler(
timeout=request_timeout, client_alias="module level aclient"
)
module_level_client = HTTPHandler(timeout=request_timeout)
#### RETRIES ####
num_retries: Optional[int] = None # per model endpoint
@ -1071,7 +1049,6 @@ openai_video_generation_models = ["sora-2"]
from .timeout import timeout
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls
from litellm.litellm_core_utils.token_counter import get_modified_max_tokens
# client must be imported immediately as it's used as a decorator at function definition time
from .utils import client
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
@ -1079,8 +1056,6 @@ from .utils import client
from .llms.bytez.chat.transformation import BytezChatConfig
from .llms.custom_llm import CustomLLM
from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from .llms.openai_like.chat.handler import OpenAILikeChatConfig
from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig
from .llms.galadriel.chat.transformation import GaladrielChatConfig
from .llms.github.chat.transformation import GithubChatConfig
@ -1265,6 +1240,7 @@ from .llms.xai.responses.transformation import XAIResponsesAPIConfig
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig,
)
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility
OpenAIOSeriesConfig,
@ -1375,6 +1351,8 @@ from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig
from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig
from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig
## Lazy loading this is not straightforward, will leave it here for now.
from .main import * # type: ignore
# Skills API
@ -1425,6 +1403,9 @@ from .batch_completion.main import * # type: ignore
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
# Interactions API is available as litellm.interactions module
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
from . import interactions
from .skills.main import (
create_skill,
acreate_skill,
@ -1478,7 +1459,6 @@ from . import rag
### CUSTOM LLMs ###
from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[str] = (
@ -1520,6 +1500,17 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
if TYPE_CHECKING:
from litellm.types.utils import ModelInfo as _ModelInfoType
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.caching.caching import Cache
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES
from litellm.types.utils import (
BudgetConfig,
CredentialItem,
PriorityReservationDict,
StandardKeyGenerationConfig,
)
from litellm.types.guardrails import GuardrailItem
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -1560,47 +1551,104 @@ if TYPE_CHECKING:
# Response types - truly lazy loaded only (not in main.py or elsewhere)
ModelResponseListIterator: Type[Any]
# HTTP handler singletons (created lazily via __getattr__ at runtime)
module_level_aclient: AsyncHTTPHandler
module_level_client: HTTPHandler
# LLM config classes - lazy loaded only
AmazonConverseConfig: Type[Any]
OpenAILikeChatConfig: Type[Any]
def __getattr__(name: str) -> Any:
"""Lazy import handler for cost_calculator and litellm_logging functions."""
# Lazy load cost_calculator functions
_cost_calculator_names = (
"completion_cost",
"cost_per_token",
"response_cost_calculator",
"""Lazy import handler"""
from ._lazy_imports import (
COST_CALCULATOR_NAMES,
LITELLM_LOGGING_NAMES,
UTILS_NAMES,
TOKEN_COUNTER_NAMES,
LLM_CLIENT_CACHE_NAMES,
BEDROCK_TYPES_NAMES,
TYPES_UTILS_NAMES,
CACHING_NAMES,
HTTP_HANDLER_NAMES,
DOTPROMPT_NAMES,
LLM_CONFIG_NAMES,
TYPES_NAMES,
)
if name in _cost_calculator_names:
# Lazy load cost_calculator functions
if name in COST_CALCULATOR_NAMES:
from ._lazy_imports import _lazy_import_cost_calculator
return _lazy_import_cost_calculator(name)
# Lazy load litellm_logging functions
_litellm_logging_names = (
"Logging",
"modify_integration",
)
if name in _litellm_logging_names:
if name in LITELLM_LOGGING_NAMES:
from ._lazy_imports import _lazy_import_litellm_logging
return _lazy_import_litellm_logging(name)
# Lazy load utils functions
_utils_names = (
"exception_type", "get_optional_params", "get_response_string", "token_counter",
"create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling",
"supports_web_search", "supports_url_context", "supports_response_schema",
"supports_parallel_function_calling", "supports_vision", "supports_audio_input",
"supports_audio_output", "supports_system_messages", "supports_reasoning",
"get_litellm_params", "acreate", "get_max_tokens", "get_model_info",
"register_prompt_template", "validate_environment", "check_valid_key",
"register_model", "encode", "decode", "_calculate_retry_after", "_should_retry",
"get_supported_openai_params", "get_api_base", "get_first_chars_messages",
"ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse",
"TranscriptionResponse", "TextCompletionResponse", "get_provider_fields",
"ModelResponseListIterator", "get_valid_models",
)
if name in _utils_names:
if name in UTILS_NAMES:
from ._lazy_imports import _lazy_import_utils
return _lazy_import_utils(name)
# Lazy load token counter utilities
if name in TOKEN_COUNTER_NAMES:
from ._lazy_imports import _lazy_import_token_counter
return _lazy_import_token_counter(name)
# Lazy load Bedrock type aliases
if name in BEDROCK_TYPES_NAMES:
from ._lazy_imports import _lazy_import_bedrock_types
return _lazy_import_bedrock_types(name)
# Lazy load common types.utils symbols
if name in TYPES_UTILS_NAMES:
from ._lazy_imports import _lazy_import_types_utils
return _lazy_import_types_utils(name)
# Lazy load LLM client cache and its singleton
if name in LLM_CLIENT_CACHE_NAMES:
from ._lazy_imports import _lazy_import_llm_client_cache
return _lazy_import_llm_client_cache(name)
# Lazy load caching classes
if name in CACHING_NAMES:
from ._lazy_imports import _lazy_import_caching
return _lazy_import_caching(name)
# Lazy-load HTTP handler singletons used across the codebase
if name in HTTP_HANDLER_NAMES:
from ._lazy_imports import _lazy_import_http_handlers
return _lazy_import_http_handlers(name)
# Lazy load dotprompt integration globals
if name in DOTPROMPT_NAMES:
from ._lazy_imports import _lazy_import_dotprompt
return _lazy_import_dotprompt(name)
# Lazy load LLM config classes
if name in LLM_CONFIG_NAMES:
from ._lazy_imports import _lazy_import_llm_configs
return _lazy_import_llm_configs(name)
# Lazy load types
if name in TYPES_NAMES:
from ._lazy_imports import _lazy_import_types
return _lazy_import_types(name)
# Lazy load encoding from main.py to avoid heavy tiktoken import
if name == "encoding":
from .main import encoding as _encoding
# Cache it in the module's __dict__ for subsequent accesses
import sys
sys.modules[__name__].__dict__["encoding"] = _encoding
return _encoding
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

468
litellm/_lazy_imports.py
View file

@ -1,10 +1,170 @@
from typing import Any
from typing import Any, Optional, cast
import sys
def _get_litellm_globals() -> dict:
"""Helper to get the globals dictionary of the litellm module."""
return sys.modules["litellm"].__dict__
# Lazy loader for default encoding to avoid importing tiktoken at module import time
_default_encoding: Optional[Any] = None
def _get_default_encoding() -> Any:
"""
Lazily load and cache the default OpenAI encoding.
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
at `litellm` import time. The encoding is cached after the first import.
This is used internally by utils.py functions that need the encoding but shouldn't
trigger its import during module load.
"""
global _default_encoding
if _default_encoding is None:
from litellm.litellm_core_utils.default_encoding import encoding
_default_encoding = encoding
return _default_encoding
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
_get_modified_max_tokens_func: Optional[Any] = None
def _get_modified_max_tokens() -> Any:
"""
Lazily load and cache the get_modified_max_tokens function.
This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time.
The function is cached after the first import.
This is used internally by utils.py functions that need the token counter but shouldn't
trigger its import during module load.
"""
global _get_modified_max_tokens_func
if _get_modified_max_tokens_func is None:
from litellm.litellm_core_utils.token_counter import (
get_modified_max_tokens as _get_modified_max_tokens_imported,
)
_get_modified_max_tokens_func = _get_modified_max_tokens_imported
return _get_modified_max_tokens_func
# Lazy loader for token_counter to avoid importing token_counter module at module import time
_token_counter_new_func: Optional[Any] = None
def _get_token_counter_new() -> Any:
"""
Lazily load and cache the token_counter function (aliased as token_counter_new).
This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time.
The function is cached after the first import.
This is used internally by utils.py functions that need the token counter but shouldn't
trigger its import during module load.
"""
global _token_counter_new_func
if _token_counter_new_func is None:
from litellm.litellm_core_utils.token_counter import (
token_counter as _token_counter_imported,
)
_token_counter_new_func = _token_counter_imported
return _token_counter_new_func
# Cost calculator names that support lazy loading via _lazy_import_cost_calculator
COST_CALCULATOR_NAMES = (
"completion_cost",
"cost_per_token",
"response_cost_calculator",
)
# Litellm logging names that support lazy loading via _lazy_import_litellm_logging
LITELLM_LOGGING_NAMES = (
"Logging",
"modify_integration",
)
# Utils names that support lazy loading via _lazy_import_utils
UTILS_NAMES = (
"exception_type", "get_optional_params", "get_response_string", "token_counter",
"create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling",
"supports_web_search", "supports_url_context", "supports_response_schema",
"supports_parallel_function_calling", "supports_vision", "supports_audio_input",
"supports_audio_output", "supports_system_messages", "supports_reasoning",
"get_litellm_params", "acreate", "get_max_tokens", "get_model_info",
"register_prompt_template", "validate_environment", "check_valid_key",
"register_model", "encode", "decode", "_calculate_retry_after", "_should_retry",
"get_supported_openai_params", "get_api_base", "get_first_chars_messages",
"ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse",
"TranscriptionResponse", "TextCompletionResponse", "get_provider_fields",
"ModelResponseListIterator", "get_valid_models",
)
# Token counter names that support lazy loading via _lazy_import_token_counter
TOKEN_COUNTER_NAMES = (
"get_modified_max_tokens",
)
# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache
LLM_CLIENT_CACHE_NAMES = (
"LLMClientCache",
"in_memory_llm_clients_cache",
)
# Bedrock type names that support lazy loading via _lazy_import_bedrock_types
BEDROCK_TYPES_NAMES = (
"COHERE_EMBEDDING_INPUT_TYPES",
)
# Common types from litellm.types.utils that support lazy loading via
# _lazy_import_types_utils
TYPES_UTILS_NAMES = (
"ImageObject",
"BudgetConfig",
"all_litellm_params",
"_litellm_completion_params",
"CredentialItem",
"PriorityReservationDict",
"StandardKeyGenerationConfig",
"SearchProviders",
"GenericStreamingChunk",
)
# Caching / cache classes that support lazy loading via _lazy_import_caching
CACHING_NAMES = (
"Cache",
"DualCache",
"RedisCache",
"InMemoryCache",
)
# HTTP handler names that support lazy loading via _lazy_import_http_handlers
HTTP_HANDLER_NAMES = (
"module_level_aclient",
"module_level_client",
)
# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt
DOTPROMPT_NAMES = (
"global_prompt_manager",
"global_prompt_directory",
"set_global_prompt_directory",
)
# LLM config classes that support lazy loading via _lazy_import_llm_configs
LLM_CONFIG_NAMES = (
"AmazonConverseConfig",
"OpenAILikeChatConfig",
)
# Types that support lazy loading via _lazy_import_types
TYPES_NAMES = (
"GuardrailItem",
)
# Lazy import for utils module - imports only the requested item by name.
# Note: PLR0915 (too many statements) is suppressed because the many if statements
# are intentional - each attribute is imported individually only when requested,
@ -218,42 +378,286 @@ def _lazy_import_utils(name: str) -> Any: # noqa: PLR0915
def _lazy_import_cost_calculator(name: str) -> Any:
"""Lazy import for cost_calculator functions."""
_globals = _get_litellm_globals()
from .cost_calculator import (
completion_cost as _completion_cost,
cost_per_token as _cost_per_token,
response_cost_calculator as _response_cost_calculator,
)
if name == "completion_cost":
from .cost_calculator import completion_cost as _completion_cost
_globals["completion_cost"] = _completion_cost
return _completion_cost
_cost_functions = {
"completion_cost": _completion_cost,
"cost_per_token": _cost_per_token,
"response_cost_calculator": _response_cost_calculator,
}
if name == "cost_per_token":
from .cost_calculator import cost_per_token as _cost_per_token
_globals["cost_per_token"] = _cost_per_token
return _cost_per_token
func = _cost_functions[name]
_globals[name] = func
return func
if name == "response_cost_calculator":
from .cost_calculator import response_cost_calculator as _response_cost_calculator
_globals["response_cost_calculator"] = _response_cost_calculator
return _response_cost_calculator
raise AttributeError(f"Cost calculator lazy import: unknown attribute {name!r}")
def _lazy_import_token_counter(name: str) -> Any:
"""Lazy import for token_counter utilities."""
_globals = _get_litellm_globals()
if name == "get_modified_max_tokens":
from litellm.litellm_core_utils.token_counter import (
get_modified_max_tokens as _get_modified_max_tokens,
)
_globals["get_modified_max_tokens"] = _get_modified_max_tokens
return _get_modified_max_tokens
raise AttributeError(f"Token counter lazy import: unknown attribute {name!r}")
def _lazy_import_bedrock_types(name: str) -> Any:
"""Lazy import for Bedrock type aliases."""
_globals = _get_litellm_globals()
if name == "COHERE_EMBEDDING_INPUT_TYPES":
from litellm.types.llms.bedrock import (
COHERE_EMBEDDING_INPUT_TYPES as _COHERE_EMBEDDING_INPUT_TYPES,
)
_globals["COHERE_EMBEDDING_INPUT_TYPES"] = _COHERE_EMBEDDING_INPUT_TYPES
return _COHERE_EMBEDDING_INPUT_TYPES
raise AttributeError(f"Bedrock types lazy import: unknown attribute {name!r}")
def _lazy_import_types_utils(name: str) -> Any:
"""Lazy import for common types and constants from litellm.types.utils."""
_globals = _get_litellm_globals()
if name == "ImageObject":
from .types.utils import ImageObject as _ImageObject
_globals["ImageObject"] = _ImageObject
return _ImageObject
if name == "BudgetConfig":
from .types.utils import BudgetConfig as _BudgetConfig
_globals["BudgetConfig"] = _BudgetConfig
return _BudgetConfig
if name == "all_litellm_params":
from .types.utils import all_litellm_params as _all_litellm_params
_globals["all_litellm_params"] = _all_litellm_params
return _all_litellm_params
if name == "_litellm_completion_params":
from .types.utils import all_litellm_params as _all_litellm_params
_globals["_litellm_completion_params"] = _all_litellm_params
return _all_litellm_params
if name == "CredentialItem":
from .types.utils import CredentialItem as _CredentialItem
_globals["CredentialItem"] = _CredentialItem
return _CredentialItem
if name == "PriorityReservationDict":
from .types.utils import (
PriorityReservationDict as _PriorityReservationDict,
)
_globals["PriorityReservationDict"] = _PriorityReservationDict
return _PriorityReservationDict
if name == "StandardKeyGenerationConfig":
from .types.utils import (
StandardKeyGenerationConfig as _StandardKeyGenerationConfig,
)
_globals["StandardKeyGenerationConfig"] = _StandardKeyGenerationConfig
return _StandardKeyGenerationConfig
if name == "SearchProviders":
from .types.utils import SearchProviders as _SearchProviders
_globals["SearchProviders"] = _SearchProviders
return _SearchProviders
if name == "GenericStreamingChunk":
from .types.utils import (
GenericStreamingChunk as _GenericStreamingChunk,
)
_globals["GenericStreamingChunk"] = _GenericStreamingChunk
return _GenericStreamingChunk
raise AttributeError(f"Types utils lazy import: unknown attribute {name!r}")
def _lazy_import_caching(name: str) -> Any:
"""Lazy import for caching module classes."""
_globals = _get_litellm_globals()
if name == "Cache":
from litellm.caching.caching import Cache as _Cache
_globals["Cache"] = _Cache
return _Cache
if name == "DualCache":
from litellm.caching.caching import DualCache as _DualCache
_globals["DualCache"] = _DualCache
return _DualCache
if name == "RedisCache":
from litellm.caching.caching import RedisCache as _RedisCache
_globals["RedisCache"] = _RedisCache
return _RedisCache
if name == "InMemoryCache":
from litellm.caching.caching import InMemoryCache as _InMemoryCache
_globals["InMemoryCache"] = _InMemoryCache
return _InMemoryCache
raise AttributeError(f"Caching lazy import: unknown attribute {name!r}")
def _lazy_import_llm_client_cache(name: str) -> Any:
"""Lazy import for LLM client cache class and singleton."""
_globals = _get_litellm_globals()
if name == "LLMClientCache":
from litellm.caching.llm_caching_handler import LLMClientCache as _LLMClientCache
_globals["LLMClientCache"] = _LLMClientCache
return _LLMClientCache
if name == "in_memory_llm_clients_cache":
from litellm.caching.llm_caching_handler import LLMClientCache as _LLMClientCache
instance = _LLMClientCache()
# Only populate the requested singleton name to keep lazy-import
# semantics consistent with other helpers (no extra symbols).
_globals["in_memory_llm_clients_cache"] = instance
return instance
raise AttributeError(f"LLM client cache lazy import: unknown attribute {name!r}")
def _lazy_import_litellm_logging(name: str) -> Any:
"""Lazy import for litellm_logging module."""
_globals = _get_litellm_globals()
try:
from litellm.litellm_core_utils.litellm_logging import (
Logging as _Logging,
modify_integration as _modify_integration,
if name == "Logging":
from litellm.litellm_core_utils.litellm_logging import Logging as _Logging
_globals["Logging"] = _Logging
return _Logging
if name == "modify_integration":
from litellm.litellm_core_utils.litellm_logging import modify_integration as _modify_integration
_globals["modify_integration"] = _modify_integration
return _modify_integration
raise AttributeError(f"Litellm logging lazy import: unknown attribute {name!r}")
def _lazy_import_http_handlers(name: str) -> Any:
"""Lazy import and instantiate module-level HTTP handlers."""
_globals = _get_litellm_globals()
if name == "module_level_aclient":
# Use shared async client factory instead of directly instantiating AsyncHTTPHandler
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
timeout = _globals.get("request_timeout")
params = {"timeout": timeout, "client_alias": "module level aclient"}
# llm_provider is only used for cache keying; use a string identifier but
# cast to Any so static type checkers don't complain about the literal.
provider_id = cast(Any, "litellm_module_level_client")
async_client = get_async_httpx_client(
llm_provider=provider_id,
params=params,
)
_logging_objects = {
"Logging": _Logging,
"modify_integration": _modify_integration,
}
obj = _logging_objects[name]
_globals[name] = obj
return obj
except Exception as e:
raise AttributeError(
f"module 'litellm' has no attribute {name!r}. "
f"Lazy import failed: {e}"
) from e
_globals["module_level_aclient"] = async_client
return async_client
if name == "module_level_client":
# Import handler type locally to avoid heavy imports at module load time
from litellm.llms.custom_httpx.http_handler import HTTPHandler
timeout = _globals.get("request_timeout")
sync_client = HTTPHandler(timeout=timeout)
_globals["module_level_client"] = sync_client
return sync_client
raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}")
def _lazy_import_dotprompt(name: str) -> Any:
"""Lazy import for dotprompt integration globals."""
_globals = _get_litellm_globals()
if name == "global_prompt_manager":
from litellm.integrations.dotprompt import (
global_prompt_manager as _global_prompt_manager,
)
_globals["global_prompt_manager"] = _global_prompt_manager
return _global_prompt_manager
if name == "global_prompt_directory":
from litellm.integrations.dotprompt import (
global_prompt_directory as _global_prompt_directory,
)
_globals["global_prompt_directory"] = _global_prompt_directory
return _global_prompt_directory
if name == "set_global_prompt_directory":
from litellm.integrations.dotprompt import (
set_global_prompt_directory as _set_global_prompt_directory,
)
_globals["set_global_prompt_directory"] = _set_global_prompt_directory
return _set_global_prompt_directory
raise AttributeError(f"Dotprompt lazy import: unknown attribute {name!r}")
def _lazy_import_types(name: str) -> Any:
"""Lazy import for type classes."""
_globals = _get_litellm_globals()
if name == "GuardrailItem":
from litellm.types.guardrails import (
GuardrailItem as _GuardrailItem,
)
_globals["GuardrailItem"] = _GuardrailItem
return _GuardrailItem
raise AttributeError(f"Types lazy import: unknown attribute {name!r}")
def _lazy_import_llm_configs(name: str) -> Any:
"""Lazy import for LLM config classes."""
_globals = _get_litellm_globals()
if name == "AmazonConverseConfig":
from .llms.bedrock.chat.converse_transformation import (
AmazonConverseConfig as _AmazonConverseConfig,
)
_globals["AmazonConverseConfig"] = _AmazonConverseConfig
return _AmazonConverseConfig
if name == "OpenAILikeChatConfig":
from .llms.openai_like.chat.handler import (
OpenAILikeChatConfig as _OpenAILikeChatConfig,
)
_globals["OpenAILikeChatConfig"] = _OpenAILikeChatConfig
return _OpenAILikeChatConfig
raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}")

48
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -18,6 +18,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2ACompletionBridgeTransformation,
A2AStreamingContext,
)
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
class A2ACompletionBridgeHandler:
@ -44,6 +45,29 @@ class A2ACompletionBridgeHandler:
Returns:
A2A SendMessageResponse dict
"""
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider}"
)
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
)
return response_data
# Extract message from params
message = params.get("message", {})
@ -119,6 +143,30 @@ class A2ACompletionBridgeHandler:
Yields:
A2A streaming response events
"""
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
async for chunk in a2a_provider_config.handle_streaming(
request_id=request_id,
params=params,
api_base=api_base,
):
yield chunk
return
# Extract message from params
message = params.get("message", {})

11
litellm/a2a_protocol/providers/__init__.py Normal file
View file

@ -0,0 +1,11 @@
"""
A2A Protocol Providers.
This module contains provider-specific implementations for the A2A protocol.
"""
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
__all__ = ["BaseA2AProviderConfig", "A2AProviderConfigManager"]

63
litellm/a2a_protocol/providers/base.py Normal file
View file

@ -0,0 +1,63 @@
"""
Base configuration for A2A protocol providers.
"""
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator, Dict
class BaseA2AProviderConfig(ABC):
"""
Base configuration class for A2A protocol providers.
Each provider should implement this interface to define how to handle
A2A requests for their specific agent type.
"""
@abstractmethod
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
**kwargs,
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
api_base: Base URL of the agent
**kwargs: Additional provider-specific parameters
Returns:
A2A SendMessageResponse dict
"""
pass
@abstractmethod
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
api_base: Base URL of the agent
**kwargs: Additional provider-specific parameters
Yields:
A2A streaming response events
"""
# This is an abstract method - subclasses must implement
# The yield is here to make this a generator function
if False: # pragma: no cover
yield {}

48
litellm/a2a_protocol/providers/config_manager.py Normal file
View file

@ -0,0 +1,48 @@
"""
A2A Provider Config Manager.
Manages provider-specific configurations for A2A protocol.
"""
from typing import Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
class A2AProviderConfigManager:
"""
Manager for A2A provider configurations.
Similar to ProviderConfigManager in litellm.utils but specifically for A2A providers.
"""
@staticmethod
def get_provider_config(
custom_llm_provider: Optional[str],
) -> Optional[BaseA2AProviderConfig]:
"""
Get the provider configuration for a given custom_llm_provider.
Args:
custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents")
Returns:
Provider configuration instance or None if not found
"""
if custom_llm_provider is None:
return None
if custom_llm_provider == "pydantic_ai_agents":
from litellm.a2a_protocol.providers.pydantic_ai_agents.config import (
PydanticAIProviderConfig,
)
return PydanticAIProviderConfig()
# Add more providers here as needed
# elif custom_llm_provider == "another_provider":
# from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig
# return AnotherProviderConfig()
return None

74
litellm/a2a_protocol/providers/litellm_completion/README.md Normal file
View file

@ -0,0 +1,74 @@
# A2A to LiteLLM Completion Bridge
Routes A2A protocol requests through `litellm.acompletion`, enabling any LiteLLM-supported provider to be invoked via A2A.
## Flow
```
A2A Request → Transform → litellm.acompletion → Transform → A2A Response
```
## SDK Usage
Use the existing `asend_message` and `asend_message_streaming` functions with `litellm_params`:
```python
from litellm.a2a_protocol import asend_message, asend_message_streaming
from a2a.types import SendMessageRequest, SendStreamingMessageRequest, MessageSendParams
from uuid import uuid4
# Non-streaming
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
)
)
response = await asend_message(
request=request,
api_base="http://localhost:2024",
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
)
# Streaming
stream_request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
)
)
async for chunk in asend_message_streaming(
request=stream_request,
api_base="http://localhost:2024",
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
):
print(chunk)
```
## Proxy Usage
Configure an agent with `custom_llm_provider` in `litellm_params`:
```yaml
agents:
- agent_name: my-langgraph-agent
agent_card_params:
name: "LangGraph Agent"
url: "http://localhost:2024" # Used as api_base
litellm_params:
custom_llm_provider: langgraph
model: agent
```
When an A2A request hits `/a2a/{agent_id}/message/send`, the bridge:
1. Detects `custom_llm_provider` in agent's `litellm_params`
2. Transforms A2A message → OpenAI messages
3. Calls `litellm.acompletion(model="langgraph/agent", api_base="http://localhost:2024")`
4. Transforms response → A2A format
## Classes
- `A2ACompletionBridgeTransformation` - Static methods for message format conversion
- `A2ACompletionBridgeHandler` - Static methods for handling requests (streaming/non-streaming)

6
litellm/a2a_protocol/providers/litellm_completion/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""
LiteLLM Completion bridge provider for A2A protocol.
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
"""

295
litellm/a2a_protocol/providers/litellm_completion/handler.py Normal file
View file

@ -0,0 +1,295 @@
"""
Handler for A2A to LiteLLM completion bridge.
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
A2A Streaming Events (in order):
1. Task event (kind: "task") - Initial task creation with status "submitted"
2. Status update (kind: "status-update") - Status change to "working"
3. Artifact update (kind: "artifact-update") - Content/artifact delivery
4. Status update (kind: "status-update") - Final status "completed" with final=true
"""
from typing import Any, AsyncIterator, Dict, Optional
import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.litellm_completion_bridge.pydantic_ai_transformation import (
PydanticAITransformation,
)
from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2ACompletionBridgeTransformation,
A2AStreamingContext,
)
class A2ACompletionBridgeHandler:
"""
Static methods for handling A2A requests via LiteLLM completion.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request via litellm.acompletion.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
Returns:
A2A SendMessageResponse dict
"""
# Check if this is a Pydantic AI agent request
custom_llm_provider = litellm_params.get("custom_llm_provider")
if custom_llm_provider == "pydantic_ai_agents":
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Routing to Pydantic AI agent at {api_base}"
)
# Send request directly to Pydantic AI agent
response_data = await PydanticAITransformation.send_non_streaming_request(
api_base=api_base,
request_id=request_id,
params=params,
)
return response_data
# Extract message from params
message = params.get("message", {})
# Transform A2A message to OpenAI format
openai_messages = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(
message
)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
model = litellm_params.get("model", "agent")
# Build full model string if provider specified
# Skip prepending if model already starts with the provider prefix
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
full_model = f"{custom_llm_provider}/{model}"
else:
full_model = model
verbose_logger.info(
f"A2A completion bridge: model={full_model}, api_base={api_base}"
)
# Build completion params dict
completion_params = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": False,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add = {
k: v for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
completion_params.update(litellm_params_to_add)
# Call litellm.acompletion
response = await litellm.acompletion(**completion_params)
# Transform response to A2A format
a2a_response = A2ACompletionBridgeTransformation.openai_response_to_a2a_response(
response=response,
request_id=request_id,
)
verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}")
return a2a_response
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request via litellm.acompletion with stream=True.
Emits proper A2A streaming events:
1. Task event (kind: "task") - Initial task with status "submitted"
2. Status update (kind: "status-update") - Status "working"
3. Artifact update (kind: "artifact-update") - Content delivery
4. Status update (kind: "status-update") - Final "completed" status
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
Yields:
A2A streaming response events
"""
# Check if this is a Pydantic AI agent request
custom_llm_provider = litellm_params.get("custom_llm_provider")
if custom_llm_provider == "pydantic_ai_agents":
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
)
# Get non-streaming response first
response_data = await PydanticAITransformation.send_non_streaming_request(
api_base=api_base,
request_id=request_id,
params=params,
)
# Convert to fake streaming
async for chunk in PydanticAITransformation.fake_streaming_from_response(
response_data=response_data,
request_id=request_id,
):
yield chunk
return
# Extract message from params
message = params.get("message", {})
# Create streaming context
ctx = A2AStreamingContext(
request_id=request_id,
input_message=message,
)
# Transform A2A message to OpenAI format
openai_messages = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(
message
)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
model = litellm_params.get("model", "agent")
# Build full model string if provider specified
# Skip prepending if model already starts with the provider prefix
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
full_model = f"{custom_llm_provider}/{model}"
else:
full_model = model
verbose_logger.info(
f"A2A completion bridge streaming: model={full_model}, api_base={api_base}"
)
# Build completion params dict
completion_params = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": True,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add = {
k: v for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
completion_params.update(litellm_params_to_add)
# 1. Emit initial task event (kind: "task", status: "submitted")
task_event = A2ACompletionBridgeTransformation.create_task_event(ctx)
yield task_event
# 2. Emit status update (kind: "status-update", status: "working")
working_event = A2ACompletionBridgeTransformation.create_status_update_event(
ctx=ctx,
state="working",
final=False,
message_text="Processing request...",
)
yield working_event
# Call litellm.acompletion with streaming
response = await litellm.acompletion(**completion_params)
# 3. Accumulate content and emit artifact update
accumulated_text = ""
chunk_count = 0
async for chunk in response: # type: ignore[union-attr]
chunk_count += 1
# Extract delta content
content = ""
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
choice = chunk.choices[0]
if hasattr(choice, "delta") and choice.delta:
content = choice.delta.content or ""
if content:
accumulated_text += content
# Emit artifact update with accumulated content
if accumulated_text:
artifact_event = A2ACompletionBridgeTransformation.create_artifact_update_event(
ctx=ctx,
text=accumulated_text,
)
yield artifact_event
# 4. Emit final status update (kind: "status-update", status: "completed", final: true)
completed_event = A2ACompletionBridgeTransformation.create_status_update_event(
ctx=ctx,
state="completed",
final=True,
)
yield completed_event
verbose_logger.info(
f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}"
)
# Convenience functions that delegate to the class methods
async def handle_a2a_completion(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> Dict[str, Any]:
"""Convenience function for non-streaming A2A completion."""
return await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
)
async def handle_a2a_completion_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> AsyncIterator[Dict[str, Any]]:
"""Convenience function for streaming A2A completion."""
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
):
yield chunk

286
litellm/a2a_protocol/providers/litellm_completion/transformation.py Normal file
View file

@ -0,0 +1,286 @@
"""
Transformation utilities for A2A <-> OpenAI message format conversion.
A2A Message Format:
{
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": "abc123"
}
OpenAI Message Format:
{"role": "user", "content": "Hello!"}
A2A Streaming Events:
- Task event (kind: "task") - Initial task creation with status "submitted"
- Status update (kind: "status-update") - Status changes (working, completed)
- Artifact update (kind: "artifact-update") - Content/artifact delivery
"""
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from uuid import uuid4
from litellm._logging import verbose_logger
class A2AStreamingContext:
"""
Context holder for A2A streaming state.
Tracks task_id, context_id, and message accumulation.
"""
def __init__(self, request_id: str, input_message: Dict[str, Any]):
self.request_id = request_id
self.task_id = str(uuid4())
self.context_id = str(uuid4())
self.input_message = input_message
self.accumulated_text = ""
self.has_emitted_task = False
self.has_emitted_working = False
class A2ACompletionBridgeTransformation:
"""
Static methods for transforming between A2A and OpenAI message formats.
"""
@staticmethod
def a2a_message_to_openai_messages(
a2a_message: Dict[str, Any],
) -> List[Dict[str, str]]:
"""
Transform an A2A message to OpenAI message format.
Args:
a2a_message: A2A message with role, parts, and messageId
Returns:
List of OpenAI-format messages
"""
role = a2a_message.get("role", "user")
parts = a2a_message.get("parts", [])
# Map A2A roles to OpenAI roles
openai_role = role
if role == "user":
openai_role = "user"
elif role == "assistant":
openai_role = "assistant"
elif role == "system":
openai_role = "system"
# Extract text content from parts
content_parts = []
for part in parts:
kind = part.get("kind", "")
if kind == "text":
text = part.get("text", "")
content_parts.append(text)
content = "\n".join(content_parts) if content_parts else ""
verbose_logger.debug(
f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}"
)
return [{"role": openai_role, "content": content}]
@staticmethod
def openai_response_to_a2a_response(
response: Any,
request_id: Optional[str] = None,
) -> Dict[str, Any]:
"""
Transform a LiteLLM ModelResponse to A2A SendMessageResponse format.
Args:
response: LiteLLM ModelResponse object
request_id: Original A2A request ID
Returns:
A2A SendMessageResponse dict
"""
# Extract content from response
content = ""
if hasattr(response, "choices") and response.choices:
choice = response.choices[0]
if hasattr(choice, "message") and choice.message:
content = choice.message.content or ""
# Build A2A message
a2a_message = {
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
}
# Build A2A response
a2a_response = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": a2a_message,
},
}
verbose_logger.debug(
f"OpenAI -> A2A transform: content_length={len(content)}"
)
return a2a_response
@staticmethod
def _get_timestamp() -> str:
"""Get current timestamp in ISO format with timezone."""
return datetime.now(timezone.utc).isoformat()
@staticmethod
def create_task_event(
ctx: A2AStreamingContext,
) -> Dict[str, Any]:
"""
Create the initial task event with status 'submitted'.
This is the first event emitted in an A2A streaming response.
"""
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"contextId": ctx.context_id,
"history": [
{
"contextId": ctx.context_id,
"kind": "message",
"messageId": ctx.input_message.get("messageId", uuid4().hex),
"parts": ctx.input_message.get("parts", []),
"role": ctx.input_message.get("role", "user"),
"taskId": ctx.task_id,
}
],
"id": ctx.task_id,
"kind": "task",
"status": {
"state": "submitted",
},
},
}
@staticmethod
def create_status_update_event(
ctx: A2AStreamingContext,
state: str,
final: bool = False,
message_text: Optional[str] = None,
) -> Dict[str, Any]:
"""
Create a status update event.
Args:
ctx: Streaming context
state: Status state ('working', 'completed')
final: Whether this is the final event
message_text: Optional message text for 'working' status
"""
status: Dict[str, Any] = {
"state": state,
"timestamp": A2ACompletionBridgeTransformation._get_timestamp(),
}
# Add message for 'working' status
if state == "working" and message_text:
status["message"] = {
"contextId": ctx.context_id,
"kind": "message",
"messageId": str(uuid4()),
"parts": [{"kind": "text", "text": message_text}],
"role": "agent",
"taskId": ctx.task_id,
}
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"contextId": ctx.context_id,
"final": final,
"kind": "status-update",
"status": status,
"taskId": ctx.task_id,
},
}
@staticmethod
def create_artifact_update_event(
ctx: A2AStreamingContext,
text: str,
) -> Dict[str, Any]:
"""
Create an artifact update event with content.
Args:
ctx: Streaming context
text: The text content for the artifact
"""
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"artifact": {
"artifactId": str(uuid4()),
"name": "response",
"parts": [{"kind": "text", "text": text}],
},
"contextId": ctx.context_id,
"kind": "artifact-update",
"taskId": ctx.task_id,
},
}
@staticmethod
def openai_chunk_to_a2a_chunk(
chunk: Any,
request_id: Optional[str] = None,
is_final: bool = False,
) -> Optional[Dict[str, Any]]:
"""
Transform a LiteLLM streaming chunk to A2A streaming format.
NOTE: This method is deprecated for streaming. Use the event-based
methods (create_task_event, create_status_update_event,
create_artifact_update_event) instead for proper A2A streaming.
Args:
chunk: LiteLLM ModelResponse chunk
request_id: Original A2A request ID
is_final: Whether this is the final chunk
Returns:
A2A streaming chunk dict or None if no content
"""
# Extract delta content
content = ""
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
choice = chunk.choices[0]
if hasattr(choice, "delta") and choice.delta:
content = choice.delta.content or ""
if not content and not is_final:
return None
# Build A2A streaming chunk (legacy format)
a2a_chunk = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": {
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
},
"final": is_final,
},
}
return a2a_chunk

17
litellm/a2a_protocol/providers/pydantic_ai_agents/__init__.py Normal file
View file

@ -0,0 +1,17 @@
"""
Pydantic AI agent provider for A2A protocol.
Pydantic AI agents follow A2A protocol but don't support streaming natively.
This provider handles fake streaming by converting non-streaming responses into streaming chunks.
"""
from litellm.a2a_protocol.providers.pydantic_ai_agents.config import (
PydanticAIProviderConfig,
)
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import (
PydanticAITransformation,
)
__all__ = ["PydanticAIHandler", "PydanticAITransformation", "PydanticAIProviderConfig"]

51
litellm/a2a_protocol/providers/pydantic_ai_agents/config.py Normal file
View file

@ -0,0 +1,51 @@
"""
Pydantic AI provider configuration.
"""
from typing import Any, AsyncIterator, Dict
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
class PydanticAIProviderConfig(BaseA2AProviderConfig):
"""
Provider configuration for Pydantic AI agents.
Pydantic AI agents follow A2A protocol but don't support streaming natively.
This config provides fake streaming by converting non-streaming responses into streaming chunks.
"""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to Pydantic AI agent."""
return await PydanticAIHandler.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
timeout=kwargs.get("timeout", 60.0),
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request with fake streaming."""
async for chunk in PydanticAIHandler.handle_streaming(
request_id=request_id,
params=params,
api_base=api_base,
timeout=kwargs.get("timeout", 60.0),
chunk_size=kwargs.get("chunk_size", 50),
delay_ms=kwargs.get("delay_ms", 10),
):
yield chunk

106
litellm/a2a_protocol/providers/pydantic_ai_agents/handler.py Normal file
View file

@ -0,0 +1,106 @@
"""
Handler for Pydantic AI agents.
Pydantic AI agents follow A2A protocol but don't support streaming natively.
This handler provides fake streaming by converting non-streaming responses into streaming chunks.
"""
from typing import Any, AsyncIterator, Dict
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import (
PydanticAITransformation,
)
class PydanticAIHandler:
"""
Handler for Pydantic AI agent requests.
Provides:
- Direct non-streaming requests to Pydantic AI agents
- Fake streaming by converting non-streaming responses into streaming chunks
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
Handle non-streaming request to Pydantic AI agent.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
api_base: Base URL of the Pydantic AI agent
timeout: Request timeout in seconds
Returns:
A2A SendMessageResponse dict
"""
verbose_logger.info(
f"Pydantic AI: Routing to Pydantic AI agent at {api_base}"
)
# Send request directly to Pydantic AI agent
response_data = await PydanticAITransformation.send_non_streaming_request(
api_base=api_base,
request_id=request_id,
params=params,
timeout=timeout,
)
return response_data
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
api_base: str,
timeout: float = 60.0,
chunk_size: int = 50,
delay_ms: int = 10,
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming request to Pydantic AI agent with fake streaming.
Since Pydantic AI agents don't support streaming natively, this method:
1. Makes a non-streaming request
2. Converts the response into streaming chunks
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
api_base: Base URL of the Pydantic AI agent
timeout: Request timeout in seconds
chunk_size: Number of characters per chunk
delay_ms: Delay between chunks in milliseconds
Yields:
A2A streaming response events
"""
verbose_logger.info(
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
)
# Get raw task response first (not the transformed A2A format)
raw_response = await PydanticAITransformation.send_and_get_raw_response(
api_base=api_base,
request_id=request_id,
params=params,
timeout=timeout,
)
# Convert raw task response to fake streaming chunks
async for chunk in PydanticAITransformation.fake_streaming_from_response(
response_data=raw_response,
request_id=request_id,
chunk_size=chunk_size,
delay_ms=delay_ms,
):
yield chunk

525
litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py Normal file
View file

@ -0,0 +1,525 @@
"""
Transformation layer for Pydantic AI agents.
Pydantic AI agents follow A2A protocol but don't support streaming.
This module provides fake streaming by converting non-streaming responses into streaming chunks.
"""
import asyncio
from typing import Any, AsyncIterator, Dict, cast
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
class PydanticAITransformation:
"""
Transformation layer for Pydantic AI agents.
Handles:
- Direct A2A requests to Pydantic AI endpoints
- Polling for task completion (since Pydantic AI doesn't support streaming)
- Fake streaming by chunking non-streaming responses
"""
@staticmethod
def _remove_none_values(obj: Any) -> Any:
"""
Recursively remove None values from a dict/list structure.
FastA2A/Pydantic AI servers don't accept None values for optional fields -
they expect those fields to be omitted entirely.
Args:
obj: Dict, list, or other value to clean
Returns:
Cleaned object with None values removed
"""
if isinstance(obj, dict):
return {
k: PydanticAITransformation._remove_none_values(v)
for k, v in obj.items()
if v is not None
}
elif isinstance(obj, list):
return [
PydanticAITransformation._remove_none_values(item)
for item in obj
if item is not None
]
else:
return obj
@staticmethod
def _params_to_dict(params: Any) -> Dict[str, Any]:
"""
Convert params to a dict, handling Pydantic models.
Args:
params: Dict or Pydantic model
Returns:
Dict representation of params
"""
if hasattr(params, "model_dump"):
# Pydantic v2 model
return params.model_dump(mode="python", exclude_none=True)
elif hasattr(params, "dict"):
# Pydantic v1 model
return params.dict(exclude_none=True)
elif isinstance(params, dict):
return params
else:
# Try to convert to dict
return dict(params)
@staticmethod
async def _poll_for_completion(
client: AsyncHTTPHandler,
endpoint: str,
task_id: str,
request_id: str,
max_attempts: int = 30,
poll_interval: float = 0.5,
) -> Dict[str, Any]:
"""
Poll for task completion using tasks/get method.
Args:
client: HTTPX async client
endpoint: API endpoint URL
task_id: Task ID to poll for
request_id: JSON-RPC request ID
max_attempts: Maximum polling attempts
poll_interval: Seconds between poll attempts
Returns:
Completed task response
"""
for attempt in range(max_attempts):
poll_request = {
"jsonrpc": "2.0",
"id": f"{request_id}-poll-{attempt}",
"method": "tasks/get",
"params": {"id": task_id},
}
response = await client.post(
endpoint,
json=poll_request,
headers={"Content-Type": "application/json"},
)
response.raise_for_status()
poll_data = response.json()
result = poll_data.get("result", {})
status = result.get("status", {})
state = status.get("state", "")
verbose_logger.debug(
f"Pydantic AI: Poll attempt {attempt + 1}/{max_attempts}, state={state}"
)
if state == "completed":
return poll_data
elif state in ("failed", "canceled"):
raise Exception(f"Task {task_id} ended with state: {state}")
await asyncio.sleep(poll_interval)
raise TimeoutError(f"Task {task_id} did not complete within {max_attempts * poll_interval} seconds")
@staticmethod
async def _send_and_poll_raw(
api_base: str,
request_id: str,
params: Any,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
Send a request to Pydantic AI agent and return the raw task response.
This is an internal method used by both non-streaming and streaming handlers.
Returns the raw Pydantic AI task format with history/artifacts.
Args:
api_base: Base URL of the Pydantic AI agent
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
timeout: Request timeout in seconds
Returns:
Raw Pydantic AI task response (with history/artifacts)
"""
# Convert params to dict if it's a Pydantic model
params_dict = PydanticAITransformation._params_to_dict(params)
# Remove None values - FastA2A doesn't accept null for optional fields
params_dict = PydanticAITransformation._remove_none_values(params_dict)
# Ensure the message has 'kind': 'message' as required by FastA2A/Pydantic AI
if "message" in params_dict:
params_dict["message"]["kind"] = "message"
# Build A2A JSON-RPC request using message/send method for FastA2A compatibility
a2a_request = {
"jsonrpc": "2.0",
"id": request_id,
"method": "message/send",
"params": params_dict,
}
# FastA2A uses root endpoint (/) not /messages
endpoint = api_base.rstrip("/")
verbose_logger.info(
f"Pydantic AI: Sending non-streaming request to {endpoint}"
)
# Send request to Pydantic AI agent using shared async HTTP client
client = get_async_httpx_client(
llm_provider=cast(Any, "pydantic_ai_agent"),
params={"timeout": timeout},
)
response = await client.post(
endpoint,
json=a2a_request,
headers={"Content-Type": "application/json"},
)
response.raise_for_status()
response_data = response.json()
# Check if task is already completed
result = response_data.get("result", {})
status = result.get("status", {})
state = status.get("state", "")
if state != "completed":
# Need to poll for completion
task_id = result.get("id")
if task_id:
verbose_logger.info(
f"Pydantic AI: Task {task_id} submitted, polling for completion..."
)
response_data = await PydanticAITransformation._poll_for_completion(
client=client,
endpoint=endpoint,
task_id=task_id,
request_id=request_id,
)
verbose_logger.info(f"Pydantic AI: Received completed response for request_id={request_id}")
return response_data
@staticmethod
async def send_non_streaming_request(
api_base: str,
request_id: str,
params: Any,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
Send a non-streaming A2A request to Pydantic AI agent and wait for completion.
Args:
api_base: Base URL of the Pydantic AI agent (e.g., "http://localhost:9999")
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message (dict or Pydantic model)
timeout: Request timeout in seconds
Returns:
Standard A2A non-streaming response format with message
"""
# Get raw task response
raw_response = await PydanticAITransformation._send_and_poll_raw(
api_base=api_base,
request_id=request_id,
params=params,
timeout=timeout,
)
# Transform to standard A2A non-streaming format
return PydanticAITransformation._transform_to_a2a_response(
response_data=raw_response,
request_id=request_id,
)
@staticmethod
async def send_and_get_raw_response(
api_base: str,
request_id: str,
params: Any,
timeout: float = 60.0,
) -> Dict[str, Any]:
"""
Send a request to Pydantic AI agent and return the raw task response.
Used by streaming handler to get raw response for fake streaming.
Args:
api_base: Base URL of the Pydantic AI agent
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
timeout: Request timeout in seconds
Returns:
Raw Pydantic AI task response (with history/artifacts)
"""
return await PydanticAITransformation._send_and_poll_raw(
api_base=api_base,
request_id=request_id,
params=params,
timeout=timeout,
)
@staticmethod
def _transform_to_a2a_response(
response_data: Dict[str, Any],
request_id: str,
) -> Dict[str, Any]:
"""
Transform Pydantic AI task response to standard A2A non-streaming format.
Pydantic AI returns a task with history/artifacts, but the standard A2A
non-streaming format expects:
{
"jsonrpc": "2.0",
"id": "...",
"result": {
"message": {
"role": "agent",
"parts": [{"kind": "text", "text": "..."}],
"messageId": "..."
}
}
}
Args:
response_data: Pydantic AI task response
request_id: Original request ID
Returns:
Standard A2A non-streaming response format
"""
# Extract the agent response text
full_text, message_id, parts = PydanticAITransformation._extract_response_text(
response_data
)
# Build standard A2A message
a2a_message = {
"role": "agent",
"parts": parts if parts else [{"kind": "text", "text": full_text}],
"messageId": message_id,
}
# Return standard A2A non-streaming format
return {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": a2a_message,
},
}
@staticmethod
def _extract_response_text(response_data: Dict[str, Any]) -> tuple[str, str, list]:
"""
Extract response text from completed task response.
Pydantic AI returns completed tasks with:
- history: list of messages (user and agent)
- artifacts: list of result artifacts
Args:
response_data: Completed task response
Returns:
Tuple of (full_text, message_id, parts)
"""
result = response_data.get("result", {})
# Try to extract from artifacts first (preferred for results)
artifacts = result.get("artifacts", [])
if artifacts:
for artifact in artifacts:
parts = artifact.get("parts", [])
for part in parts:
if part.get("kind") == "text":
text = part.get("text", "")
if text:
return text, str(uuid4()), parts
# Fall back to history - get the last agent message
history = result.get("history", [])
for msg in reversed(history):
if msg.get("role") == "agent":
parts = msg.get("parts", [])
message_id = msg.get("messageId", str(uuid4()))
full_text = ""
for part in parts:
if part.get("kind") == "text":
full_text += part.get("text", "")
if full_text:
return full_text, message_id, parts
# Fall back to message field (original format)
message = result.get("message", {})
if message:
parts = message.get("parts", [])
message_id = message.get("messageId", str(uuid4()))
full_text = ""
for part in parts:
if part.get("kind") == "text":
full_text += part.get("text", "")
return full_text, message_id, parts
return "", str(uuid4()), []
@staticmethod
async def fake_streaming_from_response(
response_data: Dict[str, Any],
request_id: str,
chunk_size: int = 50,
delay_ms: int = 10,
) -> AsyncIterator[Dict[str, Any]]:
"""
Convert a non-streaming A2A response into fake streaming chunks.
Emits proper A2A streaming events:
1. Task event (kind: "task") - Initial task with status "submitted"
2. Status update (kind: "status-update") - Status "working"
3. Artifact update chunks (kind: "artifact-update") - Content delivery in chunks
4. Status update (kind: "status-update") - Final "completed" status
Args:
response_data: Non-streaming A2A response dict (completed task)
request_id: A2A JSON-RPC request ID
chunk_size: Number of characters per chunk (default: 50)
delay_ms: Delay between chunks in milliseconds (default: 10)
Yields:
A2A streaming response events
"""
# Extract the response text from completed task
full_text, message_id, parts = PydanticAITransformation._extract_response_text(
response_data
)
# Extract input message from raw response for history
result = response_data.get("result", {})
history = result.get("history", [])
input_message = {}
for msg in history:
if msg.get("role") == "user":
input_message = msg
break
# Generate IDs for streaming events
task_id = str(uuid4())
context_id = str(uuid4())
artifact_id = str(uuid4())
input_message_id = input_message.get("messageId", str(uuid4()))
# 1. Emit initial task event (kind: "task", status: "submitted")
# Format matches A2ACompletionBridgeTransformation.create_task_event
task_event = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"history": [
{
"contextId": context_id,
"kind": "message",
"messageId": input_message_id,
"parts": input_message.get("parts", [{"kind": "text", "text": ""}]),
"role": "user",
"taskId": task_id,
}
],
"id": task_id,
"kind": "task",
"status": {
"state": "submitted",
},
},
}
yield task_event
# 2. Emit status update (kind: "status-update", status: "working")
# Format matches A2ACompletionBridgeTransformation.create_status_update_event
working_event = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"final": False,
"kind": "status-update",
"status": {
"state": "working",
},
"taskId": task_id,
},
}
yield working_event
# Small delay to simulate processing
await asyncio.sleep(delay_ms / 1000.0)
# 3. Emit artifact update chunks (kind: "artifact-update")
# Format matches A2ACompletionBridgeTransformation.create_artifact_update_event
if full_text:
# Split text into chunks
for i in range(0, len(full_text), chunk_size):
chunk_text = full_text[i:i + chunk_size]
is_last_chunk = (i + chunk_size) >= len(full_text)
artifact_event = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"kind": "artifact-update",
"taskId": task_id,
"artifact": {
"artifactId": artifact_id,
"parts": [
{
"kind": "text",
"text": chunk_text,
}
],
},
},
}
yield artifact_event
# Add delay between chunks (except for last chunk)
if not is_last_chunk:
await asyncio.sleep(delay_ms / 1000.0)
# 4. Emit final status update (kind: "status-update", status: "completed", final: true)
completed_event = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"final": True,
"kind": "status-update",
"status": {
"state": "completed",
},
"taskId": task_id,
},
}
yield completed_event
verbose_logger.info(
f"Pydantic AI: Fake streaming completed for request_id={request_id}"
)

18
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -457,6 +457,24 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raw_response.usage
),
)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
# which contain important provider information like x-request-id
raw_response_hidden_params = getattr(raw_response, "_hidden_params", {})
if raw_response_hidden_params:
if not hasattr(model_response, "_hidden_params") or model_response._hidden_params is None:
model_response._hidden_params = {}
# Merge the raw_response hidden params with model_response hidden params
# Preserve existing keys in model_response but add/override with raw_response params
for key, value in raw_response_hidden_params.items():
if key == "additional_headers" and key in model_response._hidden_params:
# Merge additional_headers to preserve both sets
existing_additional_headers = model_response._hidden_params.get("additional_headers", {})
merged_headers = {**value, **existing_additional_headers}
model_response._hidden_params[key] = merged_headers
else:
model_response._hidden_params[key] = value
return model_response
def get_model_response_iterator(

92
litellm/images/main.py
View file

@ -1,7 +1,11 @@
import asyncio
import contextvars
import importlib
from functools import partial
from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
if TYPE_CHECKING:
from litellm.images.utils import ImageEditRequestUtils
import httpx
@ -50,7 +54,20 @@ from litellm.utils import (
get_optional_params_image_gen,
)
from .utils import ImageEditRequestUtils
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
"""Get ImageEditRequestUtils, loading it lazily if needed."""
global _ImageEditRequestUtils_cache
if _ImageEditRequestUtils_cache is None:
# Access via module to trigger __getattr__ if not cached
module = importlib.import_module(__name__)
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
return _ImageEditRequestUtils_cache
##### Image Generation #######################
@ -702,6 +719,59 @@ def image_edit(
custom_llm_provider=custom_llm_provider,
)
# Check for custom provider
if custom_llm_provider in litellm._custom_providers:
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider
)
model_response = ImageResponse()
if _is_async:
async_custom_client: Optional[AsyncHTTPHandler] = None
if kwargs.get("client") is not None and isinstance(
kwargs.get("client"), AsyncHTTPHandler
):
async_custom_client = kwargs.get("client")
return custom_handler.aimage_edit(
model=model,
image=images,
prompt=prompt,
model_response=model_response,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
optional_params=kwargs,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=async_custom_client,
)
else:
custom_client: Optional[HTTPHandler] = None
if kwargs.get("client") is not None and isinstance(
kwargs.get("client"), HTTPHandler
):
custom_client = kwargs.get("client")
return custom_handler.image_edit(
model=model,
image=images,
prompt=prompt,
model_response=model_response,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
optional_params=kwargs,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=custom_client,
)
# get provider config
image_edit_provider_config: Optional[BaseImageEditConfig] = (
ProviderConfigManager.get_provider_image_edit_config(
@ -716,15 +786,17 @@ def image_edit(
local_vars.update(kwargs)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: ImageEditOptionalRequestParams = (
ImageEditRequestUtils.get_requested_image_edit_optional_param(local_vars)
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(local_vars)
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
ImageEditRequestUtils.get_optional_params_image_edit(
_get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
)
)
@ -845,3 +917,15 @@ async def aimage_edit(
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
def __getattr__(name: str) -> Any:
"""Lazy import handler for images.main module"""
if name == "ImageEditRequestUtils":
# Lazy load ImageEditRequestUtils to avoid heavy import from images.utils at module load time
from .utils import ImageEditRequestUtils as _ImageEditRequestUtils
# Cache it in the module's __dict__ for subsequent accesses
module = importlib.import_module(__name__)
module.__dict__["ImageEditRequestUtils"] = _ImageEditRequestUtils
return _ImageEditRequestUtils
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

40
litellm/images/utils.py
View file

@ -1,5 +1,5 @@
from io import BufferedReader, BytesIO
from typing import Any, Dict, cast, get_type_hints
from typing import Any, Dict, List, Optional, cast, get_type_hints
import litellm
from litellm.litellm_core_utils.token_counter import get_image_type
@ -14,41 +14,53 @@ class ImageEditRequestUtils:
model: str,
image_edit_provider_config: BaseImageEditConfig,
image_edit_optional_params: ImageEditOptionalRequestParams,
drop_params: Optional[bool] = None,
additional_drop_params: Optional[List[str]] = None,
) -> Dict:
"""
Get optional parameters for the image edit API.
Args:
params: Dictionary of all parameters
model: The model name
image_edit_provider_config: The provider configuration for image edit API
image_edit_optional_params: The optional parameters for the image edit API
drop_params: If True, silently drop unsupported parameters instead of raising
additional_drop_params: List of additional parameter names to drop
Returns:
A dictionary of supported parameters for the image edit API
"""
# Remove None values and internal parameters
# Get supported parameters for the model
supported_params = image_edit_provider_config.get_supported_openai_params(model)
# Check for unsupported parameters
should_drop = litellm.drop_params is True or drop_params is True
filtered_optional_params = dict(image_edit_optional_params)
if additional_drop_params:
for param in additional_drop_params:
filtered_optional_params.pop(param, None)
unsupported_params = [
param
for param in image_edit_optional_params
for param in filtered_optional_params
if param not in supported_params
]
if unsupported_params:
raise litellm.UnsupportedParamsError(
model=model,
message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}",
)
if should_drop:
for param in unsupported_params:
filtered_optional_params.pop(param, None)
else:
raise litellm.UnsupportedParamsError(
model=model,
message=f"The following parameters are not supported for model {model}: {', '.join(unsupported_params)}",
)
# Map parameters to provider-specific format
mapped_params = image_edit_provider_config.map_openai_params(
image_edit_optional_params=image_edit_optional_params,
image_edit_optional_params=cast(
ImageEditOptionalRequestParams, filtered_optional_params
),
model=model,
drop_params=litellm.drop_params,
drop_params=should_drop,
)
return mapped_params

4
litellm/integrations/custom_logger.py
View file

@ -16,7 +16,6 @@ from typing import (
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.types.integrations.argilla import ArgillaItem
from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
@ -33,6 +32,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from litellm.caching.caching import DualCache
from opentelemetry.trace import Span as _Span
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -334,7 +334,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
cache: "DualCache",
data: dict,
call_type: CallTypesLiteral,
) -> Optional[

2
litellm/integrations/gcs_bucket/Readme.md
View file

@ -8,5 +8,5 @@ This folder contains the GCS Bucket Logging integration for LiteLLM Gateway.
- `gcs_bucket_base.py`: This file contains the GCSBucketBase class which handles Authentication for GCS Buckets
## Further Reading
- [Doc setting up GCS Bucket Logging on LiteLLM Proxy (Gateway)](https://docs.litellm.ai/docs/proxy/bucket)
- [Doc setting up GCS Bucket Logging on LiteLLM Proxy (Gateway)](https://docs.litellm.ai/docs/observability/gcs_bucket_integration)
- [Doc on Key / Team Based logging with GCS](https://docs.litellm.ai/docs/proxy/team_logging)

Some files were not shown because too many files have changed in this diff Show more