Merge branch 'main' into fix/ssl-transport-error-on-async-exit

.circleci/requirements.txt

@@ -17,4 +17,5 @@ mcp==1.25.0    # for MCP server
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
 responses==0.25.7  # for proxy client tests
-pytest-retry==1.6.3  # for automatic test retries
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.github/codeql/codeql-config.yml

@@ -1,12 +1,19 @@
 name: "LiteLLM CodeQL config"
 
-# Exclude queries that produce result sets > 2 GiB on this codebase,
-# causing 49+ minute runs that fail and block CI resources.
+# Use security-extended suite instead of security-and-quality to avoid
+# result sets > 2 GiB on this codebase that cause fatal OOM failures.
+queries:
+  - uses: security-extended
+
+# These two queries are security queries included in security-extended that
+# individually produce result sets > 2 GiB on this codebase, causing fatal
+# OOM failures. Exclude them as a safety net until CI confirms they no longer
+# OOM; drop these exclusions in a follow-up once verified.
 query-filters:
   - exclude:
-      id: py/clear-text-logging-sensitive-data  # CWE-312/CleartextLogging.ql — result set > 2 GiB
+      id: py/clear-text-logging-sensitive-data  # CWE-312 — > 2 GiB result set
   - exclude:
-      id: py/polynomial-redos                   # CWE-730/PolynomialReDoS.ql — result set > 2 GiB
+      id: py/polynomial-redos                   # CWE-730 — > 2 GiB result set
 
 paths-ignore:
   - tests

.github/pull_request_template.md

@@ -11,6 +11,10 @@
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
 
+## Delays in PR merge?
+
+If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
+
 ## CI (LiteLLM team)
 
 > **CI status guideline:**

.github/workflows/codeql.yml

@@ -34,8 +34,6 @@ jobs:
             build-mode: none
           - language: python
             build-mode: none
-          - language: ruby
-            build-mode: none
 
     steps:
       - name: Checkout repository

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # Allow CodSpeed to trigger backtest performance analysis
+  # in order to generate initial data
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -e "."
+          pip install pytest pytest-codspeed==4.3.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: simulation
+          run: pytest tests/benchmarks/ --codspeed

.github/workflows/create_daily_staging_branch.yml

@@ -41,3 +41,39 @@ jobs:
             git push origin $BRANCH_NAME
             echo "Successfully created and pushed branch: $BRANCH_NAME"
           fi
+
+  create-internal-dev-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi

.github/workflows/test-linting.yml

@@ -33,10 +33,10 @@ jobs:
         poetry lock
         poetry install --with dev
 
-    - name: Run Black formatting
+    - name: Check Black formatting
       run: |
         cd litellm
-        poetry run black .
+        poetry run black --check --exclude '/enterprise/' .
         cd ..
 
     - name: Debug - Check file state

CLAUDE.md

@@ -91,6 +91,10 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Async/await patterns throughout
 - Type hints required for all public APIs
 - **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
@@ -98,10 +102,28 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
 - **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
 
 ### UI / Backend Consistency
 - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
+
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
@@ -110,6 +132,13 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 ### Proxy database access
 - **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
 - Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
 
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
@@ -127,4 +156,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 **Fix options:**
 1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
 2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
-3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

Dockerfile

@@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.

README.md

@@ -28,6 +28,9 @@
     <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
+    <a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
+        <img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
+    </a>
 </h4>
 
 <img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />

ci_cd/security_scans.sh

@@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..."
 install_trivy() {
     echo "Installing Trivy and required tools..."
     sudo apt-get update
-    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils
     wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
     echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
     sudo apt-get update

deploy/charts/litellm-helm/templates/_helpers.tpl

@@ -61,6 +61,20 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create the service account name used by migration jobs.
+When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
+If this chart is creating the ServiceAccount, it is not yet available for the hook job,
+so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
+*/}}
+{{- define "litellm.migrationServiceAccountName" -}}
+{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
+{{- default "default" .Values.migrationJob.serviceAccountName }}
+{{- else }}
+{{- include "litellm.serviceAccountName" . }}
+{{- end }}
+{{- end }}
+
 {{/*
 Get redis service name
 */}}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -13,9 +13,16 @@ spec:
   {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.deploymentMinReadySeconds }}
+  minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
+  {{- end }}
   template:
     metadata:
       annotations:

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -34,7 +34,7 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -306,3 +306,16 @@ tests:
       - equal:
           path: spec.template.spec.containers[0].resources
           value: {}
+  - it: should be able to set minReadySeconds
+    template: deployment.yaml
+    set:
+      deploymentMinReadySeconds: 5
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 5
+  - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
+    template: deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.minReadySeconds

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -124,4 +124,67 @@ tests:
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
-            name: DATABASE_URL
+            name: DATABASE_URL
+
+  - it: should use default service account for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+
+  - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        serviceAccountName: migration-sa
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migration-sa
+
+  - it: should use chart service account when helm hooks are disabled
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: false
+      serviceAccount:
+        create: true
+        name: my-custom-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: my-custom-sa
+
+  - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: false
+        name: pre-existing-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: pre-existing-sa

deploy/charts/litellm-helm/values.yaml

@@ -31,10 +31,20 @@ serviceAccount:
 # annotations for litellm deployment
 deploymentAnnotations: {}
 deploymentLabels: {}
+deploymentMinReadySeconds: 0
+
 # annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
+# -- Deployment strategy configuration
+# Example:
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 0
+#     maxSurge: 1
+strategy: {}
+
 terminationGracePeriodSeconds: 90
 topologySpreadConstraints:
   []
@@ -299,6 +309,10 @@ migrationJob:
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
+  # Optional service account for the migration job.
+  # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
+  # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
+  serviceAccountName: ""
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}

docker/Dockerfile.custom_ui

@@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
         libgnutls30 \
         libc6 && \
     apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -112,7 +112,7 @@ RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_au
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Build Admin UI (runtime stage)
 # Convert Windows line endings to Unix and make executable

docker/Dockerfile.dev

@@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Fix JWT dependency conflicts early
 RUN pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+    pip install PyJWT==2.12.0 --no-cache-dir
 
 # Copy only necessary files for build
 COPY pyproject.toml README.md schema.prisma poetry.lock ./
@@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
+    && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

docker/Dockerfile.non_root

@@ -32,7 +32,7 @@ RUN for i in 1 2 3; do \
 # Cache Python dependencies
 COPY requirements.txt .
 RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
-  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
+  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
 
 # Copy source after dependency layers
 COPY . .
@@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
   && apk upgrade --no-cache nodejs \
-  && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
+  && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -198,7 +198,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
     pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
+    pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
     rm -rf /wheels && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup $PRISMA_PATH && \

docs/my-website/blog/gemini_embedding_2_multimodal/index.md

@@ -0,0 +1,169 @@
+---
+slug: gemini_embedding_2_multimodal
+title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
+date: 2025-03-11T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
+tags: [gemini, embeddings, multimodal, vertex ai]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini Embedding 2 Preview: Multimodal Embeddings
+
+LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
+
+## Supported Input Types
+
+| Modality | Supported Formats | 
+|----------|-------------------|
+| **Text** | Plain text |
+| **Image** | PNG, JPEG | 
+| **Audio** | MP3, WAV | 
+| **Video** | MP4, MOV | 
+| **Documents** | PDF | 
+
+## Input Formats
+
+LiteLLM accepts three input formats for multimodal content:
+
+1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
+2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
+3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
+
+## Quick Start
+
+<Tabs>
+<TabItem value="gemini" label="Gemini API">
+
+```python
+from litellm import embedding
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Text + Image (base64)
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious and the waiter...",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="vertex" label="Vertex AI">
+
+```python
+import litellm
+from litellm import embedding
+
+litellm.vertex_project = "your-project-id"
+litellm.vertex_location = "us-central1"
+
+# Text + Image (GCS URL)
+response = embedding(
+    model="vertex_ai/gemini-embedding-2-preview",
+    input=[
+        "Describe this image",
+        "gs://my-bucket/images/photo.png"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Config (config.yaml)**
+
+```yaml
+model_list:
+  - model_name: gemini-embedding-2-preview
+    litellm_params:
+      model: gemini/gemini-embedding-2-preview
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-gemini-embedding-2-preview
+    litellm_params:
+      model: vertex_ai/gemini-embedding-2-preview
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+
+general_settings:
+  master_key: sk-1234
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config config.yaml
+```
+
+**3. Call embeddings**
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [
+      "The food was delicious and the waiter...",
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Input Format Examples
+
+| Format | Example | Provider |
+|--------|---------|----------|
+| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
+| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
+| **File reference** | `files/abc123` | Gemini API only |
+
+### Supported MIME Types for Data URIs
+
+- **Images:** `image/png`, `image/jpeg`
+- **Audio:** `audio/mpeg`, `audio/wav`
+- **Video:** `video/mp4`, `video/quicktime`
+- **Documents:** `application/pdf`
+
+### GCS URL MIME Inference
+
+For Vertex AI, MIME types are inferred from file extensions:
+
+- `.png` → `image/png`
+- `.jpg` / `.jpeg` → `image/jpeg`
+- `.mp3` → `audio/mpeg`
+- `.wav` → `audio/wav`
+- `.mp4` → `video/mp4`
+- `.mov` → `video/quicktime`
+- `.pdf` → `application/pdf`
+
+## Optional Parameters
+
+| Parameter | Description | Maps to |
+|-----------|-------------|---------|
+| `dimensions` | Output embedding size | `outputDimensionality` |
+
+```python
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=["text to embed"],
+    dimensions=768,  # Optional: control output vector size
+)
+```

docs/my-website/blog/realtime_webrtc_http_endpoints/index.md

@@ -0,0 +1,119 @@
+---
+slug: realtime_webrtc_http_endpoints
+title: "Realtime WebRTC HTTP Endpoints"
+date: 2026-03-12T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
+tags: [realtime, webrtc, proxy, openai]
+hide_table_of_contents: false
+---
+
+import WebRTCTester from '@site/src/components/WebRTCTester';
+
+Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
+
+## How it works
+
+![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
+
+**Flow of generating ephemeral token**
+
+![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png)
+
+
+## Proxy Setup
+
+```yaml
+model_list:
+  - model_name: gpt-4o-realtime
+    litellm_params:
+      model: openai/gpt-4o-realtime-preview-2024-12-17
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+```
+
+**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Try it live
+
+<WebRTCTester />
+
+## Client Usage
+
+**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`.
+
+**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <encrypted_token>` and `Content-Type: application/sdp`.
+
+**3. Events** - Use the data channel for `session.update` and other events.
+
+<details>
+<summary>Full code example</summary>
+
+```javascript
+// 1. Token
+const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
+  method: "POST",
+  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
+  body: JSON.stringify({ model: "gpt-4o-realtime" }),
+});
+const { client_secret } = await r.json();
+const token = client_secret.value;
+
+// 2. WebRTC
+const pc = new RTCPeerConnection();
+const audio = document.createElement("audio");
+audio.autoplay = true;
+pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
+const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
+pc.addTrack(ms.getTracks()[0]);
+const dc = pc.createDataChannel("oai-events");
+const offer = await pc.createOffer();
+await pc.setLocalDescription(offer);
+
+const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
+  method: "POST",
+  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
+  body: offer.sdp,
+});
+await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
+
+// 3. Events
+dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
+```
+
+</details>
+
+## FAQ
+
+**Q: What do I do if I get a 401 Token expired error?**  
+A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer.
+
+**Q: Which key should I use for `/v1/realtime/calls`?**  
+A: Use the **encrypted token** from `client_secrets`, not your raw API key.
+
+**Q: Should I pass the `model` parameter when making the call?**  
+A: No, the encrypted token already encodes all routing information including model.
+
+**Q: How do I resolve Azure `api-version` errors?**  
+A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values.
+
+**Q: What if I get no audio?**  
+A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.
+

docs/my-website/blog/video_characters_litellm/index.md

@@ -0,0 +1,128 @@
+---
+slug: video_characters_api
+title: "New Video Characters, Edit and Extension API support"
+date: 2026-03-16T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
+tags: [videos, characters, proxy, routing]
+hide_table_of_contents: false
+---
+
+LiteLLM now supoports videos character, edit and extension apis.
+
+## What's New
+
+Four new endpoints for video character operations:
+- **Create character** - Upload a video to create a reusable asset
+- **Get character** - Retrieve character metadata
+- **Edit video** - Modify generated videos
+- **Extend video** - Continue clips with character consistency
+
+**Available from:** LiteLLM v1.83.0+
+
+## Quick Example
+
+```python
+import litellm
+
+# Create character from video
+character = litellm.avideo_create_character(
+    name="Luna",
+    video=open("luna.mp4", "rb"),
+    custom_llm_provider="openai",
+    model="sora-2"
+)
+print(f"Character: {character.id}")
+
+# Use in generation
+video = litellm.avideo(
+    model="sora-2",
+    prompt="Luna dances through a magical forest.",
+    characters=[{"id": character.id}],
+    seconds="8"
+)
+
+# Get character info
+fetched = litellm.avideo_get_character(
+    character_id=character.id,
+    custom_llm_provider="openai"
+)
+
+# Edit with character preserved
+edited = litellm.avideo_edit(
+    video_id=video.id,
+    prompt="Add warm golden lighting"
+)
+
+# Extend sequence
+extended = litellm.avideo_extension(
+    video_id=video.id,
+    prompt="Luna waves goodbye",
+    seconds="5"
+)
+```
+
+## Via Proxy
+
+```bash
+# Create character
+curl -X POST "http://localhost:4000/v1/videos/characters" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -F "video=@luna.mp4" \
+  -F "name=Luna"
+
+# Get character
+curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
+  -H "Authorization: Bearer sk-litellm-key"
+
+# Edit video
+curl -X POST "http://localhost:4000/v1/videos/edits" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Add warm golden lighting and enhance colors"
+  }'
+
+# Extend video
+curl -X POST "http://localhost:4000/v1/videos/extensions" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Luna waves goodbye and walks into the sunset",
+    "seconds": "5"
+  }'
+```
+
+## Managed Character IDs
+
+LiteLLM automatically encodes provider and model metadata into character IDs:
+
+**What happens:**
+```
+Upload character "Luna" with model "sora-2" on OpenAI
+  ↓
+LiteLLM creates: char_abc123def456 (contains provider + model_id)
+  ↓
+When you reference it later, LiteLLM decodes automatically
+  ↓
+Router knows exactly which deployment to use
+```
+
+**Behind the scenes:**
+- Character ID format: `character_<base64_encoded_metadata>`
+- Metadata includes: provider, model_id, original_character_id
+- Transparent to you - just use the ID, LiteLLM handles routing

docs/my-website/docs/anthropic_count_tokens.md

@@ -138,6 +138,7 @@ The `/v1/messages/count_tokens` endpoint automatically routes to the appropriate
 | Provider | Token Counting Method |
 |----------|----------------------|
 | Anthropic | [Anthropic Token Counting API](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) |
+| OpenAI | [OpenAI Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) — see [Token Counting](./count_tokens.md) |
 | Vertex AI (Claude) | Vertex AI Partner Models Token Counter |
 | Bedrock (Claude) | AWS Bedrock CountTokens API |
 | Gemini | Google AI Studio countTokens API |

docs/my-website/docs/apply_guardrail.md

@@ -11,6 +11,7 @@ This endpoint supports various guardrail types including:
 - **Presidio** - PII detection and masking
 - **Bedrock** - AWS Bedrock guardrails for content moderation
 - **Lakera** - AI safety guardrails
+- **PANW Prisma AIRS** - Threat detection, DLP, and policy enforcement
 - **Custom guardrails** - User-defined guardrails
 
 ## Configuration

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
 - [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---

docs/my-website/docs/completion/output.md

@@ -51,6 +51,28 @@ Here's what an example response looks like
 }
 ```
 
+## Native Finish Reason
+
+LiteLLM maps all provider-specific `finish_reason` values to OpenAI-compatible values (`stop`, `length`, `tool_calls`, `function_call`, `content_filter`). When the original provider value differs from the mapped value, it is preserved in `provider_specific_fields["native_finish_reason"]`.
+
+This is useful for agent loops that need to distinguish between different stop conditions (e.g., Gemini's `MALFORMED_FUNCTION_CALL` vs a normal `stop`).
+
+```python
+response = completion(model="gemini/gemini-2.0-flash", messages=messages)
+
+choice = response.choices[0]
+print(choice.finish_reason)  # "stop" (OpenAI-compatible)
+
+# Access the original provider value when it differs:
+if hasattr(choice, "provider_specific_fields") and choice.provider_specific_fields:
+    native = choice.provider_specific_fields.get("native_finish_reason")
+    if native == "MALFORMED_FUNCTION_CALL":
+        # Handle malformed function call differently from a normal stop
+        pass
+```
+
+When the provider already returns an OpenAI-compatible value (e.g., `stop`), `native_finish_reason` is not set.
+
 ## Additional Attributes
 
 You can also access information like latency. 

docs/my-website/docs/completion/web_fetch.md

@@ -115,6 +115,11 @@ print(response)
 
 Web fetch is available on the following Anthropic API models:
 
+- `claude-opus-4-6` (Claude Opus 4.6)
+- `claude-sonnet-4-6` (Claude Sonnet 4.6)
+- `claude-opus-4-5` (Claude Opus 4.5)
+- `claude-sonnet-4-5` (Claude Sonnet 4.5)
+- `claude-haiku-4-5` (Claude Haiku 4.5)
 - `claude-opus-4-1-20250805` (Claude Opus 4.1)
 - `claude-opus-4-20250514` (Claude Opus 4)
 - `claude-sonnet-4-20250514` (Claude Sonnet 4)

docs/my-website/docs/contributing/adding_openai_compatible_providers.md

@@ -80,6 +80,36 @@ That's it! The provider is now available.
 }
 ```
 
+## Responses API Support
+
+If your provider also supports the OpenAI Responses API (`/v1/responses`), add `supported_endpoints`:
+
+```json
+{
+  "your_provider": {
+    "base_url": "https://api.yourprovider.com/v1",
+    "api_key_env": "YOUR_PROVIDER_API_KEY",
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
+  }
+}
+```
+
+This enables `litellm.responses()` with zero additional code:
+
+```python
+import litellm
+
+response = litellm.responses(
+    model="your_provider/model-name",
+    input="Hello, what can you do?",
+)
+print(response.output)
+```
+
+If `supported_endpoints` is omitted, it defaults to `[]`. Chat completions is always enabled for JSON providers regardless of this field.
+
+The provider inherits all request/response handling from OpenAI's Responses API — streaming, tools, and all standard parameters work out of the box.
+
 ## Usage
 
 ```python
@@ -89,11 +119,17 @@ import os
 # Set your API key
 os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here"
 
-# Use the provider
+# Chat completions
 response = litellm.completion(
     model="your_provider/model-name",
     messages=[{"role": "user", "content": "Hello"}],
 )
+
+# Responses API (if supported_endpoints includes "/v1/responses")
+response = litellm.responses(
+    model="your_provider/model-name",
+    input="Hello",
+)
 ```
 
 ## When to Use Python Instead
@@ -105,7 +141,9 @@ Use a Python config class if you need:
 - Provider-specific streaming logic
 - Advanced tool calling modifications
 
-For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
+For chat completions, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
+
+For responses API with small overrides, inherit from `OpenAIResponsesAPIConfig` and override only what's needed. See `litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines vs 400+).
 
 ## Testing
 

docs/my-website/docs/count_tokens.md

@@ -0,0 +1,189 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Token Counting
+
+## Overview
+
+LiteLLM provides exact token counting by calling provider-specific token counting APIs. This gives you accurate token counts before sending requests, helping with cost estimation and context window management.
+
+| Feature | Details |
+|---------|---------|
+| SDK Method | `litellm.acount_tokens()` |
+| Proxy Endpoints | `/v1/messages/count_tokens` (Anthropic format), `/v1/responses/input_tokens` (OpenAI format) |
+| Fallback | Local tiktoken-based counting for unsupported providers |
+
+## Supported Providers
+
+| Provider | Token Counting API | Format |
+|----------|-------------------|--------|
+| OpenAI | [Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) | OpenAI Responses |
+| Anthropic | [Messages `/count_tokens`](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) | Anthropic Messages |
+| Vertex AI (Claude) | Vertex AI Partner Models Token Counter | Anthropic Messages |
+| Bedrock (Claude) | AWS Bedrock CountTokens API | Anthropic Messages |
+| Gemini | Google AI Studio countTokens API | Anthropic Messages |
+| Vertex AI (Gemini) | Vertex AI countTokens API | Anthropic Messages |
+| Other providers | Local tiktoken fallback | N/A |
+
+## SDK Usage
+
+### Basic Usage
+
+```python
+import asyncio
+import litellm
+
+async def main():
+    # OpenAI
+    result = await litellm.acount_tokens(
+        model="openai/gpt-4o",
+        messages=[{"role": "user", "content": "Hello, how are you?"}],
+    )
+    print(f"Token count: {result.total_tokens}")
+    print(f"Tokenizer: {result.tokenizer_type}")  # "openai_api"
+
+    # Anthropic
+    result = await litellm.acount_tokens(
+        model="anthropic/claude-3-5-sonnet-20241022",
+        messages=[{"role": "user", "content": "Hello, how are you?"}],
+    )
+    print(f"Token count: {result.total_tokens}")
+    print(f"Tokenizer: {result.tokenizer_type}")  # "anthropic_api"
+
+asyncio.run(main())
+```
+
+### With Tools and System Message
+
+```python
+import asyncio
+import litellm
+
+async def main():
+    result = await litellm.acount_tokens(
+        model="openai/gpt-4o",
+        messages=[{"role": "user", "content": "What's the weather in Paris?"}],
+        tools=[{
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get weather for a city",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"city": {"type": "string"}},
+                },
+            },
+        }],
+        system="You are a helpful weather assistant.",
+    )
+    print(f"Token count (with tools): {result.total_tokens}")
+
+asyncio.run(main())
+```
+
+### Response Format
+
+`litellm.acount_tokens()` returns a `TokenCountResponse`:
+
+```python
+TokenCountResponse(
+    total_tokens=15,           # Token count
+    request_model="openai/gpt-4o",  # Model requested
+    model_used="gpt-4o",      # Model used for counting
+    tokenizer_type="openai_api",    # "openai_api", "anthropic_api", "local_tokenizer"
+    original_response={"input_tokens": 15},  # Raw API response
+    error=False,               # True if counting failed
+    error_message=None,        # Error details if failed
+)
+```
+
+### Fallback Behavior
+
+If a provider doesn't support a token counting API, or if the API key is missing, `acount_tokens()` automatically falls back to local tiktoken-based counting:
+
+```python
+# Unsupported provider → automatic fallback
+result = await litellm.acount_tokens(
+    model="together_ai/meta-llama/Llama-3-8b-chat-hf",
+    messages=[{"role": "user", "content": "Hello"}],
+)
+print(result.tokenizer_type)  # "local_tokenizer"
+```
+
+## Proxy Usage
+
+### OpenAI Format — `/v1/responses/input_tokens`
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl -X POST "http://localhost:4000/v1/responses/input_tokens" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4o",
+    "input": "Hello, how are you?"
+  }'
+```
+
+</TabItem>
+<TabItem value="python" label="Python (httpx)">
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:4000/v1/responses/input_tokens",
+    headers={
+        "Content-Type": "application/json",
+        "Authorization": "Bearer sk-1234"
+    },
+    json={
+        "model": "gpt-4o",
+        "input": "Hello, how are you?"
+    }
+)
+
+print(response.json())
+# {"input_tokens": 7}
+```
+
+</TabItem>
+</Tabs>
+
+**Response:**
+```json
+{"input_tokens": 7}
+```
+
+### Anthropic Format — `/v1/messages/count_tokens`
+
+See [Anthropic Token Counting](./anthropic_count_tokens.md) for full documentation.
+
+```bash
+curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "claude-3-5-sonnet-20241022",
+    "messages": [
+      {"role": "user", "content": "Hello, how are you?"}
+    ]
+  }'
+```
+
+## Proxy Configuration
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+```

docs/my-website/docs/embedding/supported_embedding.md

@@ -514,6 +514,57 @@ All models listed [here](https://ai.google.dev/gemini-api/docs/models/gemini) ar
 | Model Name         | Function Call                                         |
 | :---               | :---                                                  |
 | text-embedding-004 | `embedding(model="gemini/text-embedding-004", input)` |
+| gemini-embedding-2-preview | `embedding(model="gemini/gemini-embedding-2-preview", input)` | [Multimodal docs](#gemini-embedding-2-preview-multimodal) |
+
+### Gemini Embedding 2 Preview (Multimodal)
+
+`gemini-embedding-2-preview` supports **multimodal embeddings**—text, images, audio, video, and PDF in a single request. See [blog post](/blog/gemini_embedding_2_multimodal) for details.
+
+**Input formats:**
+- **Data URIs:** `data:image/png;base64,<encoded_data>`
+- **Gemini file references:** `files/abc123` (pre-uploaded via Gemini Files API)
+
+**Supported MIME types:** `image/png`, `image/jpeg`, `audio/mpeg`, `audio/wav`, `video/mp4`, `video/quicktime`, `application/pdf`
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+os.environ["GEMINI_API_KEY"] = ""
+
+# Text + Image (base64)
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious and the waiter...",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [
+      "The food was delicious and the waiter...",
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
 
 
 ## Vertex AI Embedding Models

docs/my-website/docs/files_endpoints.md

@@ -326,4 +326,10 @@ print("file content=", content.text)
 
 ### [Bedrock](./providers/bedrock_batches#4-retrieve-batch-results)
 
+### [Anthropic](./providers/anthropic#files-api)
+
+:::note
+Anthropic Files API has a different purpose than OpenAI's. It's **not** for Batches or Fine-tuning—it's for uploading files once and referencing them by `file_id` in multiple messages, avoiding re-uploads. File API operations are free — file content used in Messages requests is priced as input tokens.
+:::
+
 ## [Swagger API Reference](https://litellm-api.up.railway.app/#/files)

docs/my-website/docs/image_edits.md

@@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
-| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. |
+| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)**, **Black Forest Labs** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. Black Forest Labs supports FLUX Kontext models. |
 
  #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 
@@ -199,6 +199,63 @@ for idx, image_obj in enumerate(response.data):
 
 </TabItem>
 
+<TabItem value="bfl" label="Black Forest Labs">
+
+#### Basic Image Edit
+```python showLineNumbers title="Black Forest Labs Image Edit"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+response = litellm.image_edit(
+    model="black_forest_labs/flux-kontext-pro",
+    image=open("original_image.png", "rb"),
+    prompt="Add a green leaf to the scene",
+)
+
+print(response.data[0].url)
+```
+
+#### Inpainting with Mask
+```python showLineNumbers title="Black Forest Labs Inpainting"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+# Use flux-pro-1.0-fill for inpainting
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-fill",
+    image=open("original_image.png", "rb"),
+    mask=open("mask_image.png", "rb"),
+    prompt="Replace with a garden",
+)
+
+print(response.data[0].url)
+```
+
+#### Outpainting (Expand)
+```python showLineNumbers title="Black Forest Labs Outpainting"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+# Use flux-pro-1.0-expand to extend image borders
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-expand",
+    image=open("original_image.png", "rb"),
+    prompt="Continue the scene with mountains",
+    top=256,
+    bottom=256,
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
 <TabItem value="vertex_ai" label="Vertex AI">
 
 #### Basic Image Edit (Gemini)
@@ -392,6 +449,35 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
 
 </TabItem>
 
+<TabItem value="bfl" label="Black Forest Labs">
+
+1. Add Black Forest Labs image edit models to your `config.yaml`:
+```yaml showLineNumbers title="Black Forest Labs Proxy Configuration"
+model_list:
+  - model_name: bfl-kontext-pro
+    litellm_params:
+      model: black_forest_labs/flux-kontext-pro
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+```
+
+2. Start the LiteLLM proxy server:
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+```
+
+3. Make an image edit request:
+```bash showLineNumbers title="Black Forest Labs Proxy Image Edit"
+curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -F "model=bfl-kontext-pro" \
+  -F "image=@original_image.png" \
+  -F "prompt=Add a sunset in the background"
+```
+
+</TabItem>
+
 <TabItem value="vertex_ai" label="Vertex AI">
 
 1. Add Vertex AI image edit models to your `config.yaml`:

docs/my-website/docs/image_generation.md

@@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to input prompts (non-streaming only) |
-| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
+| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Black Forest Labs, Recraft, OpenRouter, Xinference, Nscale | |
 
 ## Quick Start
 

docs/my-website/docs/mcp.md

@@ -133,6 +133,21 @@ LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.
 
 <br/>
 
+### AWS SigV4 Authentication
+
+For MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html), select **AWS SigV4** as the authentication type. LiteLLM will sign every outgoing MCP request with your AWS credentials using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).
+
+<Image
+  img={require('../img/mcp_aws_sigv4_ui.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+Fill in your AWS region, service name (defaults to `bedrock-agentcore`), and optionally your AWS access key and secret. If credentials are omitted, LiteLLM falls back to the boto3 credential chain (IAM roles, environment variables, etc.).
+
+[**See full SigV4 setup guide**](./mcp_aws_sigv4.md)
+
+<br/>
+
 ### Static Headers
 
 Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
@@ -217,6 +232,7 @@ mcp_servers:
   | `bearer_token` | `Authorization: Bearer <auth_value>` |
   | `basic` | `Authorization: Basic <auth_value>` |
   | `authorization` | `Authorization: <auth_value>` |
+  | `aws_sigv4` | Per-request AWS SigV4 signature ([details](./mcp_aws_sigv4.md)) |
 
 - **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
 - **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server.
@@ -257,6 +273,16 @@ mcp_servers:
     auth_type: "authorization"
     auth_value: "Token example123"  # headers={"Authorization": "Token example123"}
 
+  # AWS SigV4 for Bedrock AgentCore MCP servers
+  agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_region_name: us-east-1
+    aws_service_name: bedrock-agentcore
+
   # Example with extra headers forwarding
   github_mcp:
     url: "https://api.githubcopilot.com/mcp"

docs/my-website/docs/mcp_aws_sigv4.md

@@ -0,0 +1,181 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP - AWS SigV4 Auth
+
+Use AWS SigV4 authentication to connect LiteLLM to MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html).
+
+## Why SigV4?
+
+AWS services authenticate requests using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) — a per-request signing protocol that includes the request body in the cryptographic signature. This is fundamentally different from static-header auth types (`api_key`, `bearer_token`, etc.) which send the same header on every request.
+
+LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP request is signed with your AWS credentials before it's sent.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="ui" label="LiteLLM UI">
+
+1. Navigate to **MCP Servers** and click **Add New MCP Server**
+2. Set the transport to **Streamable HTTP**
+3. Select **AWS SigV4** as the authentication type
+4. Fill in your AWS credentials:
+
+<Image
+  img={require('../img/mcp_aws_sigv4_ui.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| **AWS Region** | Yes | AWS region for SigV4 signing (e.g., `us-east-1`) |
+| **AWS Service Name** | No | Defaults to `bedrock-agentcore` |
+| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
+| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
+| **AWS Session Token** | No | Only needed for temporary STS credentials |
+
+Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
+
+**Editing credentials:** When editing an existing SigV4 server, leave credential fields blank to keep the current values. Only fields you fill in will be updated.
+
+</TabItem>
+<TabItem value="config" label="config.yaml">
+
+### 1. Set AWS credentials
+
+```bash
+export AWS_ACCESS_KEY_ID="AKIA..."
+export AWS_SECRET_ACCESS_KEY="..."
+export AWS_REGION_NAME="us-east-1"
+```
+
+### 2. Add your AgentCore MCP server to config.yaml
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+mcp_servers:
+  my_agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_region_name: "us-east-1"
+    aws_service_name: "bedrock-agentcore"
+```
+
+:::info URL encoding
+
+The AgentCore runtime ARN must be URL-encoded in the `url` field. For example:
+
+```
+arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/my-mcp-server
+```
+
+becomes:
+
+```
+arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fmy-mcp-server
+```
+
+:::
+
+### 3. Start the proxy
+
+```bash
+litellm --config config.yaml
+```
+
+</TabItem>
+</Tabs>
+
+## Use the MCP tools
+
+Once configured, your AgentCore MCP tools are available through LiteLLM like any other MCP server:
+
+```bash title="List available tools"
+curl http://localhost:4000/mcp-rest/tools/list \
+  -H "Authorization: Bearer sk-1234"
+```
+
+```bash title="Call a tool"
+curl http://localhost:4000/mcp-rest/tools/call \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "name": "my_agentcore_mcp_your_tool_name",
+    "arguments": {"key": "value"}
+  }'
+```
+
+## Config Reference
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `url` | Yes | AgentCore MCP server URL (with URL-encoded ARN) |
+| `transport` | Yes | Must be `"http"` |
+| `auth_type` | Yes | Must be `"aws_sigv4"` |
+| `aws_access_key_id` | No | AWS access key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
+| `aws_secret_access_key` | No | AWS secret key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
+| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
+| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
+| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
+
+## How It Works
+
+LiteLLM uses an `httpx.Auth` subclass (`MCPSigV4Auth`) that hooks into the HTTP request lifecycle:
+
+1. For every outgoing MCP request, the auth handler computes a SHA-256 hash of the request body
+2. It creates a SigV4 signature using your AWS credentials, the request URL, headers, and body hash
+3. The signed `Authorization` and `x-amz-date` headers are added to the request
+4. AWS validates the signature and processes the MCP request
+
+This happens transparently — no manual token management required.
+
+## Using Temporary Credentials (STS)
+
+If you use AWS STS temporary credentials (e.g., from IAM roles or SSO), include the session token:
+
+```yaml title="config.yaml with STS credentials" showLineNumbers
+mcp_servers:
+  my_agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_session_token: os.environ/AWS_SESSION_TOKEN
+    aws_region_name: "us-east-1"
+    aws_service_name: "bedrock-agentcore"
+```
+
+## Troubleshooting
+
+### 403 Forbidden from AWS
+
+- Verify your AWS credentials are valid and not expired
+- Check that `aws_region_name` matches the region in your AgentCore URL
+- Ensure `aws_service_name` is set to `bedrock-agentcore`
+- If using STS credentials, confirm `aws_session_token` is set and not expired
+
+### Health check errors on startup
+
+SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
+
+### "botocore not found" error
+
+Install the `botocore` package:
+
+```bash
+pip install botocore
+```
+
+`botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth.

docs/my-website/docs/mcp_guardrail.md

@@ -86,4 +86,5 @@ MCP guardrails work with all LiteLLM-supported guardrail providers:
 - **Lakera**: Content moderation
 - **Aporia**: Custom guardrails
 - **Noma**: Noma Security
+- **PANW Prisma AIRS**: Prisma AIRS guardrails
 - **Custom**: Your own guardrail implementations

docs/my-website/docs/observability/vantage.md

@@ -0,0 +1,148 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vantage Integration
+
+LiteLLM can export proxy spend data to [Vantage](https://vantage.sh) as [FOCUS 1.2](https://focus.finops.org/) formatted cost reports. This lets you visualize LLM spend alongside your cloud infrastructure costs in the Vantage dashboard.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Destination | Export LiteLLM usage data to Vantage Custom Provider |
+| Data format | FOCUS CSV (automatically transformed from LiteLLM spend data) |
+| Supported operations | Manual export, automatic scheduled export (hourly/daily/interval) |
+| Authentication | Vantage API key + Custom Provider token |
+
+## Prerequisites
+
+You need two credentials from the [Vantage console](https://console.vantage.sh):
+
+1. **API Key** — Go to **Settings → API Access Tokens** → Create a token with **Write** scope. The token looks like `vntg_tkn_...`.
+2. **Custom Provider Token** — Go to **Settings → Integrations** → Create a **Custom Provider** integration → Copy the Provider ID (looks like `accss_crdntl_...`).
+
+## Setup via API
+
+The recommended setup uses the proxy admin endpoints. No config file changes needed.
+
+### 1. Initialize credentials
+
+```bash
+curl -X POST http://localhost:4000/vantage/init \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "api_key": "vntg_tkn_YOUR_VANTAGE_API_KEY",
+    "integration_token": "accss_crdntl_YOUR_PROVIDER_TOKEN"
+  }'
+```
+
+Credentials are encrypted and stored in the proxy database.
+
+### 2. Preview data (dry run)
+
+```bash
+curl -X POST http://localhost:4000/vantage/dry-run \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"limit": 10}'
+```
+
+This returns FOCUS-transformed data without sending anything to Vantage. Use it to verify the pipeline works and inspect the data mapping.
+
+### 3. Export to Vantage
+
+```bash
+curl -X POST http://localhost:4000/vantage/export \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{}'
+```
+
+Optional parameters:
+- `limit` — Max number of records to export
+- `start_time_utc` / `end_time_utc` — Filter by time range (must be provided together)
+
+### 4. Verify in Vantage
+
+Go to **Settings → Integrations → your Custom Provider → Import Costs** tab to see uploaded CSVs. Once the status changes from "Importing and Processing" to "Stable", costs appear in **Cost Reporting → All Resources**.
+
+## Setup via Environment Variables
+
+For automatic scheduled exports, configure via environment variables and proxy config:
+
+### Environment variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `VANTAGE_API_KEY` | Yes | Vantage API access token |
+| `VANTAGE_INTEGRATION_TOKEN` | Yes | Custom Provider token from Vantage dashboard |
+| `VANTAGE_BASE_URL` | No | API URL override (default: `https://api.vantage.sh`) |
+| `VANTAGE_EXPORT_FREQUENCY` | No | `hourly` (default), `daily`, or `interval` |
+| `VANTAGE_EXPORT_INTERVAL_SECONDS` | No | Seconds between exports when frequency is `interval` |
+
+### Proxy config
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-your-key
+
+litellm_settings:
+  callbacks: ["vantage"]
+```
+
+```bash
+export VANTAGE_API_KEY="vntg_tkn_..."
+export VANTAGE_INTEGRATION_TOKEN="accss_crdntl_..."
+litellm --config /path/to/config.yaml
+```
+
+The proxy registers a background job that exports data on the configured schedule.
+
+## API Endpoints
+
+All endpoints require admin authentication.
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| `POST` | `/vantage/init` | Store Vantage credentials (encrypted) |
+| `GET` | `/vantage/settings` | View current config (credentials masked) |
+| `PUT` | `/vantage/settings` | Update credentials or base URL |
+| `POST` | `/vantage/dry-run` | Preview FOCUS data without uploading |
+| `POST` | `/vantage/export` | Upload cost data to Vantage |
+| `DELETE` | `/vantage/delete` | Remove credentials and stop scheduled exports |
+
+## FOCUS Field Mapping
+
+LiteLLM spend data is transformed into the FOCUS 1.2 schema:
+
+| LiteLLM Field | FOCUS Column | Description |
+|---------------|-------------|-------------|
+| `spend` | BilledCost, EffectiveCost | Cost of the usage |
+| `model` | ChargeDescription, ResourceId | Model identifier |
+| `model_group` | ServiceName | Model group / deployment |
+| `custom_llm_provider` | ProviderName, PublisherName | Provider (openai, anthropic, etc.) |
+| `api_key` | BillingAccountId | Hashed API key |
+| `api_key_alias` | BillingAccountName | Human-readable key alias |
+| `team_id` | SubAccountId | Team identifier |
+| `team_alias` | SubAccountName | Team name |
+
+Additional metadata (user_id, model_group, etc.) is included in the `Tags` column as JSON.
+
+## Upload Limits
+
+Vantage enforces per-upload limits. LiteLLM handles these automatically:
+
+- **10,000 rows** per upload — large exports are split into batches
+- **2 MB** per upload — oversized batches are further split by size
+- **Unsupported columns** are stripped before upload
+
+## Related Links
+
+- [Vantage](https://vantage.sh)
+- [Vantage Custom Providers](https://docs.vantage.sh/connecting_custom_providers)
+- [FOCUS Specification](https://focus.finops.org/)
+- [Focus Export (S3/Parquet)](./focus.md)

docs/my-website/docs/provider_registration/add_model_pricing.md

@@ -13,6 +13,7 @@ Here's the full specification with all available fields:
 ```json
 {
     "sample_spec": {
+        "aliases": ["optional list of alternate names for this model, e.g. dated versions like sample_spec-20250101"],
         "code_interpreter_cost_per_session": 0.0,
         "computer_use_input_cost_per_1k_tokens": 0.0,
         "computer_use_output_cost_per_1k_tokens": 0.0,
@@ -121,4 +122,28 @@ Here's the full specification with all available fields:
 }
 ```
 
-That's it! Your PR will be reviewed and merged.
+### Using Aliases
+
+Many providers release the same model under multiple names — for example, a `latest` tag and a dated version like `claude-sonnet-4-5-20250929`. Instead of duplicating the entire entry, you can use the `aliases` field:
+
+```json
+{
+    "claude-sonnet-4-5": {
+        "aliases": ["claude-sonnet-4-5-20250929"],
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    }
+}
+```
+
+At load time, each alias is expanded into a top-level entry sharing the same data as the canonical entry. The example above makes both `claude-sonnet-4-5` and `claude-sonnet-4-5-20250929` resolve with the same pricing and capabilities.
+
+:::info
+This is different from [`model_alias_map`](../completion/model_alias.md), which is a runtime SDK/proxy feature for mapping user-facing model names to LiteLLM model identifiers. The `aliases` field here is for the model cost JSON only — it avoids duplicate entries for models that share identical pricing and capabilities.
+:::

docs/my-website/docs/providers/anthropic.md

@@ -1965,6 +1965,98 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 </TabItem>
 </Tabs>
 
+## Files API
+
+Upload files once and reference them by `file_id` in multiple requests—no need to re-upload content each time.
+
+:::info
+The `file_id` obtained from Anthropic only works with Anthropic Claude models. You cannot use it with other providers (OpenAI, Bedrock, etc.).
+:::
+
+- **Max file size:** 500 MB | **Total storage:** 100 GB per org
+- **Pricing:** File API operations are free. File content used in Messages requests is priced as input tokens.
+
+**Supported models by file type:**
+- **Images:** All Claude 3+ models
+- **PDFs:** All Claude 3.5+ models
+- **Other file types** (for code execution): Claude 3.5 Haiku + all Claude 3.7+ models
+
+### Quick Start
+
+```python
+import litellm
+import os
+
+os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
+
+# 1. Upload a file once
+file = litellm.create_file(
+    file=open("document.pdf", "rb"),
+    purpose="messages",
+    custom_llm_provider="anthropic",
+)
+
+# 2. Use file_id in messages (no re-upload needed)
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[{
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "Summarize this document"},
+            {"type": "file", "file": {"file_id": file.id, "format": "application/pdf"}}
+        ]
+    }]
+)
+```
+
+### File Operations
+
+| Operation | Function |
+|-----------|----------|
+| Upload | `litellm.create_file(file, purpose="messages", custom_llm_provider="anthropic")` |
+| List | `litellm.file_list(custom_llm_provider="anthropic")` |
+| Retrieve | `litellm.file_retrieve(file_id, custom_llm_provider="anthropic")` |
+| Delete | `litellm.file_delete(file_id, custom_llm_provider="anthropic")` |
+| Download | `litellm.file_content(file_id, custom_llm_provider="anthropic")` |
+
+:::note
+Download only works for files created by the [code execution tool](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/code-execution-tool), not uploaded files.
+:::
+
+### Supported Formats
+
+| File Type | Format Value |
+|-----------|-------------|
+| PDF | `application/pdf` |
+| Plain text | `text/plain` |
+| JPEG | `image/jpeg` |
+| PNG | `image/png` |
+| GIF | `image/gif` |
+| WebP | `image/webp` |
+
+### Using Images
+
+```python
+# Upload image
+image = litellm.create_file(
+    file=open("photo.jpg", "rb"),
+    purpose="messages",
+    custom_llm_provider="anthropic",
+)
+
+# Use in message
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[{
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "What's in this image?"},
+            {"type": "file", "file": {"file_id": image.id, "format": "image/jpeg"}}
+        ]
+    }]
+)
+```
+
 ## Usage - passing 'user_id' to Anthropic
 
 LiteLLM translates the OpenAI `user` param to Anthropic's `metadata[user_id]` param.

docs/my-website/docs/providers/aws_sagemaker.md

@@ -526,3 +526,98 @@ print(f"response: {response}")
 ```
 
 
+
+## Nova Models on SageMaker
+
+LiteLLM supports Amazon Nova models (Nova Micro, Nova Lite, Nova 2 Lite) deployed on SageMaker Inference real-time endpoints. These custom/fine-tuned Nova models use an OpenAI-compatible API format.
+
+**Reference:** [AWS Blog - Amazon SageMaker Inference for Custom Amazon Nova Models](https://aws.amazon.com/blogs/aws/announcing-amazon-sagemaker-inference-for-custom-amazon-nova-models/)
+
+### Usage
+
+Use the `sagemaker_nova/` prefix with your SageMaker endpoint name:
+
+```python
+import litellm
+import os
+
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+# Basic chat completion
+response = litellm.completion(
+    model="sagemaker_nova/my-nova-endpoint",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+    temperature=0.7,
+    max_tokens=512,
+)
+print(response.choices[0].message.content)
+```
+
+### Streaming
+
+```python
+response = litellm.completion(
+    model="sagemaker_nova/my-nova-endpoint",
+    messages=[{"role": "user", "content": "Write a short poem"}],
+    stream=True,
+    stream_options={"include_usage": True},
+)
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+### Multimodal (Images)
+
+Nova models on SageMaker support image inputs using base64 data URIs:
+
+```python
+response = litellm.completion(
+    model="sagemaker_nova/my-nova-endpoint",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
+            ]
+        }
+    ],
+)
+```
+
+### Proxy Config
+
+```yaml
+model_list:
+  - model_name: nova-micro
+    litellm_params:
+      model: sagemaker_nova/my-nova-micro-endpoint
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+```
+
+### Supported Parameters
+
+All standard OpenAI parameters are supported, plus these Nova-specific parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `top_k` | integer | Limits token selection to top K most likely tokens |
+| `reasoning_effort` | `"low"` \| `"high"` | Reasoning effort level (Nova 2 Lite custom models only) |
+| `allowed_token_ids` | array[int] | Restrict output to specified token IDs |
+| `truncate_prompt_tokens` | integer | Truncate prompt to N tokens if it exceeds limit |
+
+```python
+response = litellm.completion(
+    model="sagemaker_nova/my-nova-endpoint",
+    messages=[{"role": "user", "content": "Think step by step: what is 2+2?"}],
+    top_k=40,
+    reasoning_effort="low",
+    logprobs=True,
+    top_logprobs=2,
+)
+```

docs/my-website/docs/providers/bedrock_agentcore.md

@@ -13,7 +13,7 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
 
 :::info
 
-This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
+This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers with LiteLLM, see the [MCP AWS SigV4 Auth](https://docs.litellm.ai/docs/mcp_aws_sigv4) guide for setup instructions.
 
 :::
 

docs/my-website/docs/providers/black_forest_labs.md

@@ -0,0 +1,291 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Black Forest Labs Image Generation
+
+Black Forest Labs provides state-of-the-art text-to-image generation using their FLUX models.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Black Forest Labs FLUX models for high-quality text-to-image generation |
+| Provider Route on LiteLLM | `black_forest_labs/` |
+| Provider Doc | [Black Forest Labs API ↗](https://docs.bfl.ai/) |
+| Supported Operations | [`/images/generations`](#image-generation) |
+
+## Setup
+
+### API Key
+
+```python showLineNumbers
+import os
+
+# Set your Black Forest Labs API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+```
+
+Get your API key from [Black Forest Labs](https://blackforestlabs.ai/).
+
+## Supported Models
+
+| Model Name | Description | Price |
+|------------|-------------|-------|
+| `black_forest_labs/flux-pro-1.1` | Fast & reliable standard generation | $0.04/image |
+| `black_forest_labs/flux-pro-1.1-ultra` | Ultra high-resolution (up to 4MP) | $0.06/image |
+| `black_forest_labs/flux-dev` | Development/open-source variant | $0.025/image |
+| `black_forest_labs/flux-pro` | Original pro model | $0.05/image |
+
+## Image Generation
+
+### Usage - LiteLLM Python SDK
+
+<Tabs>
+<TabItem value="basic" label="Basic Usage">
+
+```python showLineNumbers title="Basic Image Generation"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Generate an image
+response = litellm.image_generation(
+    model="black_forest_labs/flux-pro-1.1",
+    prompt="A beautiful sunset over the ocean with sailing boats",
+)
+
+# BFL returns URLs
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="async" label="Async Usage">
+
+```python showLineNumbers title="Async Image Generation"
+import os
+import asyncio
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+async def generate_image():
+    response = await litellm.aimage_generation(
+        model="black_forest_labs/flux-pro-1.1",
+        prompt="A futuristic city skyline at night",
+    )
+    print(response.data[0].url)
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+</TabItem>
+
+<TabItem value="size" label="Custom Size">
+
+```python showLineNumbers title="Image Generation with Custom Size"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Generate with specific dimensions
+response = litellm.image_generation(
+    model="black_forest_labs/flux-pro-1.1",
+    prompt="A majestic mountain landscape",
+    size="1792x1024",  # Maps to width/height
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="ultra" label="Ultra High-Res">
+
+```python showLineNumbers title="Ultra High Resolution with flux-pro-1.1-ultra"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Generate ultra high-resolution image
+response = litellm.image_generation(
+    model="black_forest_labs/flux-pro-1.1-ultra",
+    prompt="Detailed portrait of a fantasy character",
+    size="2048x2048",    # Up to 4MP supported
+    quality="hd",        # Maps to raw=True for natural look
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="advanced" label="Advanced Parameters">
+
+```python showLineNumbers title="Advanced Image Generation with BFL Parameters"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Generate with BFL-specific parameters
+response = litellm.image_generation(
+    model="black_forest_labs/flux-pro-1.1",
+    prompt="A cute orange cat sitting on a windowsill",
+    seed=42,                    # For reproducible results
+    output_format="png",        # png or jpeg
+    safety_tolerance=2,         # 0-6, higher = more permissive
+    prompt_upsampling=True,     # Enhance prompt for better results
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+</Tabs>
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Black Forest Labs Image Generation Configuration"
+model_list:
+  - model_name: flux-pro
+    litellm_params:
+      model: black_forest_labs/flux-pro-1.1
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_generation
+
+  - model_name: flux-ultra
+    litellm_params:
+      model: black_forest_labs/flux-pro-1.1-ultra
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_generation
+
+  - model_name: flux-dev
+    litellm_params:
+      model: black_forest_labs/flux-dev
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_generation
+
+general_settings:
+  master_key: sk-1234
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make image generation requests
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="Black Forest Labs via Proxy - OpenAI SDK"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="sk-1234"
+)
+
+# Generate image with FLUX Pro
+response = client.images.generate(
+    model="flux-pro",
+    prompt="A beautiful garden with colorful flowers",
+    size="1024x1024",
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="Black Forest Labs via Proxy - cURL"
+curl -X POST 'http://localhost:4000/v1/images/generations' \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer sk-1234' \
+  -d '{
+    "model": "flux-pro",
+    "prompt": "A beautiful garden with colorful flowers",
+    "size": "1024x1024"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Parameters
+
+### OpenAI-Compatible Parameters
+
+| Parameter | Type | Description | Mapping |
+|-----------|------|-------------|---------|
+| `prompt` | string | Text description of the image to generate | Direct |
+| `model` | string | The FLUX model to use | Direct |
+| `size` | string | Image dimensions (e.g., `1024x1024`) | Maps to `width` and `height` |
+| `n` | integer | Number of images (ultra model only, up to 4) | Maps to `num_images` |
+| `quality` | string | `hd` for natural look | Maps to `raw=True` for ultra |
+| `response_format` | string | `url` or `b64_json` | Direct |
+
+### Black Forest Labs Specific Parameters
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `width` | integer | Image width (256-1920, multiples of 16) | 1024 |
+| `height` | integer | Image height (256-1920, multiples of 16) | 1024 |
+| `aspect_ratio` | string | Alternative to width/height (e.g., `16:9`, `1:1`) | - |
+| `seed` | integer | Seed for reproducible results | Random |
+| `output_format` | string | Output format: `png` or `jpeg` | `png` |
+| `safety_tolerance` | integer | Safety filter tolerance (0-6, higher = more permissive) | 2 |
+| `prompt_upsampling` | boolean | Enhance prompt for better results | `false` |
+
+### Ultra Model Specific Parameters
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `raw` | boolean | Raw mode for more natural, less synthetic look | `false` |
+| `num_images` | integer | Number of images to generate (1-4) | 1 |
+
+## How It Works
+
+Black Forest Labs uses a polling-based API:
+
+1. **Submit Request**: LiteLLM sends your prompt to BFL
+2. **Get Task ID**: BFL returns a task ID and polling URL
+3. **Poll for Result**: LiteLLM automatically polls until the image is ready
+4. **Return Result**: The generated image URL is returned
+
+This polling is handled automatically by LiteLLM - you just call `image_generation()` and get the result.
+
+## Getting Started
+
+1. Create an account at [Black Forest Labs](https://blackforestlabs.ai/)
+2. Get your API key from the dashboard
+3. Set your `BFL_API_KEY` environment variable
+4. Use `litellm.image_generation()` with any supported model
+
+## Additional Resources
+
+- [Black Forest Labs Documentation](https://docs.bfl.ai/)
+- [Black Forest Labs Image Editing](./black_forest_labs_img_edit.md) - For editing existing images
+- [FLUX Model Information](https://blackforestlabs.ai/)

docs/my-website/docs/providers/black_forest_labs_img_edit.md

@@ -0,0 +1,301 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Black Forest Labs Image Editing
+
+Black Forest Labs provides powerful image editing capabilities using their FLUX models to modify existing images based on text descriptions.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Black Forest Labs Image Editing uses FLUX Kontext and other models to modify, inpaint, and expand images based on text prompts. |
+| Provider Route on LiteLLM | `black_forest_labs/` |
+| Provider Doc | [Black Forest Labs API ↗](https://docs.bfl.ai/) |
+| Supported Operations | [`/images/edits`](#image-editing) |
+
+## Setup
+
+### API Key
+
+```python showLineNumbers
+import os
+
+# Set your Black Forest Labs API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+```
+
+Get your API key from [Black Forest Labs](https://blackforestlabs.ai/).
+
+## Supported Models
+
+| Model Name | Description | Use Case |
+|------------|-------------|----------|
+| `black_forest_labs/flux-kontext-pro` | FLUX Kontext Pro - General image editing with prompts | General editing, style transfer |
+| `black_forest_labs/flux-kontext-max` | FLUX Kontext Max - Premium quality editing | High-quality edits |
+| `black_forest_labs/flux-pro-1.0-fill` | FLUX Pro Fill - Inpainting with mask | Remove/replace objects |
+| `black_forest_labs/flux-pro-1.0-expand` | FLUX Pro Expand - Outpainting | Expand image borders |
+
+## Image Editing
+
+### Usage - LiteLLM Python SDK
+
+<Tabs>
+<TabItem value="basic-edit" label="Basic Usage">
+
+```python showLineNumbers title="Basic Image Editing"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Edit an image with a prompt
+response = litellm.image_edit(
+    model="black_forest_labs/flux-kontext-pro",
+    image=open("path/to/your/image.png", "rb"),
+    prompt="Add a green leaf to the scene",
+)
+
+# BFL returns URLs
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="async-edit" label="Async Usage">
+
+```python showLineNumbers title="Async Image Editing"
+import os
+import asyncio
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+async def edit_image():
+    response = await litellm.aimage_edit(
+        model="black_forest_labs/flux-kontext-pro",
+        image=open("path/to/your/image.png", "rb"),
+        prompt="Make this image look like a watercolor painting",
+    )
+    print(response.data[0].url)
+
+# Run the async function
+asyncio.run(edit_image())
+```
+
+</TabItem>
+
+<TabItem value="inpainting" label="Inpainting (Fill)">
+
+```python showLineNumbers title="Inpainting with Mask"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Use flux-pro-1.0-fill for inpainting
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-fill",
+    image=open("path/to/your/image.png", "rb"),
+    mask=open("path/to/mask.png", "rb"),  # White areas will be edited
+    prompt="Replace with a beautiful garden",
+    steps=50,  # BFL-specific parameter
+    guidance=30,  # BFL-specific parameter
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="outpainting" label="Outpainting (Expand)">
+
+```python showLineNumbers title="Outpainting - Expand Image Borders"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Use flux-pro-1.0-expand to extend image borders
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-expand",
+    image=open("path/to/your/image.png", "rb"),
+    prompt="Continue the scene with a mountain landscape",
+    top=256,     # Expand 256 pixels at top
+    bottom=256,  # Expand 256 pixels at bottom
+    left=128,    # Expand 128 pixels at left
+    right=128,   # Expand 128 pixels at right
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="advanced" label="Advanced Parameters">
+
+```python showLineNumbers title="Advanced Image Editing with BFL Parameters"
+import os
+import litellm
+
+# Set your API key
+os.environ["BFL_API_KEY"] = "your-api-key-here"
+
+# Edit image with BFL-specific parameters
+response = litellm.image_edit(
+    model="black_forest_labs/flux-kontext-pro",
+    image=open("path/to/your/image.png", "rb"),
+    prompt="Transform into cyberpunk style with neon lights",
+    seed=42,                    # For reproducible results
+    output_format="png",        # png or jpeg
+    safety_tolerance=2,         # 0-6, higher = more permissive
+    aspect_ratio="16:9",        # Output aspect ratio
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+</Tabs>
+
+### Usage - LiteLLM Proxy Server
+
+#### 1. Configure your config.yaml
+
+```yaml showLineNumbers title="Black Forest Labs Image Editing Configuration"
+model_list:
+  - model_name: bfl-kontext-pro
+    litellm_params:
+      model: black_forest_labs/flux-kontext-pro
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+
+  - model_name: bfl-kontext-max
+    litellm_params:
+      model: black_forest_labs/flux-kontext-max
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+
+  - model_name: bfl-fill
+    litellm_params:
+      model: black_forest_labs/flux-pro-1.0-fill
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+
+  - model_name: bfl-expand
+    litellm_params:
+      model: black_forest_labs/flux-pro-1.0-expand
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+
+general_settings:
+  master_key: sk-1234
+```
+
+#### 2. Start LiteLLM Proxy Server
+
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### 3. Make image editing requests
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="Black Forest Labs via Proxy - OpenAI SDK"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="sk-1234"
+)
+
+# Edit image with FLUX Kontext Pro
+response = client.images.edit(
+    model="bfl-kontext-pro",
+    image=open("path/to/your/image.png", "rb"),
+    prompt="Add magical sparkles and fairy dust",
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="Black Forest Labs via Proxy - cURL"
+curl --location 'http://localhost:4000/v1/images/edits' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'model="bfl-kontext-pro"' \
+--form 'prompt="Add a sunset in the background"' \
+--form 'image=@"path/to/your/image.png"'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Parameters
+
+### OpenAI-Compatible Parameters
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `image` | file | The image file to edit | Required |
+| `prompt` | string | Text description of the desired changes | Required |
+| `model` | string | The FLUX model to use | Required |
+| `mask` | file | Mask image for inpainting (flux-pro-1.0-fill) | Optional |
+| `n` | integer | Number of images (BFL returns 1 per request) | `1` |
+| `size` | string | Maps to aspect_ratio | Optional |
+| `response_format` | string | `url` or `b64_json` | `url` |
+
+### Black Forest Labs Specific Parameters
+
+| Parameter | Type | Description | Default | Models |
+|-----------|------|-------------|---------|--------|
+| `seed` | integer | Seed for reproducible results | Random | All |
+| `output_format` | string | Output format: `png` or `jpeg` | `png` | All |
+| `safety_tolerance` | integer | Safety filter tolerance (0-6) | 2 | All |
+| `aspect_ratio` | string | Output aspect ratio (e.g., `16:9`, `1:1`) | Original | Kontext models |
+| `steps` | integer | Number of inference steps | Model default | Fill |
+| `guidance` | float | Guidance scale | Model default | Fill |
+| `grow_mask` | integer | Pixels to grow mask | 0 | Fill |
+| `top` | integer | Pixels to expand at top | 0 | Expand |
+| `bottom` | integer | Pixels to expand at bottom | 0 | Expand |
+| `left` | integer | Pixels to expand at left | 0 | Expand |
+| `right` | integer | Pixels to expand at right | 0 | Expand |
+
+## How It Works
+
+Black Forest Labs uses a polling-based API:
+
+1. **Submit Request**: LiteLLM sends your image and prompt to BFL
+2. **Get Task ID**: BFL returns a task ID and polling URL
+3. **Poll for Result**: LiteLLM automatically polls until the image is ready
+4. **Return Result**: The generated image URL is returned
+
+This polling is handled automatically by LiteLLM - you just call `image_edit()` and get the result.
+
+## Getting Started
+
+1. Create an account at [Black Forest Labs](https://blackforestlabs.ai/)
+2. Get your API key from the dashboard
+3. Set your `BFL_API_KEY` environment variable
+4. Use `litellm.image_edit()` with any supported model
+
+## Additional Resources
+
+- [Black Forest Labs Documentation](https://docs.bfl.ai/)
+- [FLUX Model Information](https://blackforestlabs.ai/)

docs/my-website/docs/providers/gemini.md

@@ -1562,13 +1562,18 @@ LiteLLM Supports the following image types passed in `url`
 
 ## Media Resolution Control (Images & Videos)
 
-For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
+LiteLLM supports OpenAI's `detail` parameter for specifying the image resolution when using Gemini models. The behavior differs between Gemini versions:
+
+| Gemini Version | Resolution Control | Behavior |
+|----------------|-------------------|----------|
+| Gemini 3+ | Per-part | Each image/video can have its own `detail` setting |
+| Gemini 2.x (2.0, 2.5) | Global | The highest `detail` from all images is applied globally via `mediaResolution` in `generationConfig` |
 
 **Supported `detail` values:**
-- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
-- `"medium"` - Maps to `media_resolution: "medium"`
-- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
-- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
+- `"low"` - Maps to `MEDIA_RESOLUTION_LOW` (280 tokens for images, 70 tokens per frame for videos)
+- `"medium"` - Maps to `MEDIA_RESOLUTION_MEDIUM`
+- `"high"` - Maps to `MEDIA_RESOLUTION_HIGH` (1120 tokens for images)
+- `"ultra_high"` - Maps to `MEDIA_RESOLUTION_ULTRA_HIGH`
 - `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
 
 **Usage Examples:**
@@ -1605,8 +1610,9 @@ messages = [
     }
 ]
 
+# Works with both Gemini 2.x and 3+
 response = completion(
-    model="gemini/gemini-3-pro-preview",
+    model="gemini/gemini-2.5-flash",  # or gemini-3-pro-preview
     messages=messages,
 )
 ```
@@ -1647,7 +1653,9 @@ response = completion(
 </Tabs>
 
 :::info
-**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
+**Gemini 3+ Per-Part Resolution:** Each image or video can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This works with both `image_url` and `file` content types.
+
+**Gemini 2.x Global Resolution:** When multiple images have different `detail` values, LiteLLM uses the highest resolution found and applies it globally via `mediaResolution` in `generationConfig` (e.g., if one image has `"low"` and another has `"high"`, all images will use `"high"`).
 :::
 
 ## Video Metadata Control

docs/my-website/docs/providers/mistral.md

@@ -311,6 +311,79 @@ print(response)
 - **Model Compatibility**: Reasoning parameters only work with magistral models
 - **Backward Compatibility**: Non-magistral models will ignore reasoning parameters and work normally
 
+## Audio Transcription
+
+Use Mistral's Voxtral models for audio transcription via `litellm.transcription()`.
+
+### SDK Usage
+
+```python
+from litellm import transcription
+import os
+
+os.environ["MISTRAL_API_KEY"] = ""
+
+audio_file = open("path/to/audio.wav", "rb")
+
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+)
+
+print(response.text)
+```
+
+### With Optional Parameters
+
+```python
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+    language="en",
+    temperature=0.0,
+    response_format="json",
+)
+```
+
+### Mistral-Specific Parameters
+
+Mistral supports additional parameters beyond the OpenAI-compatible ones:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `diarize` | `bool` | Enable speaker diarization |
+
+```python
+response = transcription(
+    model="mistral/voxtral-mini-latest",
+    file=audio_file,
+    diarize=True,
+)
+```
+
+### Usage with LiteLLM Proxy
+
+```yaml
+model_list:
+  - model_name: voxtral
+    litellm_params:
+      model: mistral/voxtral-mini-latest
+      api_key: os.environ/MISTRAL_API_KEY
+    model_info:
+      mode: audio_transcription
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+```bash
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"audio.wav"' \
+--form 'model="voxtral"'
+```
+
 ## Sample Usage - Embedding
 ```python
 from litellm import embedding

docs/my-website/docs/providers/openai.md

@@ -632,14 +632,75 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 ## OpenAI Chat Completion to Responses API Bridge
 
-Call any Responses API model from OpenAI's `/chat/completions` endpoint. 
+LiteLLM offers a chat completion to Responses API bridge. This lets you use the completion interface while calling the Responses API under the hood.
+
+This is useful when you want to use [Responses API](https://platform.openai.com/docs/api-reference/responses) specific features (like built-in tools, web search preview, or code interpreter).
+
+:::tip gpt-5.4 + reasoning_effort + function tools
+
+LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API.
+
+If you need reasoning **and** tools together, use the responses bridge instead:
+
+```python
+response = litellm.completion(
+    model="openai/responses/gpt-5.4",  # routes to /v1/responses
+    messages=[{"role": "user", "content": "What's the weather?"}],
+    tools=[...],
+    reasoning_effort="low",
+)
+```
+
+:::
+
+### When to use the `openai/responses/` prefix
+
+Each model has a `mode` property defined in [`model_prices_and_context_window.json`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) that determines which API endpoint it uses by default:
+
+- **`mode: responses`** - Model automatically uses the Responses API
+- **`mode: chat`** - Model defaults to the Chat Completions API
+
+**Models with `mode: responses`** (automatic Responses API):
+- `o3-deep-research`, `o4-mini-deep-research`
+- `o1-pro`, `o3-pro`
+- `gpt-5.1-codex`, `gpt-5.1-codex-mini`, `gpt-5.1-codex-max`
+- `codex-mini-latest`
+
+**Models with `mode: chat`** (require `openai/responses/` prefix for built-in tools):
+- `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`
+- `gpt-5`, `gpt-5-mini`
+- `o3`, `o4-mini`
+
+To use built-in tools like `web_search_preview` with `mode: chat` models, add the `openai/responses/` prefix:
+
+```python
+# This will FAIL - gpt-4o has mode: chat, uses Chat Completions API
+response = litellm.completion(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "What is the weather in Paris today?"}],
+    tools=[{"type": "web_search_preview"}],  # Not supported in Chat Completions
+    # ... other kwargs
+)
+
+# This will WORK - prefix forces Responses API
+response = litellm.completion(
+    model="openai/responses/gpt-4o",
+    messages=[{"role": "user", "content": "What is the weather in Paris today?"}],
+    tools=[{"type": "web_search_preview"}],  # Supported in Responses API
+    # ... other kwargs
+)
+```
+
+### Examples
 
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
+**Using a model with `mode: responses` (automatic):**
+
 ```python
 import litellm
-import os 
+import os
 
 os.environ["OPENAI_API_KEY"] = "sk-1234"
 
@@ -653,6 +714,26 @@ response = litellm.completion(
 )
 print(response)
 ```
+
+**Using a model with `mode: chat` (requires prefix):**
+
+```python
+import litellm
+import os
+
+os.environ["OPENAI_API_KEY"] = "sk-1234"
+
+# Use the openai/responses/ prefix to enable built-in tools
+response = litellm.completion(
+    model="openai/responses/gpt-4o",
+    messages=[{"role": "user", "content": "What is the weather in Paris today?"}],
+    tools=[
+        {"type": "web_search_preview"},
+    ],
+)
+print(response)
+```
+
 </TabItem>
 <TabItem value="proxy" label="PROXY">
 
@@ -660,10 +741,17 @@ print(response)
 
 ```yaml
 model_list:
-  - model_name: openai-model
+  # Model with mode: responses (automatic)
+  - model_name: o3-deep-research
     litellm_params:
       model: o3-deep-research-2025-06-26
       api_key: os.environ/OPENAI_API_KEY
+
+  # Model with mode: chat (use prefix for built-in tools)
+  - model_name: gpt-4o-with-tools
+    litellm_params:
+      model: openai/responses/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
 ```
 
 2. Start the proxy
@@ -678,15 +766,14 @@ litellm --config config.yaml
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer sk-1234' \
--d '{ 
-    "model": "openai-model",
+-d '{
+    "model": "gpt-4o-with-tools",
     "messages": [
-        {"role": "user", "content": "What is the capital of France?"}
+        {"role": "user", "content": "What is the weather in Paris today?"}
     ],
     "tools": [
-        {"type": "web_search_preview"},
-        {"type": "code_interpreter", "container": {"type": "auto"}},
-    ],
+        {"type": "web_search_preview"}
+    ]
 }'
 ```
 

docs/my-website/docs/providers/openai/responses_api.md

@@ -693,6 +693,236 @@ print(final_response.output)
 
 Set `parallel_tool_calls=False` to ensure zero or one tool is called per turn. [More details](https://platform.openai.com/docs/guides/function-calling#parallel-function-calling).
 
+## Tool Search & Namespaces
+
+Tool search lets models dynamically load tools at runtime instead of sending every tool definition in the prompt. Group functions into **namespaces** and mark them with `defer_loading: true` — the model only loads the schemas it actually needs, saving tokens.
+
+Requires `gpt-5.4` or later. See [OpenAI Tool Search docs](https://developers.openai.com/api/docs/guides/tools-tool-search) for full details.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python showLineNumbers title="Tool Search with Namespaces"
+import litellm
+
+# Define namespaces with deferred tools
+tools = [
+    {"type": "tool_search"},  # Enable tool search
+    {
+        "type": "namespace",
+        "name": "crm",
+        "description": "CRM tools for customer management",
+        "tools": [
+            {
+                "type": "function",
+                "name": "get_customer",
+                "description": "Get customer details by ID",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "customer_id": {"type": "string"}
+                    },
+                    "required": ["customer_id"],
+                },
+                "defer_loading": True,
+            },
+            {
+                "type": "function",
+                "name": "list_customers",
+                "description": "List customers with optional filters",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "status": {"type": "string", "enum": ["active", "inactive"]},
+                    },
+                },
+                "defer_loading": True,
+            },
+        ],
+    },
+    {
+        "type": "namespace",
+        "name": "billing",
+        "description": "Billing and invoicing tools",
+        "tools": [
+            {
+                "type": "function",
+                "name": "get_invoice",
+                "description": "Get an invoice by ID",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "invoice_id": {"type": "string"}
+                    },
+                    "required": ["invoice_id"],
+                },
+                "defer_loading": True,
+            },
+        ],
+    },
+]
+
+response = litellm.responses(
+    model="openai/gpt-5.4",
+    input="Look up invoice INV-2024-001 from the billing system",
+    tools=tools,
+)
+
+# The response contains tool_search_call, tool_search_output, and function_call items
+for item in response.output:
+    if isinstance(item, dict):
+        if item["type"] == "tool_search_call":
+            print(f"Searched namespaces: {item['arguments']['paths']}")
+        elif item["type"] == "tool_search_output":
+            print(f"Loaded {len(item['tools'])} tool(s)")
+        elif item["type"] == "function_call":
+            print(f"Called: {item.get('namespace', '')}.{item['name']}({item['arguments']})")
+    else:
+        if item.type == "function_call":
+            print(f"Called: {item.namespace}.{item.name}({item.arguments})")
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+1. Set up config.yaml
+
+```yaml showLineNumbers title="OpenAI Proxy Configuration"
+model_list:
+  - model_name: openai/gpt-5.4
+    litellm_params:
+      model: openai/gpt-5.4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start LiteLLM Proxy Server
+
+```bash title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```python showLineNumbers title="Tool Search via OpenAI SDK with LiteLLM Proxy"
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-api-key"
+)
+
+response = client.responses.create(
+    model="openai/gpt-5.4",
+    input="Look up invoice INV-2024-001 from the billing system",
+    tools=[
+        {"type": "tool_search"},
+        {
+            "type": "namespace",
+            "name": "billing",
+            "description": "Billing and invoicing tools",
+            "tools": [
+                {
+                    "type": "function",
+                    "name": "get_invoice",
+                    "description": "Get an invoice by ID",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"invoice_id": {"type": "string"}},
+                        "required": ["invoice_id"],
+                    },
+                    "defer_loading": True,
+                },
+            ],
+        },
+    ],
+)
+
+print(response.output)
+```
+
+</TabItem>
+</Tabs>
+
+### Tool Search via Chat Completions Bridge
+
+You can also use tool search through the `/v1/chat/completions` endpoint by prefixing the model with `openai/responses/`. The request is routed through the Responses API but returns a standard chat completions response.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python showLineNumbers title="Tool Search via Chat Completions Bridge"
+import litellm
+
+response = litellm.completion(
+    model="openai/responses/gpt-5.4",
+    messages=[{"role": "user", "content": "Look up invoice INV-2024-001"}],
+    tools=[
+        {"type": "tool_search"},
+        {
+            "type": "namespace",
+            "name": "billing",
+            "description": "Billing and invoicing tools",
+            "tools": [
+                {
+                    "type": "function",
+                    "name": "get_invoice",
+                    "description": "Get an invoice by ID",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"invoice_id": {"type": "string"}},
+                        "required": ["invoice_id"],
+                    },
+                    "defer_loading": True,
+                },
+            ],
+        },
+    ],
+)
+
+# Standard chat completions response
+for tool_call in response.choices[0].message.tool_calls:
+    print(f"Called: {tool_call.function.name}({tool_call.function.arguments})")
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```bash showLineNumbers title="Tool Search via /v1/chat/completions"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "openai/responses/gpt-5.4",
+    "messages": [{"role": "user", "content": "Look up invoice INV-2024-001"}],
+    "tools": [
+      {"type": "tool_search"},
+      {
+        "type": "namespace",
+        "name": "billing",
+        "description": "Billing and invoicing tools",
+        "tools": [
+          {
+            "type": "function",
+            "name": "get_invoice",
+            "description": "Get an invoice by ID",
+            "parameters": {
+              "type": "object",
+              "properties": {"invoice_id": {"type": "string"}},
+              "required": ["invoice_id"]
+            },
+            "defer_loading": true
+          }
+        ]
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 ## Free-form Function Calling
 
 <Tabs>

docs/my-website/docs/providers/openai/videos.md

@@ -135,6 +135,81 @@ curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix'
 }'
 ```
 
+### Character, Edit, and Extension Routes
+
+OpenAI video routes supported by LiteLLM proxy:
+
+- `POST /v1/videos/characters`
+- `GET /v1/videos/characters/{character_id}`
+- `POST /v1/videos/edits`
+- `POST /v1/videos/extensions`
+
+#### `target_model_names` support on character creation
+
+`POST /v1/videos/characters` supports `target_model_names` for model-based routing (same behavior as video create).
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/characters' \
+--header 'Authorization: Bearer sk-1234' \
+-F 'name=hero' \
+-F 'target_model_names=gpt-4' \
+-F 'video=@/path/to/character.mp4'
+```
+
+When `target_model_names` is used, LiteLLM returns an encoded character ID:
+
+```json
+{
+  "id": "character_...",
+  "object": "character",
+  "created_at": 1712697600,
+  "name": "hero"
+}
+```
+
+Use that encoded ID directly on get:
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/characters/character_...' \
+--header 'Authorization: Bearer sk-1234'
+```
+
+#### Encoded and non-encoded video IDs for edit/extension
+
+Both routes accept either plain or encoded `video.id`:
+
+- `POST /v1/videos/edits`
+- `POST /v1/videos/extensions`
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/edits' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "prompt": "Make this brighter",
+  "video": { "id": "video_..." }
+}'
+```
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/extensions' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "prompt": "Continue this scene",
+  "seconds": "4",
+  "video": { "id": "video_..." }
+}'
+```
+
+#### `custom_llm_provider` input sources
+
+For these routes, `custom_llm_provider` may be supplied via:
+
+- header: `custom-llm-provider`
+- query: `?custom_llm_provider=...`
+- body: `custom_llm_provider` (and `extra_body.custom_llm_provider` where supported)
+
 Test OpenAI video generation request
 
 ```bash

docs/my-website/docs/providers/vertex_embedding.md

@@ -79,6 +79,7 @@ All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a02
 | textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | 
 | text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
 | text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | 
+| gemini-embedding-2-preview | `embedding(model="vertex_ai/gemini-embedding-2-preview", input)` | [Multimodal docs](#gemini-embedding-2-preview-multimodal) |
 | Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` | 
 
 ### Supported OpenAI (Unified) Params
@@ -257,6 +258,71 @@ model_list:
 
 ## **Multi-Modal Embeddings**
 
+### Gemini Embedding 2 Preview (Multimodal)
+
+`gemini-embedding-2-preview` supports **unified multimodal embeddings**—text, images, audio, video, and PDF in a single request. See [blog post](/blog/gemini_embedding_2_multimodal) for details.
+
+**Input formats:**
+- **Data URIs:** `data:image/png;base64,<encoded_data>`
+- **GCS URLs:** `gs://bucket/path/to/file.png` (MIME type inferred from extension)
+
+**Supported MIME types:** `image/png`, `image/jpeg`, `audio/mpeg`, `audio/wav`, `video/mp4`, `video/quicktime`, `application/pdf`
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+from litellm import embedding
+
+litellm.vertex_project = "your-project-id"
+litellm.vertex_location = "us-central1"
+
+# Text + Image (GCS URL)
+response = embedding(
+    model="vertex_ai/gemini-embedding-2-preview",
+    input=[
+        "Describe this image",
+        "gs://my-bucket/images/photo.png"
+    ],
+)
+
+# Text + Image (base64)
+response = embedding(
+    model="vertex_ai/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM PROXY">
+
+```yaml
+model_list:
+  - model_name: vertex-gemini-embedding-2-preview
+    litellm_params:
+      model: vertex_ai/gemini-embedding-2-preview
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+```
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-gemini-embedding-2-preview",
+    "input": ["Describe this", "gs://bucket/image.png"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### multimodalembedding@001 (Legacy)
 
 Known Limitations:
 - Only supports 1 image / video / image per request

docs/my-website/docs/providers/vertex_partner.md

@@ -11,6 +11,7 @@ import TabItem from '@theme/TabItem';
 |----------|---------------|---------------|
 | Anthropic (Claude) | `vertex_ai/claude-*` | [Vertex AI - Anthropic Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude) |
 | DeepSeek | `vertex_ai/deepseek-ai/{MODEL}` | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) |
+| ZAI (GLM) | `vertex_ai/zai-org/{MODEL}` | [Vertex AI - GLM Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/zaiorg/glm-47) |
 | Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) |
 | Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
 | AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
@@ -226,6 +227,79 @@ ModelResponse(
 |------------------|------------------------------|
 | vertex_ai/deepseek-ai/deepseek-r1-0528-maas | `completion('vertex_ai/deepseek-ai/deepseek-r1-0528-maas', messages)` |
 
+## VertexAI ZAI (GLM)
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/zai-org/{MODEL}` |
+| Vertex Documentation | [Vertex AI - GLM Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/zaiorg/glm-47) |
+
+**LiteLLM Supports all Vertex AI GLM Models.** Ensure you use the `vertex_ai/zai-org/` prefix for all Vertex AI GLM models.
+
+| Model Name | Usage |
+|------------|-------|
+| vertex_ai/zai-org/glm-4.7-maas | `completion('vertex_ai/zai-org/glm-4.7-maas', messages)` |
+
+#### Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+response = completion(
+    model="vertex_ai/zai-org/glm-4.7-maas",
+    messages=[{"role": "user", "content": "hi"}],
+    vertex_project="your-vertex-project",
+    # vertex_location routes to "global"
+)
+print("\nModel Response", response)
+```
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml
+model_list:
+  - model_name: glm-4.7
+    litellm_params:
+      model: vertex_ai/zai-org/glm-4.7-maas
+      vertex_project: "my-project"
+      # vertex_location routes to "global"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "glm-4.7",
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
 
 ## VertexAI Meta/Llama API
  

docs/my-website/docs/proxy/config_settings.md

@@ -355,7 +355,7 @@ router_settings:
 | set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
 | retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
 | provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
-| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
+| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. **Required** for `model_info.max_input_tokens` enforcement. Default: false. [More information here](reliability) |
 | model_group_retry_policy | Dict[str, RetryPolicy] | [SDK-only arg] Set retry policy for model groups. |
 | context_window_fallbacks | List[Dict[str, List[str]]] | Fallback models for context window violations. |
 | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
@@ -778,6 +778,7 @@ router_settings:
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
 | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
+| LITELLM_DISABLE_REDACT_SECRETS | When set to "true", disables automatic redaction of secrets (API keys, tokens, credentials) from proxy log output. Secret redaction is enabled by default.
 | LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
 | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
 | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
@@ -804,6 +805,7 @@ router_settings:
 | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
 | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
 | LITELLM_MASTER_KEY | Master key for proxy authentication
+| LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour)
 | LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
 | LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
@@ -909,6 +911,7 @@ router_settings:
 | PILLAR_API_BASE | Base URL for Pillar API Guardrails
 | PILLAR_API_KEY | API key for Pillar API Guardrails
 | PILLAR_ON_FLAGGED_ACTION | Action to take when content is flagged ('block' or 'monitor')
+| PKCE_STRICT_CACHE_MISS | When set to `true`, the SSO callback will return a 401 error if the PKCE code_verifier is not found in the cache (e.g. due to a cache miss across pods). When `false` (default), it logs a warning and continues without the code_verifier.
 | POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME` 
 | POSTHOG_API_KEY | API key for PostHog analytics integration
 | POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
@@ -933,6 +936,9 @@ router_settings:
 | PROXY_BASE_URL | Base URL for proxy service
 | PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
+| PROXY_BATCH_POLLING_ENABLED | Set to `false` to disable the `CheckBatchCost` and `CheckResponsesCost` background polling jobs entirely. Useful for emergency mitigation on installs with large numbers of stale managed objects. Default is `true`
+| MAX_OBJECTS_PER_POLL_CYCLE | Maximum number of managed objects (batches / responses) fetched per polling cycle. Prevents OOM on installs with many stale rows. Default is `50`
+| MANAGED_OBJECT_STALENESS_CUTOFF_DAYS | Managed objects older than this many days in a non-terminal state are marked `stale_expired` at the start of each poll cycle and skipped. Default is `7`
 | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
 | PYTHON_GC_THRESHOLD | GC thresholds ('gen0,gen1,gen2', e.g. '1000,50,50'); defaults to Python’s values.
@@ -943,7 +949,7 @@ router_settings:
 | QDRANT_URL | Connection URL for Qdrant database
 | QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
 | REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
-| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: '[{"host": "node1", "port": 6379}]'
+| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: `[{"host": "node1", "port": 6379}]`
 | REDIS_HOST | Hostname for Redis server
 | REDIS_PASSWORD | Password for Redis service
 | REDIS_PORT | Port number for Redis server
@@ -1016,6 +1022,11 @@ router_settings:
 | UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
 | USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
 | USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
+| VANTAGE_API_KEY | API key for Vantage cost-import integration
+| VANTAGE_BASE_URL | Base URL for Vantage API. Default is `https://api.vantage.sh`
+| VANTAGE_EXPORT_FREQUENCY | Export frequency for Vantage — `hourly` (default), `daily`, or `interval`
+| VANTAGE_EXPORT_INTERVAL_SECONDS | Interval in seconds when VANTAGE_EXPORT_FREQUENCY is `interval`
+| VANTAGE_INTEGRATION_TOKEN | Vantage integration token for the cost-import endpoint
 | WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration
 | WANDB_HOST | Host URL for Weights & Biases (W&B) service
 | WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration

docs/my-website/docs/proxy/guardrails/guardrail_policies.md

@@ -309,6 +309,10 @@ Response:
 </TabItem>
 </Tabs>
 
+## Policy Flow Builder
+
+For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step pass/fail actions.
+
 ## Config Reference
 
 ### `policies`
@@ -323,6 +327,7 @@ policies:
       remove: [...]
     condition:
       model: ...
+    pipeline: ...  # optional; see Policy Flow Builder
 ```
 
 | Field | Type | Description |
@@ -332,6 +337,7 @@ policies:
 | `guardrails.add` | `list[string]` | Guardrails to enable. |
 | `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
 | `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
+| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions. See [Policy Flow Builder](./policy_flow_builder). |
 
 ### `policy_attachments`
 

docs/my-website/docs/proxy/guardrails/panw_prisma_airs.md

@@ -1,24 +1,15 @@
 import Image from '@theme/IdealImage';
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
 
 # PANW Prisma AIRS
 
-LiteLLM supports PANW Prisma AIRS (AI Runtime Security) guardrails via the [Prisma AIRS Scan API](https://pan.dev/prisma-airs/api/airuntimesecurity/airuntimesecurityapi//). This integration provides **Security-as-Code** for AI applications using Palo Alto Networks' AI security platform.
+LiteLLM supports PANW Prisma AIRS (AI Runtime Security) guardrails via the [Prisma AIRS Scan API](https://pan.dev/prisma-airs/api/airuntimesecurity/airuntimesecurityapi/). This integration provides Security-as-Code for AI applications using Palo Alto Networks' AI security platform.
 
-## Features
+- **Prompt injection and malicious URL detection** — real-time scanning before or after LLM calls
+- **Data loss prevention (DLP)** — detect and block sensitive data in prompts and responses
+- **Sensitive content masking** — automatically mask PII, credit cards, SSNs instead of blocking
+- **MCP tool call scanning** — scan tool name and arguments on direct MCP tool invocations
+- **Configurable fail-open / fail-closed** — choose between maximum security or high availability
 
-- ✅ **Real-time prompt injection detection**
-- ✅ **Malicious URL detection** 
-- ✅ **Data loss prevention (DLP)**
-- ✅ **Sensitive content masking** - Automatically mask PII, credit cards, SSNs instead of blocking
-- ✅ **Comprehensive threat detection** for AI models and datasets
-- ✅ **Model-agnostic protection** across public and private models
-- ✅ **Synchronous scanning** with immediate response
-- ✅ **Configurable security profiles**
-- ✅ **Streaming support** - Real-time masking for streaming responses
-- ✅ **Multi-turn conversation tracking** - Automatic session grouping in Prisma AIRS SCM logs
-- ✅ **Configurable fail-open/fail-closed** - Choose between maximum security (block on API errors) or high availability (allow on transient errors)
 
 ## Quick Start
 
@@ -32,7 +23,14 @@ For detailed setup instructions, see the [Prisma AIRS API Overview](https://docs
 
 ### 2. Define Guardrails on your LiteLLM config.yaml
 
-Define your guardrails under the `guardrails` section:
+Set `api_base` to the regional endpoint for your Prisma AIRS deployment profile:
+
+| Region | Endpoint |
+|--------|----------|
+| US | `https://service.api.aisecurity.paloaltonetworks.com` |
+| EU (Germany) | `https://service-de.api.aisecurity.paloaltonetworks.com` |
+| India | `https://service-in.api.aisecurity.paloaltonetworks.com` |
+| Singapore | `https://service-sg.api.aisecurity.paloaltonetworks.com` |
 
 ```yaml
 model_list:
@@ -45,21 +43,15 @@ guardrails:
   - guardrail_name: "panw-prisma-airs-guardrail"
     litellm_params:
       guardrail: panw_prisma_airs
-      mode: "pre_call"                    # Run before LLM call
-      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY    # Your Prisma AIRS API key
-      profile_name: os.environ/PANW_PRISMA_AIRS_PROFILE_NAME  # Security profile from Strata Cloud Manager
-      api_base: "https://service.api.aisecurity.paloaltonetworks.com"  
+      mode: "pre_call"
+      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
+      profile_name: os.environ/PANW_PRISMA_AIRS_PROFILE_NAME
+      api_base: "https://service.api.aisecurity.paloaltonetworks.com"  # US — change to your region
 ```
 
-#### Supported values for `mode`
-
-- `pre_call` Run **before** LLM call, on **input**
-- `post_call` Run **after** LLM call, on **input & output**  
-- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel with LLM call
-
 ### 3. Start LiteLLM Gateway
 
-```bash title="Set environment variables"
+```bash
 export PANW_PRISMA_AIRS_API_KEY="your-panw-api-key"
 export PANW_PRISMA_AIRS_PROFILE_NAME="your-security-profile"
 export OPENAI_API_KEY="sk-proj-..."
@@ -69,15 +61,8 @@ export OPENAI_API_KEY="sk-proj-..."
 litellm --config config.yaml --detailed_debug
 ```
 
-
 ### 4. Test Request
 
-**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
-
-<Tabs>
-<TabItem label="Blocked request" value="blocked">
-
-Expect this to fail due to prompt injection attempt:
 
 ```shell
 curl -i http://localhost:4000/v1/chat/completions \
@@ -92,254 +77,57 @@ curl -i http://localhost:4000/v1/chat/completions \
   }'
 ```
 
-Expected response on failure:
+Expected response when the guardrail blocks:
 
 ```json
 {
   "error": {
-    "message": {
-      "error": "Violated PANW Prisma AIRS guardrail policy",
-      "panw_response": {
-        "action": "block",
-        "category": "malicious",
-        "profile_id": "03b32734-d06d-4bb7-a8df-ac5147630ce8",
-        "profile_name": "dev-block-all-profile",
-        "prompt_detected": {
-          "dlp": false,
-          "injection": true,
-          "toxic_content": false,
-          "url_cats": false
-        },
-        "report_id": "Rbd251eac-6e67-433b-b3ef-8eb42d2c7d2c",
-        "response_detected": {
-          "dlp": false,
-          "toxic_content": false,
-          "url_cats": false
-        },
-        "scan_id": "bd251eac-6e67-433b-b3ef-8eb42d2c7d2c",
-        "tr_id": "string"
-      }
-    },
-    "type": "None",
-    "param": "None",
-    "code": "400"
+    "message": "Prompt blocked by PANW Prisma AI Security policy (Category: malicious)",
+    "type": "guardrail_violation",
+    "code": "panw_prisma_airs_blocked",
+    "guardrail": "panw-prisma-airs-guardrail",
+    "category": "malicious"
   }
 }
 ```
 
-</TabItem>
-<TabItem label="Successful Call" value="allowed">
+LiteLLM wraps this detail in an endpoint-specific HTTP error envelope. Optional fields that may also appear: `scan_id`, `report_id`, `profile_name`, `profile_id`, `tr_id`, `prompt_detected`.
 
-```shell
-curl -i http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-your-api-key" \
-  -d '{
-    "model": "gpt-4o",
-    "messages": [
-      {"role": "user", "content": "What is the weather like today?"}
-    ],
-    "guardrails": ["panw-prisma-airs-guardrail"]
-  }'
-```
+On success, the guardrail name appears in the `x-litellm-applied-guardrails` response header.
 
-Expected successful response:
+## Configuration
 
-```json
-{
-  "choices": [
-    {
-      "finish_reason": "stop",
-      "index": 0,
-      "message": {
-        "content": "I don't have access to real-time weather data, but I can help you find weather information through various weather services or apps...",
-        "role": "assistant",
-        "tool_calls": null,
-        "function_call": null,
-        "annotations": []
-      }
-    }
-  ],
-  "created": 1736028456,
-  "id": "chatcmpl-AqQj8example",
-  "model": "gpt-4o",
-  "object": "chat.completion",
-  "usage": {
-    "completion_tokens": 25,
-    "prompt_tokens": 12,
-    "total_tokens": 37
-  },
-  "x-litellm-panw-scan": {
-    "action": "allow",
-    "category": "benign",
-    "profile_id": "03b32734-d06d-4bb7-a8df-ac5147630ce8",
-    "profile_name": "dev-block-all-profile",
-    "prompt_detected": {
-      "dlp": false,
-      "injection": false,
-      "toxic_content": false,
-      "url_cats": false
-    },
-    "report_id": "Rbd251eac-6e67-433b-b3ef-8eb42d2c7d2c",
-    "response_detected": {
-      "dlp": false,
-      "toxic_content": false,
-      "url_cats": false
-    },
-    "scan_id": "bd251eac-6e67-433b-b3ef-8eb42d2c7d2c",
-    "tr_id": "string"
-  }
-}
-```
+### Supported Modes
 
-</TabItem>
-</Tabs>
+| Mode | Timing | What is scanned |
+|------|--------|-----------------|
+| `pre_call` | Before LLM call | Request input |
+| `during_call` | Parallel with LLM call | Request input |
+| `post_call` | After LLM call | Response output |
+| `pre_mcp_call` | Before MCP tool execution | MCP tool input |
+| `during_mcp_call` | Parallel with MCP tool execution | MCP tool input |
 
-## Configuration Parameters
+
+### Configuration Parameters
 
 | Parameter | Required | Description | Default |
 |-----------|----------|-------------|---------|
 | `api_key` | Yes | Your PANW Prisma AIRS API key from Strata Cloud Manager | - |
 | `profile_name` | No | Security profile name configured in Strata Cloud Manager. Optional if API key has linked profile | - |
-| `app_name` | No | Application identifier for tracking in Prisma AIRS analytics (will be prefixed with "LiteLLM-") | `LiteLLM` |
-| `api_base` | No | Regional API endpoint (see [Regional Endpoints](#regional-endpoints) below) | `https://service.api.aisecurity.paloaltonetworks.com` (US) |
-| `mode` | No | When to run the guardrail | `pre_call` |
-| `fallback_on_error` | No | Action when PANW API is unavailable: `"block"` (fail-closed, default) or `"allow"` (fail-open). Config errors always block. | `block` |
-| `timeout` | No | PANW API call timeout in seconds (1-60) | `10.0` |
-| `violation_message_template` | No | Custom template for error message when request is blocked. Supports `{guardrail_name}`, `{category}`, `{action_type}`, `{default_message}` placeholders. | - |
+| `app_name` | No | Application identifier for tracking in Prisma AIRS analytics (prefixed with "LiteLLM-") | `LiteLLM` |
+| `api_base` | No | Regional API endpoint. US: `https://service.api.aisecurity.paloaltonetworks.com`, EU: `https://service-de.api.aisecurity.paloaltonetworks.com`, India: `https://service-in.api.aisecurity.paloaltonetworks.com`, Singapore: `https://service-sg.api.aisecurity.paloaltonetworks.com` | US |
+| `mode` | No | When to run the guardrail (see mode table above) | `pre_call` |
+| `fallback_on_error` | No | Action when PANW API is unavailable: `"block"` (fail-closed) or `"allow"` (fail-open). Config errors always block. | `block` |
+| `timeout` | No | PANW API call timeout in seconds (recommended: 1-60) | `10.0` |
+| `violation_message_template` | No | Custom template for blocked requests. Supports `{guardrail_name}`, `{category}`, `{action_type}`, `{default_message}` placeholders. | - |
+| `mask_request_content` | No | Mask sensitive data in prompts instead of blocking | `false` |
+| `mask_response_content` | No | Mask sensitive data in responses instead of blocking | `false` |
+| `mask_on_block` | No | Backwards-compatible flag that enables both request and response masking | `false` |
+| `experimental_use_latest_role_message_only` | No | Anthropic `/v1/messages` only. When unset: scans only latest user message on request side. Set `false` to scan all user/system/developer messages. Non-Anthropic unaffected. | Unset (true for Anthropic) |
 
-### Regional Endpoints
+Use the regional `api_base` that matches your Prisma AIRS deployment profile region for lower latency and data residency compliance.
 
-PANW Prisma AIRS supports multiple regional endpoints based on your deployment profile region:
-
-| Region | API Base URL |
-|--------|--------------|
-| **US** (default) | `https://service.api.aisecurity.paloaltonetworks.com` |
-| **EU (Germany)** | `https://service-de.api.aisecurity.paloaltonetworks.com` |
-| **India** | `https://service-in.api.aisecurity.paloaltonetworks.com` |
-
-**Example configuration for EU region:**
-
-```yaml
-guardrails:
-  - guardrail_name: "panw-eu"
-    litellm_params:
-      guardrail: panw_prisma_airs
-      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      api_base: "https://service-de.api.aisecurity.paloaltonetworks.com"
-      profile_name: "production"
-```
-
-:::tip Region Selection
-Use the regional endpoint that matches your Prisma AIRS deployment profile region configured in Strata Cloud Manager. Using the correct region ensures:
-- Lower latency (requests stay in-region)
-- Compliance with data residency requirements
-- Optimal performance
-:::
-
-## Per-Request Metadata Overrides
-
-You can override guardrail settings on a per-request basis using the `metadata` field:
-
-```json
-{
-  "model": "gpt-4",
-  "messages": [...],
-  "metadata": {
-    "profile_name": "dev-allow-all",            // Override profile name
-    "profile_id": "uuid-here",                  // Override profile ID (takes precedence)
-    "user_ip": "192.168.1.100",                 // Track user IP
-    "app_name": "MyApp"                         // Custom app name (becomes "LiteLLM-MyApp")
-  }
-}
-```
-
-**Supported Metadata Fields:**
-
-| Field | Description | Priority |
-|-------|-------------|----------|
-| `profile_name` | PANW AI security profile name | Per-request > config |
-| `profile_id` | PANW AI security profile ID (takes precedence over profile_name) | Per-request only |
-| `user_ip` | User IP address for tracking in Prisma AIRS | Per-request only |
-| `app_name` | Application identifier (prefixed with "LiteLLM-") | Per-request > config > "LiteLLM" |
-| `app_user` | Custom user identifier for tracking in Prisma AIRS | `app_user` > `user` > "litellm_user" |
-
-:::info Profile Resolution
-- If both `profile_id` and `profile_name` are provided, PANW API uses `profile_id` (it takes precedence)
-- If no profile is specified in metadata, uses the config `profile_name`
-- If no profile is specified at all, PANW API will use the profile linked to your API key in Strata Cloud Manager
-- **Note:** If your API key is not linked to a profile, you must provide `profile_name` or `profile_id`
-:::
-
-## Multi-Turn Conversation Tracking
-
-PANW Prisma AIRS automatically tracks multi-turn conversations using LiteLLM's `litellm_trace_id`. This enables you to:
-
-- **Group related requests** - All requests in a conversation share the same AI Session ID in Prisma AIRS SCM logs
-- **Track conversation context** - See the full history of prompts and responses for a user session
-- **Analyze attack patterns** - Identify sophisticated multi-turn attacks across conversation history
-
-### How It Works
-
-LiteLLM automatically generates a unique `litellm_trace_id` for each conversation session. The PANW guardrail uses this as the PANW transaction ID (which maps to "AI Session ID" in Strata Cloud Manager):
-
-```
-Conversation Session: litellm_trace_id = "abc-123-def-456"
-
-Turn 1 (User):    "What's the capital of France?"
-  → Scan ID: scan_001 | Prisma AIRS AI Session ID: abc-123-def-456
-
-Turn 2 (Assistant): "Paris is the capital of France."
-  → Scan ID: scan_002 | Prisma AIRS AI Session ID: abc-123-def-456
-
-Turn 3 (User):    "What's the population?"
-  → Scan ID: scan_003 | Prisma AIRS AI Session ID: abc-123-def-456
-
-Turn 4 (Assistant): "Paris has approximately 2.1 million residents."
-  → Scan ID: scan_004 | Prisma AIRS AI Session ID: abc-123-def-456
-```
-
-All scans appear under the same AI Session ID in Prisma AIRS logs, making it easy to:
-- Review complete conversation history (all 4 turns grouped together)
-- Identify patterns across multiple turns
-- Correlate security events within a session
-- Track the flow of user prompts and AI responses
-
-### Session Tracking
-
-LiteLLM automatically generates a unique `litellm_trace_id` for each request, which the PANW guardrail uses as the AI Session ID in Strata Cloud Manager. All prompt and response scans for a request are automatically grouped under the same session.
-
-#### Custom Session IDs (Per-App Tracking)
-
-You can provide your own `litellm_trace_id` to track sessions on a per-app or per-conversation basis:
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gpt-3.5-turbo",
-    "messages": [{"role": "user", "content": "capital of France"}],
-    "litellm_trace_id": "my-app-session-123",             # Custom AI Session ID
-    "metadata": {
-      "profile_name": "dev-allow-all-profile",            # Override security profile
-      "user_ip": "192.168.1.1",                           # Track user IP
-      "app_name": "eng"                                   # Custom app identifier
-    },
-    "guardrails": ["panw-prisma-airs-pre-guard", "panw-prisma-airs-post-guard"]
-  }'
-```
-
-**Result in PANW SCM:**
-- AI Session ID: `my-app-session-123`
-- All prompt and response scans will be grouped under this custom session ID
-- Perfect for tracking multi-turn conversations or per-application sessions
-
-:::tip Viewing Sessions in Prisma AIRS SCM Logs
-In Strata Cloud Manager, navigate to **AI Runtime > Sessions** to view all AI Session IDs and their associated scans. Click on a session to see the complete conversation history with security analysis.
-:::
-
-## Environment Variables
+### Environment Variables
 
 ```bash
 export PANW_PRISMA_AIRS_API_KEY="your-panw-api-key"
@@ -348,12 +136,31 @@ export PANW_PRISMA_AIRS_PROFILE_NAME="your-security-profile"
 export PANW_PRISMA_AIRS_API_BASE="https://custom-endpoint.com"
 ```
 
-## Advanced Configuration
+### Per-Request Metadata Overrides
+
+| Field | Description | Priority |
+|-------|-------------|----------|
+| `profile_name` | PANW AI security profile name | Per-request > config |
+| `profile_id` | PANW AI security profile ID (takes precedence over `profile_name`) | Per-request only |
+| `user_ip` | User IP address for tracking in Prisma AIRS | Per-request only |
+| `app_name` | Application identifier (prefixed with "LiteLLM-") | Per-request > config > "LiteLLM" |
+| `app_user` | Custom user identifier for tracking in Prisma AIRS | `app_user` > `user` > "litellm_user" |
+
+```json
+{
+  "model": "gpt-4",
+  "messages": [...],
+  "metadata": {
+    "profile_name": "dev-allow-all",
+    "profile_id": "uuid-here",
+    "user_ip": "192.168.1.100",
+    "app_name": "MyApp"
+  }
+}
+```
 
 ### Multiple Security Profiles
 
-You can configure different security profiles for different use cases:
-
 ```yaml
 guardrails:
   - guardrail_name: "panw-strict-security"
@@ -361,126 +168,40 @@ guardrails:
       guardrail: panw_prisma_airs
       mode: "pre_call"
       api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      profile_name: "strict-policy"       # High security profile
-      
-  - guardrail_name: "panw-permissive-security"  
+      profile_name: "strict-policy"
+
+  - guardrail_name: "panw-permissive-security"
     litellm_params:
       guardrail: panw_prisma_airs
       mode: "post_call"
       api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      profile_name: "permissive-policy"   # Lower security profile
+      profile_name: "permissive-policy"
 ```
 
-### Multiple API Keys (Multi-Tenant)
-
-For multi-tenant deployments where different customers need different PANW API keys, create separate guardrail instances:
-
-```yaml
-guardrails:
-  - guardrail_name: "panw-customer-a"
-    litellm_params:
-      guardrail: panw_prisma_airs
-      mode: "pre_call"
-      api_key: os.environ/PANW_CUSTOMER_A_KEY  # Linked to Customer A profile in SCM
-      
-  - guardrail_name: "panw-customer-b"
-    litellm_params:
-      guardrail: panw_prisma_airs
-      mode: "pre_call"
-      api_key: os.environ/PANW_CUSTOMER_B_KEY  # Linked to Customer B profile in SCM
-```
-
-Then route requests to the appropriate guardrail:
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gpt-4",
-    "messages": [{"role": "user", "content": "Hello"}],
-    "guardrails": ["panw-customer-a"]
-  }'
-```
-
-**Use Cases:**
-- **Multi-tenant deployments**: Different customers with different security policies
-- **Environment-specific policies**: Dev/staging/prod with different API keys and profiles
-- **A/B testing**: Compare different security profiles side-by-side
-
 ### Content Masking
 
-PANW Prisma AIRS can automatically mask sensitive content (PII, credit cards, SSNs, etc.) instead of blocking requests. This allows your application to continue functioning while protecting sensitive data.
-
-#### How It Works
-
-1. **Detection**: PANW scans content and identifies sensitive data
-2. **Masking**: Sensitive data is replaced with placeholders (e.g., `XXXXXXXXXX` or `{PHONE}`)
-3. **Pass-through**: Masked content is sent to the LLM or returned to the user
-
-#### Configuration Options
+:::warning Important: Masking is Controlled by PANW Security Profile
+The actual masking behavior (what content gets masked and how) is controlled by your PANW Prisma AIRS security profile in Strata Cloud Manager. The LiteLLM flags (`mask_request_content`, `mask_response_content`) only control whether to apply the masked content and allow the request to continue, or block entirely.
+:::
 
 ```yaml
 guardrails:
   - guardrail_name: "panw-with-masking"
     litellm_params:
       guardrail: panw_prisma_airs
-      mode: "post_call"                      # Scan response output
+      mode: "post_call"
       api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
       profile_name: "default"
-      mask_request_content: true             # Mask sensitive data in prompts
-      mask_response_content: true            # Mask sensitive data in responses
+      mask_request_content: true
+      mask_response_content: true
 ```
 
-**Masking Parameters:**
-
-- `mask_request_content: true` - When PANW detects sensitive data in prompts, mask it instead of blocking
-- `mask_response_content: true` - When PANW detects sensitive data in responses, mask it instead of blocking  
-- `mask_on_block: true` - Backwards compatible flag that enables both request and response masking
-
-:::warning Important: Masking is Controlled by PANW Security Profile
-The **actual masking behavior** (what content gets masked and how) is controlled by your **PANW Prisma AIRS security profile** configured in Strata Cloud Manager. The LiteLLM config settings (`mask_request_content`, `mask_response_content`) only control whether to:
-- **Apply the masked content** returned by PANW and allow the request to continue, OR
-- **Block the request** entirely when sensitive data is detected
-
-LiteLLM does not alter or configure your PANW security profile. To change what content gets masked, update your profile settings in Strata Cloud Manager.
-:::
-
-:::info Security Posture
-The guardrail is **fail-closed** by default - if the PANW API is unavailable, requests are blocked to ensure no unscanned content reaches your LLM. This provides maximum security.
-:::
-
-### Custom Violation Messages
-
-You can customize the error message returned to the user when a request is blocked by configuring the `violation_message_template` parameter. This is useful for providing user-friendly feedback instead of technical details.
-
-```yaml
-guardrails:
-  - guardrail_name: "panw-custom-message"
-    litellm_params:
-      guardrail: panw_prisma_airs
-      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      # Simple message
-      violation_message_template: "Your request was blocked by our AI Security Policy."
-
-  - guardrail_name: "panw-detailed-message"
-    litellm_params:
-      guardrail: panw_prisma_airs
-      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      # Message with placeholders
-      violation_message_template: "{action_type} blocked due to {category} violation. Please contact support."
-```
-
-**Supported Placeholders:**
-- `{guardrail_name}`: Name of the guardrail (e.g. "panw-custom-message")
-- `{category}`: Violation category (e.g. "malicious", "injection", "dlp")
-- `{action_type}`: "Prompt" or "Response"
-- `{default_message}`: The original technical error message
+- `mask_request_content: true` — mask sensitive data in prompts instead of blocking
+- `mask_response_content: true` — mask sensitive data in responses instead of blocking
+- `mask_on_block: true` — backwards-compatible flag that enables both request and response masking
 
 ### Fail-Open Configuration
 
-By default, the PANW guardrail operates in **fail-closed** mode for maximum security. If the PANW API is unavailable (timeout, rate limit, network error), requests are blocked. You can configure **fail-open** mode for high-availability scenarios where service continuity is critical.
-
 ```yaml
 guardrails:
   - guardrail_name: "panw-high-availability"
@@ -488,135 +209,86 @@ guardrails:
       guardrail: panw_prisma_airs
       api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
       profile_name: "production"
-      fallback_on_error: "allow"  # Enable fail-open mode
-      timeout: 5.0                 # Shorter timeout for fail-open
+      fallback_on_error: "allow"
+      timeout: 5.0
 ```
 
-**Configuration Options:**
-
-| Parameter | Value | Behavior |
-|-----------|-------|----------|
-| `fallback_on_error` | `"block"` (default) | **Fail-closed**: Block requests when API unavailable (maximum security) |
-| `fallback_on_error` | `"allow"` | **Fail-open**: Allow requests when API unavailable (high availability) |
-| `timeout` | `1.0` - `60.0` | API call timeout in seconds (default: `10.0`) |
-
 **Error Handling Matrix:**
 
 | Error Type | `fallback_on_error="block"` | `fallback_on_error="allow"` |
 |------------|----------------------------|----------------------------|
-| 401 Unauthorized | Block (500) | Block (500) ⚠️ |
-| 403 Forbidden | Block (500) | Block (500) ⚠️ |
-| Profile Error | Block (500) | Block (500) ⚠️ |
+| 401 Unauthorized | Block (500) | Block (500) |
+| 403 Forbidden | Block (500) | Block (500) |
+| Profile Error | Block (500) | Block (500) |
 | 429 Rate Limit | Block (500) | Allow (`:unscanned`) |
 | Timeout | Block (500) | Allow (`:unscanned`) |
 | Network Error | Block (500) | Allow (`:unscanned`) |
 | 5xx Server Error | Block (500) | Allow (`:unscanned`) |
 | Content Blocked | Block (400) | Block (400) |
 
-⚠️ = Always blocks regardless of fail-open setting 
+Authentication and configuration errors (401, 403, invalid profile) always block. Only transient errors (429, timeout, network) trigger fail-open.
 
-:::warning Security Trade-Off
-Enabling `fallback_on_error="allow"` reduces security in exchange for availability. Requests may proceed **without scanning** when the PANW API is unavailable. Use only when:
-- Service availability is more critical than security scanning
-- You have other security controls in place
-- You monitor the `:unscanned` header for audit trails
+When fail-open is triggered, the response includes a tracking header: `X-LiteLLM-Applied-Guardrails: panw-airs:unscanned`
 
-**Authentication and configuration errors (401, 403, invalid profile) always block** - only transient errors (429, timeout, network) trigger fail-open behavior.
-:::
-
-**Observability:**
-
-When fail-open is triggered, the response includes a special header for tracking:
-
-```
-X-LiteLLM-Applied-Guardrails: panw-airs:unscanned
-```
-
-This allows you to:
-- Track which requests bypassed scanning
-- Alert on unscanned request volumes
-- Audit compliance requirements
-
-#### Example: Masking Credit Card Numbers
-
-<Tabs>
-<TabItem label="Without Masking" value="no-mask">
-
-**Request:**
-```json
-{
-  "messages": [
-    {"role": "user", "content": "My credit card is 4929-3813-3266-4295"}
-  ]
-}
-```
-
-**Response:** ❌ **Blocked with 400 error**
-
-</TabItem>
-<TabItem label="With Masking" value="with-mask">
-
-**Request:**
-```json
-{
-  "messages": [
-    {"role": "user", "content": "My credit card is 4929-3813-3266-4295"}
-  ]
-}
-```
-
-**Masked prompt sent to LLM:**
-```json
-{
-  "messages": [
-    {"role": "user", "content": "My credit card is XXXXXXXXXXXXXXXXXX"}
-  ]
-}
-```
-
-**Response:** ✅ **Allowed with masked content**
-
-</TabItem>
-</Tabs>
-
-#### Masking Capabilities
-
-The guardrail masks sensitive content in:
-
-- ✅ **Chat messages** - User prompts and assistant responses
-- ✅ **Streaming responses** - Real-time masking of streamed content
-- ✅ **Multi-choice responses** - All choices in the response
-- ✅ **Tool/function calls** - Arguments passed to tools and functions
-- ✅ **Content lists** - Mixed content types (text, images, etc.)
-
-#### Complete Example
+### Custom Violation Messages
 
 ```yaml
 guardrails:
-  - guardrail_name: "panw-production-security"
+  - guardrail_name: "panw-custom-message"
     litellm_params:
       guardrail: panw_prisma_airs
-      mode: "post_call"                      # Scan input and output
       api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
-      profile_name: "production-profile"
-      mask_request_content: true             # Mask sensitive prompts
-      mask_response_content: true            # Mask sensitive responses
+      violation_message_template: "Your request was blocked by our AI Security Policy."
+
+  - guardrail_name: "panw-detailed-message"
+    litellm_params:
+      guardrail: panw_prisma_airs
+      api_key: os.environ/PANW_PRISMA_AIRS_API_KEY
+      violation_message_template: "{action_type} blocked due to {category} violation. Please contact support."
 ```
 
-## Use Cases
+**Supported Placeholders:** `{guardrail_name}`, `{category}`, `{action_type}`, `{default_message}`
 
-From [official Prisma AIRS documentation](https://docs.paloaltonetworks.com/ai-runtime-security/activation-and-onboarding/ai-runtime-security-api-intercept-overview):
+## Behavior and Limitations
 
-- **Secure AI models in production**: Validate prompt requests and responses to protect deployed AI models
-- **Detect data poisoning**: Identify contaminated training data before fine-tuning
-- **Protect against adversarial input**: Safeguard AI agents from malicious inputs and outputs
-- **Prevent sensitive data leakage**: Use API-based threat detection to block sensitive data leaks
+### Transaction Tracking
+
+For standard request/response scans, `tr_id` maps to `litellm_call_id`. MCP tool scans use the parent `litellm_call_id` when available; if missing, PANW synthesizes a fallback MCP transaction ID. The real limitation is correlation loss — synthesized MCP `tr_id` values are not grouped with the parent request's prompt/response scans in AIRS dashboards.
+
+By default, LiteLLM generates a UUID for `litellm_call_id`. To provide your own:
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -H "x-litellm-call-id: my-custom-call-id-789" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "capital of France"}],
+    "guardrails": ["panw-prisma-airs-guardrail"]
+  }'
+```
+
+The `x-litellm-call-id` is also returned in response headers. If you pass `litellm_trace_id` in request metadata (or via the `x-litellm-trace-id` header), it is included in the PANW API payload metadata but does not affect `tr_id` or appear in Prisma AIRS.
+
+### Streaming
+
+- Response masking works on OpenAI chat streaming (`mask_response_content: true`)
+- `/v1/messages` and `/v1/responses` raw streaming blocks instead of masking when violations are detected
+- Request-side masking (`mask_request_content`) is unaffected by endpoint type
+- When `fallback_on_error: "allow"` is set, streaming responses fail open on transient PANW API errors (timeout, 5xx, network) — original chunks are yielded unchanged
+
+## MCP Tool Security
+
+Tool invocations are sent to AIRS as structured `tool_event` payloads containing tool name, ecosystem, and serialized arguments. Tool-event scans always use request mode.
+
+**What is scanned:** LLM-driven `tool_calls` (name + arguments) and MCP request-side invocations when `mcp_tool_name` (or fallback `name`) is present. Response-side OpenAI-compatible `tool_calls` are also scanned when surfaced into `apply_guardrail()`.
+
+**What is not scanned:** Tool definitions in `inputs["tools"]` and post-MCP tool results (no `post_mcp_call` hook exists yet).
 
 
-## Next Steps
+### Current Limitations
 
-- Configure your security policies in [Strata Cloud Manager](https://apps.paloaltonetworks.com/)
-- Review the [Prisma AIRS API documentation](https://pan.dev/airs/) for advanced features
-- Set up monitoring and alerting for threat detections in your PANW dashboard
-- Consider implementing both pre_call and post_call guardrails for comprehensive protection
-- Monitor detection events and tune your security profiles based on your application needs
+- **No post-MCP response scanning.** Actual post-MCP tool-result scanning is not supported because there is no `post_mcp_call` hook in the framework. Response-side MCP events are only scanned when they appear as regular `tool_calls` in the LLM response.
+- **Guardrail selection not inherited by MCP sub-calls.** With `default_on: false`, MCP request-side child-call scans can be skipped because the parent request's guardrail selection is not propagated to the synthetic MCP payload. Workaround: use a dedicated guardrail with `mode: pre_mcp_call` and `default_on: true`.
+- **MCP transaction correlation.** MCP tool scans use the parent `litellm_call_id` when available; otherwise a fallback ID is synthesized and will not be grouped with the parent request in AIRS dashboards.

docs/my-website/docs/proxy/guardrails/policy_flow_builder.md

@@ -0,0 +1,219 @@
+# Policy Flow Builder
+
+The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail passes or fails.
+
+Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors).
+
+## When to use the Flow Builder
+
+| Approach | Use case |
+|----------|----------|
+| **Simple policy** (`guardrails.add`) | All guardrails run in parallel; any failure blocks the request. |
+| **Flow Builder** (pipeline) | Guardrails run in sequence; you choose actions per step (next, block, allow, custom response). |
+
+Use the Flow Builder when you need:
+
+- **Guardrail fallbacks** — use `on_fail: next` to try a different guardrail when one fails (e.g., fast filter → stricter filter)
+- **Retrying the same guardrail** — add the same guardrail as multiple steps; if it fails, `on_fail: next` moves to the next step, which can be the same guardrail again (useful for transient API errors or rate limits)
+- **Conditional routing** — e.g., if a fast guardrail fails, run a more advanced one instead of blocking immediately
+- **Custom responses** — return a specific message when a guardrail fails instead of a generic block
+- **Data chaining** — pass modified data (e.g., PII-masked content) from one step to the next
+- **Fine-grained control** — different actions on pass vs. fail per step
+
+## Concepts
+
+### Pipeline
+
+A pipeline has:
+
+- **Mode**: `pre_call` (before the LLM) or `post_call` (after the LLM)
+- **Steps**: Ordered list of guardrail steps
+
+### Step actions
+
+Each step defines what happens when the guardrail **passes** and when it **fails**:
+
+| Action | Description |
+|--------|-------------|
+| **Next Step** | Continue to the next guardrail in the pipeline |
+| **Allow** | Stop the pipeline and allow the request to proceed |
+| **Block** | Stop the pipeline and block the request |
+| **Custom Response** | Return a custom message instead of the default block |
+
+### Step options
+
+| Field | Type | Description |
+|-------|------|--------------|
+| `guardrail` | `string` | Name of the guardrail to run |
+| `on_pass` | `string` | Action when guardrail passes: `next`, `allow`, `block`, `modify_response` |
+| `on_fail` | `string` | Action when guardrail fails: `next`, `allow`, `block`, `modify_response` |
+| `pass_data` | `boolean` | Forward modified request data (e.g., PII-masked) to the next step |
+| `modify_response_message` | `string` | Custom message when using `modify_response` action |
+
+## Using the Flow Builder (UI)
+
+1. Go to **Policies** in the LiteLLM Admin UI
+2. Click **+ Create New Policy** or **Edit** on an existing policy
+3. Select **Flow Builder** (instead of the simple form)
+4. Design your flow:
+   - **Trigger** — Incoming LLM request (runs when the policy matches)
+   - **Steps** — Add guardrails, set ON PASS and ON FAIL actions per step
+   - **End** — Request proceeds to the LLM
+5. Use the **+** between steps to insert new steps
+6. Use the **Test** panel to run sample messages through the pipeline before saving
+7. Click **Save** to create or update the policy
+
+## Config (YAML)
+
+Define a pipeline in your policy config:
+
+```yaml showLineNumbers title="config.yaml"
+guardrails:
+  - guardrail_name: pii_masking
+    litellm_params:
+      guardrail: presidio
+      mode: pre_call
+
+  - guardrail_name: prompt_injection
+    litellm_params:
+      guardrail: lakera
+      mode: pre_call
+
+policies:
+  my-pipeline-policy:
+    description: "PII mask first, then check for prompt injection"
+    guardrails:
+      add:
+        - pii_masking
+        - prompt_injection
+    pipeline:
+      mode: pre_call
+      steps:
+        - guardrail: pii_masking
+          on_pass: next
+          on_fail: block
+          pass_data: true
+        - guardrail: prompt_injection
+          on_pass: allow
+          on_fail: block
+
+policy_attachments:
+  - policy: my-pipeline-policy
+    scope: "*"
+```
+
+## Fallbacks and retries
+
+### Guardrail fallbacks
+
+Use `on_fail: next` to fall back to another guardrail when one fails. Run a lightweight guardrail first; if it fails, escalate to a stricter or different provider:
+
+```yaml
+policies:
+  fallback-policy:
+    guardrails:
+      add:
+        - fast_content_filter
+        - strict_content_filter
+    pipeline:
+      mode: pre_call
+      steps:
+        - guardrail: fast_content_filter
+          on_pass: allow
+          on_fail: next
+        - guardrail: strict_content_filter
+          on_pass: allow
+          on_fail: block
+```
+
+If `fast_content_filter` passes → allow. If it fails → run `strict_content_filter`; pass → allow, fail → block.
+
+### Retrying the same guardrail
+
+Add the same guardrail as multiple steps to retry on failure. Useful for transient errors (API timeouts, rate limits):
+
+```yaml
+policies:
+  retry-policy:
+    guardrails:
+      add:
+        - lakera_prompt_injection
+    pipeline:
+      mode: pre_call
+      steps:
+        - guardrail: lakera_prompt_injection
+          on_pass: allow
+          on_fail: next
+        - guardrail: lakera_prompt_injection
+          on_pass: allow
+          on_fail: block
+```
+
+First attempt passes → allow. First attempt fails → retry the same guardrail; second pass → allow, second fail → block.
+
+## Example: Custom response on fail
+
+Return a branded message instead of a generic block:
+
+```yaml
+policies:
+  branded-block-policy:
+    guardrails:
+      add:
+        - pii_detector
+    pipeline:
+      mode: pre_call
+      steps:
+        - guardrail: pii_detector
+          on_pass: allow
+          on_fail: modify_response
+          modify_response_message: "Your message contains sensitive information. Please remove PII and try again."
+```
+
+## Test a pipeline (API)
+
+Test a pipeline with sample messages before attaching it:
+
+```bash
+curl -X POST "http://localhost:4000/policies/test-pipeline" \
+  -H "Authorization: Bearer <your_api_key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "pipeline": {
+      "mode": "pre_call",
+      "steps": [
+        {
+          "guardrail": "pii_masking",
+          "on_pass": "next",
+          "on_fail": "block",
+          "pass_data": true
+        },
+        {
+          "guardrail": "prompt_injection",
+          "on_pass": "allow",
+          "on_fail": "block"
+        }
+      ]
+    },
+    "test_messages": [
+      {"role": "user", "content": "What is 2+2?"},
+      {"role": "user", "content": "My SSN is 123-45-6789"}
+    ]
+  }'
+```
+
+Response includes per-step outcomes (pass/fail/error), actions taken, and timing.
+
+## Pipeline vs simple policy
+
+When a policy has a `pipeline`, the pipeline defines execution order and actions. The `guardrails.add` list must include all guardrails used in the pipeline steps.
+
+| Policy type | Execution |
+|-------------|-----------|
+| Simple (`guardrails.add` only) | All guardrails run; any failure blocks |
+| Pipeline (`pipeline` present) | Steps run in order; actions control flow |
+
+## Related docs
+
+- [Guardrail Policies](./guardrail_policies) — Policy basics, attachments, inheritance
+- [Policy Templates](./policy_templates) — Pre-built policy templates

docs/my-website/docs/proxy/guardrails/team_based_guardrails.md

@@ -1,6 +1,6 @@
 import Image from '@theme/IdealImage';
 
-# Team-Based Guardrails
+# Team Bring-Your-Own Guardrails
 
 Team-based guardrails let **developers** register a guardrail for their team via the API; an **admin** then reviews and approves or rejects it in the LiteLLM UI. Only [Generic Guardrail API](/docs/adding_provider/generic_guardrail_api) guardrails can be registered this way.
 

docs/my-website/docs/proxy/prometheus.md

@@ -561,26 +561,9 @@ Use these metrics to monitor the health of the DB Transaction Queue. Eg. Monitor
 | `litellm_in_memory_spend_update_queue_size`         | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory    |
 | `litellm_redis_spend_update_queue_size`             | Redis aggregate spend values for keys, users, teams, etc.                  | Redis        |
 
-#### DB Connection Pool and Engine Health Metrics
 
-Monitor PostgreSQL connection pool utilization and Prisma query engine health. These metrics are collected every 30 seconds by default.
 
-| Metric Name                              | Type    | Labels  | Description                                                |
-|------------------------------------------|---------|---------|-----------------------------------------------------------|
-| `litellm_db_pool_connections`            | Gauge   | `state` | Number of DB connections by state (active, idle, etc.)    |
-| `litellm_db_pool_lock_waiting_connections`    | Gauge   |         | Number of connections blocked on row/table locks           |
-| `litellm_db_engine_up`                   | Gauge   |         | Whether the Prisma query engine is alive (1=up, 0=down)   |
-| `litellm_db_engine_restarts_total`       | Counter |         | Total number of Prisma query engine restarts               |
-
-The `state` label values come from PostgreSQL's `pg_stat_activity.state` column: `active`, `idle`, `idle in transaction`, `idle in transaction (aborted)`, `fastpath function call`, `disabled`.
-
-**Prerequisites:** Metrics collection requires both:
-- `prometheus_system` in `service_callback` (see [Monitor System Health](#monitor-system-health))
-- `PRISMA_HEALTH_WATCHDOG_ENABLED` not set to `false` (default: `true`). If disabled, a warning is logged and no DB metrics are collected.
-
-The collection interval can be configured via the `PRISMA_METRICS_COLLECTION_INTERVAL_SECONDS` environment variable (default: 30, minimum: 5).
-
-## 🔥 LiteLLM Maintained Grafana Dashboards
+## 🔥 LiteLLM Maintained Grafana Dashboards 
 
 Link to Grafana Dashboards maintained by LiteLLM
 

docs/my-website/docs/proxy/realtime_webrtc.md

@@ -0,0 +1,84 @@
+# /realtime - WebRTC Support
+
+Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth; audio streams directly to OpenAI/Azure.
+
+**Providers:** OpenAI · Azure
+
+:::info **WebRTC vs WebSocket**
+- **WebSocket** (`/v1/realtime`) — server-to-server
+- **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) — browser/mobile, lower latency
+:::
+
+## How it works
+
+LiteLLM issues tokens and relays SDP; audio never passes through the proxy.
+
+```
+Browser                  LiteLLM Proxy              OpenAI/Azure
+  |                           |                          |
+  |-- POST client_secrets --->|-- POST sessions -------->|
+  |<-- encrypted_token -------|<-- ek_... ---------------|
+  |-- POST calls [SDP+token] ->|-- POST calls ----------->|
+  |<-- SDP answer ------------|<-- SDP answer -----------|
+  |===== audio P2P direct ===============================>|
+```
+
+## Proxy Setup
+
+```yaml
+model_list:
+  - model_name: gpt-4o-realtime
+    litellm_params:
+      model: openai/gpt-4o-realtime-preview-2024-12-17
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+```
+
+**Azure:** `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Client Usage
+
+1. **Token** — `POST /v1/realtime/client_secrets` with LiteLLM key and `{ model }`.
+2. **WebRTC** — Create `RTCPeerConnection`, add mic, data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <token>`, `Content-Type: application/sdp`.
+3. **Events** — Use data channel for `session.update` and other events.
+
+```javascript
+const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
+  method: "POST",
+  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
+  body: JSON.stringify({ model: "gpt-4o-realtime" }),
+});
+const token = (await r.json()).client_secret.value;
+
+const pc = new RTCPeerConnection();
+const audio = document.createElement("audio");
+audio.autoplay = true;
+pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
+const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
+pc.addTrack(ms.getTracks()[0]);
+const dc = pc.createDataChannel("oai-events");
+const offer = await pc.createOffer();
+await pc.setLocalDescription(offer);
+
+const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
+  method: "POST",
+  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
+  body: offer.sdp,
+});
+await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
+
+dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
+```
+
+## FAQ
+
+- **401 Token expired** — Get a fresh token right before creating the WebRTC offer.
+- **Which key for `/calls`?** — Encrypted token from `client_secrets`, not raw key.
+- **Pass `model`?** — No. Token encodes routing.
+- **Azure `api-version`** — Set `api_version` in `litellm_params` and correct `api_base`.
+- **No audio** — Grant mic; ensure `pc.ontrack` sets autoplay audio; check firewall/WebRTC; inspect console.

docs/my-website/docs/proxy/reliability.md

@@ -713,6 +713,34 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 [**See Code**](https://github.com/BerriAI/litellm/blob/c9e6b05cfb20dfb17272218e2555d6b496c47f6f/litellm/router.py#L2163)
 
+:::important
+**`enable_pre_call_checks` is required** for context-window enforcement. Without it, requests are sent to the provider regardless of input token count. Set `enable_pre_call_checks: true` in `router_settings` in your config.
+:::
+
+#### Custom max_input_tokens per deployment
+
+You can override the default context limit for a deployment by setting `max_input_tokens` in `model_info`. This is useful for testing, rate-limiting long prompts, or enforcing stricter limits than the provider's default.
+
+**Both** of the following are required:
+
+1. **`router_settings.enable_pre_call_checks: true`** — enables pre-call checks
+2. **`model_info.max_input_tokens`** on the deployment — overrides the limit for that model
+
+```yaml
+router_settings:
+  enable_pre_call_checks: true  # Required for enforcement
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      max_input_tokens: 10  # Override: reject prompts > 10 tokens
+```
+
+If a request exceeds the limit, LiteLLM raises `ContextWindowExceededError` with details like `Model=gpt-4o, Max Input Tokens=10, Got=306`.
+
 **1. Setup config**
 
 For azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with azure/.

docs/my-website/docs/proxy/tag_routing.md

@@ -209,6 +209,106 @@ Expect to see the following response header when this works
 x-litellm-model-id: default-model
 ```
 
+## Regex-based tag routing (`tag_regex`)
+
+Use `tag_regex` to route requests based on regex patterns matched against request headers, without requiring clients to pass a tag explicitly. This is useful when clients already send a recognisable header, such as `User-Agent`.
+
+**Use case: route all Claude Code traffic to dedicated AWS accounts**
+
+Claude Code always sends `User-Agent: claude-code/<version>`. With `tag_regex` you can route that traffic to a dedicated deployment automatically — no per-developer configuration needed.
+
+### 1. Config
+
+```yaml
+model_list:
+  # Claude Code traffic → dedicated deployment, matched by User-Agent
+  - model_name: claude-sonnet
+    litellm_params:
+      model: bedrock/converse/anthropic-claude-sonnet-4-6
+      aws_region_name: us-east-1
+      aws_role_name: arn:aws:iam::111122223333:role/LiteLLMClaudeCode
+      tag_regex:
+        - "^User-Agent: claude-code\\/"   # matches claude-code/1.x, 2.x, etc.
+    model_info:
+      id: claude-code-deployment
+
+  # All other traffic falls back to the default deployment
+  - model_name: claude-sonnet
+    litellm_params:
+      model: bedrock/converse/anthropic-claude-sonnet-4-6
+      aws_region_name: us-east-1
+      aws_role_name: arn:aws:iam::444455556666:role/LiteLLMDefault
+      tags:
+        - default
+    model_info:
+      id: regular-deployment
+
+router_settings:
+  enable_tag_filtering: true
+  tag_filtering_match_any: true
+
+general_settings:
+  master_key: sk-1234
+```
+
+### 2. Verify routing
+
+Claude Code sets `User-Agent: claude-code/<version>` automatically — no client config needed:
+
+```shell
+# Claude Code request (User-Agent set automatically by Claude Code)
+curl http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -H "User-Agent: claude-code/1.2.3" \
+  -d '{"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]}'
+# → x-litellm-model-id: claude-code-deployment
+
+# Any other client (no matching User-Agent) → default deployment
+curl http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]}'
+# → x-litellm-model-id: regular-deployment
+```
+
+### How matching works
+
+| Priority | Condition | Result |
+|----------|-----------|--------|
+| 1 | Request has `tags` AND deployment has `tags` | Exact tag match (respects `match_any` setting) |
+| 2 | Deployment has `tag_regex` AND request has a `User-Agent` | Regex match (always OR logic — any pattern match suffices) |
+| 3 | Deployment has `tags: [default]` | Default fallback |
+| 4 | No default set | All healthy deployments returned |
+
+`tag_regex` always uses OR semantics — `tag_filtering_match_any=False` applies only to exact tag matching, not to regex patterns.
+
+### Observability
+
+When a regex matches, `tag_routing` is written into request metadata and flows to SpendLogs:
+
+```json
+{
+  "tag_routing": {
+    "matched_via": "tag_regex",
+    "matched_value": "^User-Agent: claude-code\\/",
+    "user_agent": "claude-code/1.2.3",
+    "request_tags": []
+  }
+}
+```
+
+### Security note
+
+:::caution
+
+**`User-Agent` is a client-supplied header and can be set to any value.** Any API consumer can send `User-Agent: claude-code/1.0` regardless of whether they are actually using Claude Code.
+
+Do not rely on `tag_regex` routing to enforce access controls or spend limits — use [team/key-based routing](./users) for that. `tag_regex` is a **traffic classification hint** (useful for billing visibility, capacity planning, and routing convenience), not a security boundary.
+
+:::
+
+
+---
+
 ## ✨ Team based tag routing (Enterprise)
 
 LiteLLM Proxy supports team-based tag routing, allowing you to associate specific tags with teams and route requests accordingly. Example **Team A can access gpt-4 deployment A, Team B can access gpt-4 deployment B** (LLM Access Control For Teams)

docs/my-website/docs/proxy/team_budgets.md

@@ -177,3 +177,7 @@ Expect to see this metric on prometheus to track the Remaining Budget for the te
 ```shell
 litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06
 ```
+
+## See Also
+
+- [Per-model TPM/RPM for teams](./users.md#per-team-model) - Set rate limits per model for all keys in a team

docs/my-website/docs/proxy/ui/ui_edit_logo.md

@@ -0,0 +1,138 @@
+import Image from '@theme/IdealImage';
+
+# Customize UI Logo
+
+Personalize your LiteLLM dashboard by replacing the default logo with your own company branding. You can set a custom logo via the UI or the API.
+
+## Via the UI
+
+### 1. Navigate to Settings
+
+Click the **Settings** icon in the sidebar.
+
+![Navigate to Settings](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/57a15404-51f7-481e-9db2-cea94566d3ce/ascreenshot_7a348567c839448bb806fd71cf4abca0_text_export.jpeg)
+
+### 2. Open UI Theme Settings
+
+Click **UI Theme** from the settings menu.
+
+![Open UI Theme](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/30663fe1-9f78-4496-96d4-c53513cbaf82/ascreenshot_ac1eb59eda0e423fbd0e7d3a6cabd4c7_text_export.jpeg)
+
+### 3. Click the Logo URL Field
+
+Click the **Logo URL** text field to start editing.
+
+![Click Logo URL Field](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/069e8412-8ec1-4d36-ba38-6b2e2858a45a/ascreenshot_8fc7fb4a3af74815bc1b69a8554bc110_text_export.jpeg)
+
+### 4. Find Your Logo Image
+
+Open a new browser tab and find the logo image you want to use (e.g., search Google Images for your company logo).
+
+![Find Logo Image](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/d9b55dac-bc4e-4728-b422-4afbc21f9034/ascreenshot_2a805f39c83d4b5e95f43495a6ea4e79_text_export.jpeg)
+
+### 5. Right-Click on the Logo Image
+
+Right-click the image you want to use as your logo.
+
+![Right-Click Image](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/9d42d13e-6028-4710-acb2-c6af04a855c7/ascreenshot_0f21f29ba0e44132afe483a4b88e8b70_text_export.jpeg)
+
+### 6. Copy the Image Address
+
+Select **Copy Image Address** from the context menu to copy the URL.
+
+![Copy Image Address](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/c25637be-383a-498b-ad11-eb1761d52757/ascreenshot_b237ee800979462189a02c1e1942ebf1_text_export.jpeg)
+
+### 7. Switch Back to LiteLLM
+
+Navigate back to the LiteLLM UI tab (e.g., press **Cmd + Left** or click the tab).
+
+![Switch Back](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/f0647856-679c-4591-9ff7-7fd3cfbc70b4/ascreenshot_3ce46dae64c94891ac0983f5ed8f085a_text_export.jpeg)
+
+### 8. Paste the Logo URL
+
+Paste the copied image URL into the **Logo URL** field with **Cmd + V**.
+
+![Paste URL](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/54dd30d9-7a88-41e8-a580-a6acf707c7fa/ascreenshot_8a772218ac0743d9ae8ffd3311eccd5a_text_export.jpeg)
+
+### 9. Save Changes
+
+Click **Save Changes** to apply your new logo.
+
+![Save Changes](https://colony-recorder.s3.amazonaws.com/files/2026-03-13/4baf6494-d146-4600-b6f2-ef667338d580/ascreenshot_722cbcd568ec4267af5122b3958bb248_text_export.jpeg)
+
+Your custom logo will now appear in the LiteLLM dashboard sidebar and login page.
+
+## Via the API
+
+### Set a Custom Logo
+
+```bash
+curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \
+  -H 'Authorization: Bearer <your-admin-key>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "logo_url": "https://example.com/your-company-logo.png"
+  }'
+```
+
+### Set a Custom Favicon
+
+You can also customize the browser tab favicon:
+
+```bash
+curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \
+  -H 'Authorization: Bearer <your-admin-key>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "logo_url": "https://example.com/your-company-logo.png",
+    "favicon_url": "https://example.com/your-favicon.ico"
+  }'
+```
+
+### Get Current Theme Settings
+
+```bash
+curl -X GET 'http://localhost:4000/settings/get/ui_theme_settings'
+```
+
+### Reset to Default Logo
+
+Send an empty `logo_url` to restore the default LiteLLM logo:
+
+```bash
+curl -X PATCH 'http://localhost:4000/settings/update/ui_theme_settings' \
+  -H 'Authorization: Bearer <your-admin-key>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "logo_url": ""
+  }'
+```
+
+## Via `proxy_config.yaml`
+
+You can also set the logo URL in your proxy configuration file:
+
+```yaml
+litellm_settings:
+  ui_theme_config:
+    logo_url: "https://example.com/your-company-logo.png"
+    favicon_url: "https://example.com/your-favicon.ico"  # optional
+```
+
+Or set it as an environment variable:
+
+```yaml
+environment_variables:
+  UI_LOGO_PATH: "https://example.com/your-company-logo.png"
+```
+
+## Supported Logo Formats
+
+| Format | Supported |
+|--------|-----------|
+| JPEG / JPG | Yes |
+| PNG | Yes |
+| SVG | Yes |
+| ICO (favicon only) | Yes |
+| HTTP/HTTPS URL | Yes |
+| Local file path | Yes |

docs/my-website/docs/proxy/users.md

@@ -641,7 +641,7 @@ You can set:
 - tpm limits (tokens per minute)
 - rpm limits (requests per minute)
 - max parallel requests
-- rpm / tpm limits per model for a given key
+- rpm / tpm limits per model for a given key or team
 
 ### TPM Rate Limit Type (Input/Output/Total)
 
@@ -689,6 +689,62 @@ curl --location 'http://0.0.0.0:4000/team/new' \
 }
 ```
 
+</TabItem>
+<TabItem value="per-team-model" label="Per Team Per Model">
+
+**Set rate limits per model for a team**
+
+Use `model_rpm_limit` and `model_tpm_limit` to set rate limits per model for all keys belonging to a team. These limits apply across all keys in the team and are inherited by keys unless overridden at the key level.
+
+Use `/team/new` or `/team/update` with `model_rpm_limit` and `model_tpm_limit` as dictionaries mapping model names to their limits:
+
+```shell
+curl --location 'http://0.0.0.0:4000/team/new' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "team_id": "my-prod-team",
+  "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200},
+  "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000}
+}'
+```
+
+**Update existing team with per-model limits:**
+
+```shell
+curl --location 'http://0.0.0.0:4000/team/update' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "team_id": "my-prod-team",
+  "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200},
+  "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000}
+}'
+```
+
+**Alternative: Use metadata**
+
+You can also pass per-model limits via the `metadata` field:
+
+```shell
+curl --location 'http://0.0.0.0:4000/team/update' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "team_id": "my-prod-team",
+  "metadata": {
+    "model_rpm_limit": {"gpt-4": 100, "gpt-3.5-turbo": 200},
+    "model_tpm_limit": {"gpt-4": 10000, "gpt-3.5-turbo": 20000}
+  }
+}'
+```
+
+**Resolution order:** When a key belongs to a team, rate limits are resolved as: **Key metadata > Key model_max_budget > Team metadata**. Keys can override team-level per-model limits with their own `model_rpm_limit` or `model_tpm_limit`.
+
+**Verify:** Make a `/chat/completions` request and check response headers `x-litellm-key-remaining-requests-{model}` and `x-litellm-key-remaining-tokens-{model}` for the model-specific limits.
+
+[**See Swagger**](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post)
+
 </TabItem>
 <TabItem value="per-user" label="Per Internal User">
 

docs/my-website/docs/reasoning_content.md

@@ -592,6 +592,14 @@ Expected Response
 </TabItem>
 </Tabs>
 
+:::tip gpt-5.4: reasoning_effort + function tools
+
+LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API.
+
+If you need reasoning **and** tools together, use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
+
+:::
+
 ## OpenAI Responses API - Auto-Summary Control
 
 When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter.

docs/my-website/docs/troubleshoot/pip_venv_upgrade.md

@@ -0,0 +1,121 @@
+# Upgrading LiteLLM Proxy (pip/venv)
+
+Guide for upgrading LiteLLM Proxy when installed via pip in a virtual environment.
+
+:::info Important
+Always activate your virtual environment before running any `litellm` or `prisma` commands. All commands in this guide assume you're working inside an activated venv.
+:::
+
+## How pip/venv Upgrades Work
+
+There are two pieces that need to stay in sync:
+
+1. **Prisma client** - Generated Python code that talks to the DB
+2. **DB schema** - Tables/columns in PostgreSQL
+
+When you upgrade via pip, the `litellm-proxy-extras` package ships with a new `schema.prisma` and a `migrations/` directory. But unlike the Docker image, pip install does NOT automatically regenerate the Prisma client or run migrations. You have to do both manually.
+
+## Upgrade Workflow (pip/venv)
+
+### 1. Stop the proxy
+
+Stop your running LiteLLM proxy instance.
+
+### 2. (Optional) Back up your DB
+
+```bash
+pg_dump -h <host> -U <user> -d <db> -F c -f backup_$(date +%Y%m%d).dump
+```
+
+### 3. Upgrade the package
+
+```bash
+pip install 'litellm[proxy]==<version>'
+```
+
+### 4. Regenerate the Prisma client
+
+```bash
+prisma generate --schema <venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma
+```
+
+Replace `<venv>` with your virtual environment path and `<version>` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`).
+
+### 5. Apply DB migrations
+
+You have two options:
+
+**Option A: Just start the proxy** (simplest)
+
+The proxy automatically runs `prisma migrate deploy` on startup, which applies any new migrations.
+
+First, activate your virtual environment:
+
+```bash
+source <venv>/bin/activate
+```
+
+Then start the proxy:
+
+```bash
+litellm --config your_config.yaml --port 4000
+```
+
+**Option B: Run manually before starting**
+
+Activate your virtual environment first:
+
+```bash
+source <venv>/bin/activate
+```
+
+Then run the migration with the explicit schema path:
+
+```bash
+prisma migrate deploy --schema <venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma
+```
+
+Replace `<venv>` with your virtual environment path and `<version>` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`).
+
+### 6. Start the proxy
+
+If you used Option B above, now start the proxy (with venv still activated):
+
+```bash
+litellm --config your_config.yaml --port 4000
+```
+
+## How to Verify Migrations
+
+> **Note:** `<schema-path>` = `<venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma`
+
+### Before applying migrations: Preview what will change
+
+Run `pip install 'litellm[proxy]==<version>'` first (Step 3) so the new `schema.prisma` is available.
+
+```bash
+prisma migrate diff \
+  --from-url $DATABASE_URL \
+  --to-schema-datamodel <schema-path> \
+  --script
+```
+
+### After applying migrations: Check status
+
+```bash
+prisma migrate status --schema <schema-path>
+```
+
+All migrations should have a `finished_at` timestamp and no `rolled_back_at`.
+
+## Key Things to Know
+
+- **`DISABLE_SCHEMA_UPDATE=true`** env var prevents auto-migration on startup - useful if you want full manual control
+
+- **`prisma db push`** is the nuclear option: force-syncs the DB to match the schema, bypassing migration history. Safe when all changes are additive (new columns/tables), but always have a backup.
+
+- **The `schema.prisma` inside `litellm_proxy_extras` is the source of truth** - always use that one, not one from a different version or from the git repo
+
+## Troubleshooting
+
+If you encounter migration errors, see the [Prisma Migration Troubleshooting Guide](./prisma_migrations).

docs/my-website/docs/tutorials/claude_code_skills.md

@@ -0,0 +1,99 @@
+# LiteLLM Skills
+
+[litellm-skills](https://github.com/BerriAI/litellm-skills) is a collection of [Agent Skills](https://agentskills.io) for managing a live LiteLLM proxy. Install them once and any agent that supports the Agent Skills standard (Claude Code, OpenCode, OpenClaw, etc.) can create users, teams, keys, models, MCP servers, agents, and query usage — all by running `curl` commands against your proxy.
+
+## Install
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm-skills/main/install.sh | sh
+```
+
+## Requirements
+
+- `curl` installed
+- A running LiteLLM proxy (local or remote)
+- A proxy admin key — not a virtual key scoped to `llm_api_routes`
+
+## Available Skills
+
+### Users
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-user` | Create a user — email, role, budget, model access |
+| `/update-user` | Update budget, role, or models for an existing user |
+| `/delete-user` | Delete one or more users |
+
+### Teams
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-team` | Create a team with budget and model limits |
+| `/update-team` | Update budget, models, or rate limits |
+| `/delete-team` | Delete one or more teams |
+
+### API Keys
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-key` | Generate a key scoped to a user, team, budget, and expiry |
+| `/update-key` | Update budget, models, or expiry |
+| `/delete-key` | Delete by key value or alias |
+
+### Organizations
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-org` | Create an org with budget and model access |
+| `/delete-org` | Delete one or more orgs |
+
+### Models
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-model` | Add any provider (OpenAI, Azure, Anthropic, Bedrock, Ollama…) and test it |
+| `/update-model` | Rotate credentials or swap the underlying deployment |
+| `/delete-model` | Remove a model |
+
+### MCP Servers
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-mcp` | Register an MCP server (SSE, HTTP, or stdio) |
+| `/update-mcp` | Update URL, credentials, or allowed tools |
+| `/delete-mcp` | Remove an MCP server |
+
+### Agents
+
+| Skill | What it does |
+|-------|-------------|
+| `/add-agent` | Create an agent backed by a model and optional MCP servers |
+| `/update-agent` | Swap the model or update description and limits |
+| `/delete-agent` | Remove an agent |
+
+### Usage
+
+| Skill | What it does |
+|-------|-------------|
+| `/view-usage` | Daily spend and token activity — by user, team, org, or model |
+
+## How it works
+
+When you invoke a skill, the agent asks for your `LITELLM_BASE_URL` and admin key, collects the fields needed for that operation, runs the `curl`, and shows the result. For example:
+
+```
+/add-model
+```
+→ Agent asks: provider, public name, credentials. Adds the model, runs a test completion, reports pass/fail.
+
+```
+/view-usage
+```
+→ Agent asks: date range (defaults to current month), optional team/model filter. Prints a table of daily requests, tokens, and spend.
+
+## Related
+
+- [litellm-skills on GitHub](https://github.com/BerriAI/litellm-skills)
+- [Virtual Keys](../proxy/virtual_keys.md) — managing API keys on the proxy
+- [Team-based routing](../proxy/team_based_routing.md) — setting up teams
+- [Model Management](../proxy/model_management.md) — adding models via config or API

docs/my-website/docs/tutorials/retool_assist.md

@@ -0,0 +1,143 @@
+import Image from '@theme/IdealImage';
+
+# Retool Assist
+
+This guide walks you through connecting [Retool Assist](https://docs.retool.com/apps/guides/assist/) to LiteLLM Proxy. Retool Assist uses AI to generate and edit apps from within the Retool app IDE. Using LiteLLM with Retool Assist allows you to:
+
+- Access 100+ LLMs through Retool Assist
+- Track spend and usage, set budget limits per virtual key
+- Control which models Retool Assist can access
+- Use your own LLM providers via a unified OpenAI-compatible API
+
+<div style={{ maxWidth: '100%', overflow: 'hidden', paddingBottom: '59.52%', position: 'relative', height: 0 }}>
+  <iframe 
+    style={{ position: 'absolute', top: 0, left: 0, width: '100%', height: '100%', maxWidth: '840px' }}
+    src="https://www.youtube.com/embed/aN-Iua5dHGg" 
+    frameborder="0" 
+    webkitallowfullscreen 
+    mozallowfullscreen 
+    allowfullscreen
+  ></iframe>
+</div>
+
+---
+
+:::info
+**Hosted Retool requires a public URL.** Retool Cloud runs on Retool's servers, so `localhost` will not work. You must expose your LiteLLM proxy via ngrok, Cloudflare Tunnel, or by deploying to a cloud provider.
+:::
+
+## Quick Reference
+
+| Setting | Value |
+|---------|-------|
+| Provider Schema | OpenAI |
+| Base URL | Your ngrok URL (e.g. `https://abc123.ngrok-free.app`) or deployed proxy URL |
+| API Key | Your LiteLLM Virtual Key |
+| Model | Public model name from LiteLLM (e.g. `openai/gpt-4o-mini`, `openai/gpt-5.2-2025-12-11`) |
+
+---
+
+## Prerequisites
+
+- LiteLLM Proxy running locally or deployed
+- [ngrok](https://ngrok.com/download) (or similar tunnel) for local development with hosted Retool
+- A [Retool](https://retool.com) account (Cloud or self-hosted)
+
+## 1. Start LiteLLM Proxy
+
+Set up LiteLLM Proxy following the [Getting Started Guide](https://docs.litellm.ai/docs/proxy/docker_quick_start). Ensure your proxy is running on port 4000.
+
+## 2. Expose LiteLLM with a Public URL
+
+<Image img={require('../../img/ngrok_public_url.gif')} />
+
+Retool Cloud runs on Retool's servers. You must expose your local LiteLLM proxy with a public URL.
+
+### Using ngrok
+
+- Install [ngrok](https://ngrok.com/download)
+- In a separate terminal, run:
+  
+```bash
+ngrok http 4000
+```
+- Copy the generated HTTPS URL (e.g. `https://abc123.ngrok-free.app`). This is your **Base URL** for Retool.
+
+
+### Alternative
+
+If you deploy LiteLLM to Railway, Render, Fly.io, or another cloud provider, use that public URL as your Base URL. See the [Deploy guide](https://docs.litellm.ai/docs/proxy/deploy) for details.
+
+## 3. Generate a Virtual Key
+
+<Image img={require('../../img/litellm_virtual_key.gif')} />
+
+Create a virtual key that Retool Assist will use to authenticate with LiteLLM. The key must have access to the models you want to use (e.g. `openai/*` for all OpenAI models).
+
+### Via LiteLLM UI
+
+- Navigate to [http://localhost:4000/ui](http://localhost:4000/ui)
+- Go to **Virtual Keys** → **+ Create New Key**
+- Select the models you need (or `openai/*` for all OpenAI models)
+- Copy the key
+
+## 4. Add LiteLLM as a Custom Provider in Retool
+
+Inside your Retool dashboard, configure LiteLLM as a custom AI resource:
+
+<Image img={require('../../img/retool_resource_setup.gif')} />
+
+1. Go to **Resources**
+
+2. Under the **AI** category, select **Custom Provider**
+
+3. Fill in the form:
+   - **Name:** `LiteLLM`
+   - **Description:** (optional) e.g. `LiteLLM Proxy - 100+ LLMs`
+   - **Provider Schema:** `OpenAI`
+   - **Base URL:** Your ngrok-generated URL (e.g. `https://abc123.ngrok-free.app`) or deployed proxy URL—do not add `/v1` unless Retool requires it
+   - **API Key:** Your LiteLLM virtual key from Step 3
+4. **Add model names** from your LiteLLM proxy (e.g. `openai/gpt-4o-mini`, `openai/gpt-5.2-2025-12-11`).
+5. Click **Create Resource**
+
+<Image img={require('../../img/retool_llm_setup.gif')} />
+
+## 5. Test the Connection
+
+<Image img={require('../../img/retool_litellm_connection.gif')} />
+
+- Open an app in Retool and enable **Assist** (if not already enabled in your organization)
+- Use Assist to generate or edit app elements, it will route requests through LiteLLM
+- Use the code option from the Sidebar to add a resource query, select the LiteLLM resource, and run it to test the setup.
+- Check the LiteLLM **Logs** section to verify requests and track usage
+
+<Image img={require('../../img/retool_litellm_logs.gif')} />
+
+---
+
+## Troubleshooting
+
+### 401 Unauthorized
+
+- Ensure the **API Key** in Retool matches your LiteLLM virtual key exactly
+- Verify the key is not expired or blocked in LiteLLM
+
+### 401 "key not allowed to access model"
+
+Your virtual key is restricted to specific models. Generate a new key with `openai/*` or include the model you need (e.g. `openai/gpt-5.2-2025-12-11`) in the key's allowed models list.
+
+### 500 "api_key client option must be set"
+
+LiteLLM could not use your OpenAI API key to call the provider. Ensure `OPENAI_API_KEY` is set in your LiteLLM environment (e.g. in `.env` or `docker-compose.yml`) when using `openai/*` models.
+
+### localhost does not work
+
+Retool Cloud cannot reach `localhost` it points to Retool's servers. Use ngrok or deploy LiteLLM to a public URL.
+
+---
+
+## Additional Resources
+
+- [Virtual Keys](https://docs.litellm.ai/docs/proxy/virtual_keys) – Create and manage API keys
+- [Deploy LiteLLM](https://docs.litellm.ai/docs/proxy/deploy) – Production deployment options
+- [Retool Assist Documentation](https://docs.retool.com/apps/guides/assist/) – Configure Assist and prompting guides

docs/my-website/docs/videos.md

@@ -290,6 +290,82 @@ curl --location 'http://localhost:4000/v1/videos' \
 --header 'custom-llm-provider: azure'
 ```
 
+### Character, Edit, and Extension Endpoints
+
+LiteLLM proxy also supports these OpenAI-compatible video routes:
+
+- `POST /v1/videos/characters`
+- `GET /v1/videos/characters/{character_id}`
+- `POST /v1/videos/edits`
+- `POST /v1/videos/extensions`
+
+#### Routing Behavior (`target_model_names`, encoded IDs, and provider overrides)
+
+- `POST /v1/videos/characters` supports `target_model_names` like `POST /v1/videos`.
+- When `target_model_names` is provided on character creation, LiteLLM encodes the returned `character_id` with routing metadata.
+- `GET /v1/videos/characters/{character_id}` accepts encoded character IDs directly. LiteLLM decodes the ID internally and routes with the correct model/provider metadata.
+- `POST /v1/videos/edits` and `POST /v1/videos/extensions` support both:
+  - plain `video.id`
+  - encoded `video.id` values returned by LiteLLM
+- `custom_llm_provider` can be supplied using the same patterns as other proxy endpoints:
+  - header: `custom-llm-provider`
+  - query: `?custom_llm_provider=...`
+  - body: `custom_llm_provider` (or `extra_body.custom_llm_provider` where applicable)
+
+#### Character create with `target_model_names`
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/characters' \
+--header 'Authorization: Bearer sk-1234' \
+-F 'name=hero' \
+-F 'target_model_names=gpt-4' \
+-F 'video=@/path/to/character.mp4'
+```
+
+Example response (encoded `id`):
+
+```json
+{
+  "id": "character_...",
+  "object": "character",
+  "created_at": 1712697600,
+  "name": "hero"
+}
+```
+
+#### Get character using encoded `character_id`
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/characters/character_...' \
+--header 'Authorization: Bearer sk-1234'
+```
+
+#### Video edit with encoded `video.id`
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/edits' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "prompt": "Make this brighter",
+  "video": { "id": "video_..." }
+}'
+```
+
+#### Video extension with provider override from `extra_body`
+
+```bash
+curl --location 'http://localhost:4000/v1/videos/extensions' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{
+  "prompt": "Continue this scene",
+  "seconds": "4",
+  "video": { "id": "video_..." },
+  "extra_body": { "custom_llm_provider": "openai" }
+}'
+```
+
 Test Azure video generation request
 
 ```bash

docs/my-website/release_notes/v1.81.9.md

@@ -279,7 +279,7 @@ Let's dive in.
     - Add HTTP support to custom code guardrails + Unified guardrails for MCP + Agent guardrail support - [PR #20619](https://github.com/BerriAI/litellm/pull/20619)
     - Custom Code Guardrails UI Playground - [PR #20377](https://github.com/BerriAI/litellm/pull/20377)
 
-- **Team-Based Guardrails**
+- **Team Bring-Your-Own Guardrails**
     - Implement team-based isolation guardrails management - [PR #20318](https://github.com/BerriAI/litellm/pull/20318)
 
 - **[OpenAI Moderations](../../docs/apply_guardrail)**

docs/my-website/release_notes/v1.82.0.md

@@ -1,5 +1,5 @@
 ---
-title: "[Preview] v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
+title: "v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
 slug: "v1-82-0"
 date: 2026-02-28T00:00:00
 authors:
@@ -26,7 +26,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-ghcr.io/berriai/litellm:main-1.82.0
+ghcr.io/berriai/litellm:main-1.82.0-stable
 ```
 
 </TabItem>

docs/my-website/sidebars.js

@@ -100,6 +100,7 @@ const sidebars = {
       label: "Policies",
       items: [
         "proxy/guardrails/guardrail_policies",
+        "proxy/guardrails/policy_flow_builder",
         "proxy/guardrails/policy_templates",
         "proxy/guardrails/policy_tags",
       ],
@@ -172,7 +173,8 @@ const sidebars = {
         "tutorials/litellm_gemini_cli",
         "tutorials/google_genai_sdk",
         "tutorials/litellm_qwen_code_cli",
-        "tutorials/openai_codex"
+        "tutorials/openai_codex",
+        "tutorials/retool_assist"
       ]
     },
     {
@@ -193,6 +195,19 @@ const sidebars = {
         "projects/openai-agents"
       ]
     },
+    {
+      type: "category",
+      label: "Manage with AI Agents",
+      link: {
+        type: "generated-index",
+        title: "Manage with AI Agents",
+        description: "Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language.",
+        slug: "/manage_with_ai_agents"
+      },
+      items: [
+        "tutorials/claude_code_skills",
+      ]
+    },
 
   ],
   // But you can create a sidebar manually
@@ -330,6 +345,7 @@ const sidebars = {
               label: "Setup & SSO",
               items: [
                 "proxy/admin_ui_sso",
+                "proxy/ui/ui_edit_logo",
                 "proxy/custom_sso",
                 "proxy/custom_root_ui",
                 "tutorials/scim_litellm",
@@ -614,6 +630,7 @@ const sidebars = {
             "mcp_usage",
             "mcp_openapi",
             "mcp_oauth",
+            "mcp_aws_sigv4",
             "mcp_public_internet",
             "mcp_semantic_filter",
             "mcp_control",
@@ -666,6 +683,7 @@ const sidebars = {
         "rag_ingest",
         "rag_query",
         "realtime",
+        "proxy/realtime_webrtc",
         "rerank",
         "response_api",
         "response_api_compact",
@@ -813,6 +831,8 @@ const sidebars = {
         "providers/anyscale",
         "providers/apertis",
         "providers/baseten",
+        "providers/black_forest_labs",
+        "providers/black_forest_labs_img_edit",
         "providers/bytez",
         "providers/cerebras",
         "providers/chutes",
@@ -1156,6 +1176,7 @@ const sidebars = {
             "troubleshoot/prisma_migrations",
           ],
         },
+        "troubleshoot/pip_venv_upgrade",
         "troubleshoot/rollback",
         "troubleshoot",
       ],

docs/my-website/src/components/WebRTCTester.jsx

@@ -0,0 +1,83 @@
+import DashboardWebRTCTester from "../../../../ui/litellm-dashboard/src/components/WebRTCTester.jsx";
+
+const LIGHT_MODE_OVERRIDES = `
+.wrt-wrap {
+  background: #1f2937;
+  border: 1px solid #334155;
+}
+.wrt-toggle,
+.wrt-toggle:hover {
+  background: #111827;
+}
+.wrt-toggle-title,
+.we-msg {
+  color: #e2e8f0;
+}
+.wrt-toggle-sub,
+.wrt-label,
+.wrt-field label,
+.wrt-flow-box,
+.wrt-flow-arrow,
+.wrt-meta-row span:first-child,
+.wrt-header-title,
+.wrt-tab,
+.we-time {
+  color: #94a3b8;
+}
+.wrt-body,
+.wrt-sidebar,
+.wrt-main,
+.wrt-header,
+.wrt-tabs,
+.wrt-sdp-box,
+.wrt-sdp-hdr,
+.wrt-divider {
+  border-color: #334155;
+}
+.wrt-header {
+  background: #111827;
+}
+.wrt-field input,
+.wrt-mic-btn,
+.wrt-status-pill {
+  background: #0b1220;
+  border-color: #334155;
+  color: #e2e8f0;
+}
+.wrt-field input:focus,
+.wrt-btn-ghost:hover {
+  border-color: #60a5fa;
+}
+.wrt-btn-ghost {
+  background: #0b1220;
+  border-color: #334155;
+  color: #e2e8f0;
+}
+.wrt-log::-webkit-scrollbar-thumb {
+  background: #475569;
+}
+.wrt-tab.active {
+  color: #93c5fd;
+  border-bottom-color: #93c5fd;
+}
+.wrt-empty,
+.wrt-audio-status,
+.wrt-meta-row span:last-child {
+  color: #cbd5e1;
+}
+.wrt-sdp-dot {
+  background: #475569;
+}
+.wrt-sdp-pane textarea {
+  color: #e2e8f0;
+}
+`;
+
+export default function WebRTCTester() {
+  return (
+    <>
+      <DashboardWebRTCTester />
+      <style>{LIGHT_MODE_OVERRIDES}</style>
+    </>
+  );
+}

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -2,11 +2,15 @@
 Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
 """
 
-from litellm._uuid import uuid
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Optional
 
 from litellm._logging import verbose_proxy_logger
+from litellm._uuid import uuid
+from litellm.constants import (
+    MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
+    MAX_OBJECTS_PER_POLL_CYCLE,
+)
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -29,6 +33,9 @@ class CheckBatchCost:
         self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
         self.prisma_client: PrismaClient = prisma_client
         self.llm_router: Router = llm_router
+        # Cached after the first poll cycle. Once we know the column is absent we skip
+        # the guaranteed-failing primary query on every subsequent cycle.
+        self._has_batch_processed_column: bool = True
 
     async def _get_user_info(self, batch_id, user_id) -> dict:
         """
@@ -49,6 +56,47 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
+    async def _cleanup_stale_managed_objects(self) -> None:
+        """
+        Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
+        in non-terminal states as 'stale_expired'. These will never complete and
+        should not be polled.
+        """
+        cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
+        result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            where={
+                "file_purpose": "batch",
+                "status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
+                "created_at": {"lt": cutoff},
+            },
+            data={"status": "stale_expired"},
+        )
+        if result > 0:
+            verbose_proxy_logger.warning(
+                f"CheckBatchCost: marked {result} stale managed objects "
+                f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
+            )
+
+    async def _fallback_find_jobs(self) -> list:
+        """Query batch jobs without the batch_processed filter (for older schemas)."""
+        return await self.prisma_client.db.litellm_managedobjecttable.find_many(
+            where={
+                "file_purpose": "batch",
+                "status": {
+                    "not_in": [
+                        "failed",
+                        "expired",
+                        "cancelled",
+                        "complete",
+                        "completed",
+                        "stale_expired",
+                    ]
+                },
+            },
+            take=MAX_OBJECTS_PER_POLL_CYCLE,
+            order={"created_at": "asc"},
+        )
+
     async def check_batch_cost(self):
         """
         Check if the batch JOB has been tracked.
@@ -70,14 +118,50 @@ class CheckBatchCost:
             get_model_id_from_unified_batch_id,
         )
 
-        # Look for all batches that have not yet been processed by CheckBatchCost
-        jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
-            where={
-                "file_purpose": "batch",
-                "batch_processed" : False,
-                "status": {"not_in": ["failed", "expired", "cancelled"]}
-            }
-        )
+        try:
+            await self._cleanup_stale_managed_objects()
+        except Exception as cleanup_err:
+            verbose_proxy_logger.warning(
+                f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}"
+            )
+
+        # Look for all batches that have not yet been processed by CheckBatchCost.
+        # self._has_batch_processed_column is cached after the first probe so that
+        # older schemas don't pay a guaranteed-failing primary query + warning on
+        # every subsequent poll cycle.
+        if self._has_batch_processed_column:
+            try:
+                # Include "complete"/"completed" batches: the retrieve_batch
+                # endpoint may transition a batch to "complete" before
+                # CheckBatchCost runs.  The batch_processed=False filter
+                # already prevents reprocessing finished batches.
+                jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
+                    where={
+                        "file_purpose": "batch",
+                        "batch_processed": False,
+                        "status": {
+                            "not_in": [
+                                "failed",
+                                "expired",
+                                "cancelled",
+                                "stale_expired",
+                            ]
+                        },
+                    },
+                    take=MAX_OBJECTS_PER_POLL_CYCLE,
+                    order={"created_at": "asc"},
+                )
+            except Exception as query_err:
+                if "batch_processed" not in str(query_err).lower() and "unknown column" not in str(query_err).lower() and "does not exist" not in str(query_err).lower():
+                    raise
+                # Permanent schema gap — cache the result so future cycles skip straight to fallback
+                self._has_batch_processed_column = False
+                verbose_proxy_logger.warning(
+                    "CheckBatchCost: batch_processed column not found, querying without it"
+                )
+                jobs = await self._fallback_find_jobs()
+        else:
+            jobs = await self._fallback_find_jobs()
         for job in jobs:
             # get the model from the job
             unified_object_id = job.unified_object_id
@@ -163,14 +247,14 @@ class CheckBatchCost:
 
                 # Access content - handle both direct attribute and method call
                 if hasattr(_file_content, 'content'):
-                    content_bytes = _file_content.content
+                    content_bytes = _file_content.content  # type: ignore[union-attr]
                 elif hasattr(_file_content, 'read'):
-                    content_bytes = await _file_content.read()
+                    content_bytes = await _file_content.read()  # type: ignore[misc]
                 else:
-                    content_bytes = _file_content
+                    content_bytes = _file_content  # type: ignore[assignment]
 
                 file_content_as_dict = _get_file_content_as_dictionary(
-                    content_bytes
+                    content_bytes  # type: ignore[arg-type]
                 )
 
                 deployment_info = self.llm_router.get_deployment(model_id=model_id)
@@ -195,7 +279,7 @@ class CheckBatchCost:
                         file_content_dictionary=file_content_as_dict,
                         custom_llm_provider=llm_provider,  # type: ignore
                         model_name=model_name,
-                        model_info=deployment_model_info,
+                        model_info=deployment_model_info,  # type: ignore[arg-type]
                     )
                 )
                 logging_obj = LiteLLMLogging(
@@ -236,13 +320,15 @@ class CheckBatchCost:
 
                 # mark the job as complete
                 try:
+                    update_data: dict = {
+                        "status": "complete",
+                        "file_object": response.model_dump_json(),
+                    }
+                    if self._has_batch_processed_column:
+                        update_data["batch_processed"] = True
                     await self.prisma_client.db.litellm_managedobjecttable.update(
                         where={"id": job.id},
-                        data={
-                            "batch_processed": True,
-                            "status": "complete",
-                            "file_object": response.model_dump_json(),
-                        },
+                        data=update_data,
                     )
                 except Exception as db_err:
                     verbose_proxy_logger.error(

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -3,10 +3,15 @@ Polls LiteLLM_ManagedObjectTable to check if the response is complete.
 Cost tracking is handled automatically by litellm.aget_responses().
 """
 
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import (
+    MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
+    MAX_OBJECTS_PER_POLL_CYCLE,
+)
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -27,6 +32,27 @@ class CheckResponsesCost:
         self.prisma_client: PrismaClient = prisma_client
         self.llm_router: Router = llm_router
 
+    async def _cleanup_stale_managed_objects(self) -> None:
+        """
+        Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
+        in non-terminal states as 'stale_expired'. These will never complete and
+        should not be polled.
+        """
+        cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
+        result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            where={
+                "file_purpose": "response",
+                "status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
+                "created_at": {"lt": cutoff},
+            },
+            data={"status": "stale_expired"},
+        )
+        if result > 0:
+            verbose_proxy_logger.warning(
+                f"CheckResponsesCost: marked {result} stale managed objects "
+                f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
+            )
+
     async def check_responses_cost(self):
         """
         Check if background responses are complete and track their cost.
@@ -35,11 +61,20 @@ class CheckResponsesCost:
         - Cost is automatically tracked by litellm.aget_responses()
         - Mark completed/failed/cancelled responses as complete in the database
         """
+        try:
+            await self._cleanup_stale_managed_objects()
+        except Exception as cleanup_err:
+            verbose_proxy_logger.warning(
+                f"CheckResponsesCost: stale cleanup failed (poll will continue): {cleanup_err}"
+            )
+
         jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
             where={
                 "status": {"in": ["queued", "in_progress"]},
                 "file_purpose": "response",
-            }
+            },
+            take=MAX_OBJECTS_PER_POLL_CYCLE,
+            order={"created_at": "asc"},
         )
         
         verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -26,6 +26,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     get_batch_id_from_unified_batch_id,
     get_content_type_from_file_object,
     get_model_id_from_unified_batch_id,
+    get_models_from_unified_file_id,
     normalize_mime_type_for_provider,
 )
 from litellm.types.llms.openai import (
@@ -904,6 +905,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )  # managed batch id
             model_id = cast(Optional[str], response._hidden_params.get("model_id"))
             model_name = cast(Optional[str], response._hidden_params.get("model_name"))
+            resolved_model_name = model_name
+
+            # Some providers (e.g. Vertex batch retrieve) do not set model_name on
+            # the response. In that case, recover target_model_names from the input
+            # managed file metadata so unified output IDs preserve routing metadata.
+            if not resolved_model_name and isinstance(unified_file_id, str):
+                decoded_unified_file_id = (
+                    _is_base64_encoded_unified_file_id(unified_file_id)
+                    or unified_file_id
+                )
+                target_model_names = get_models_from_unified_file_id(
+                    decoded_unified_file_id
+                )
+                if target_model_names:
+                    resolved_model_name = ",".join(target_model_names)
             original_response_id = response.id
 
             if (unified_batch_id or unified_file_id) and model_id:
@@ -919,7 +935,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         unified_file_id = self.get_unified_output_file_id(
                             output_file_id=original_file_id,
                             model_id=model_id,
-                            model_name=model_name,
+                            model_name=resolved_model_name,
                         )
                         setattr(response, file_attr, unified_file_id)
                         

litellm-js/spend-logs/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "dependencies": {
         "@hono/node-server": "^1.10.1",
-        "hono": "^4.10.3"
+        "hono": "^4.12.7"
       },
       "devDependencies": {
         "@types/node": "^20.11.17",
@@ -548,9 +548,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.10.6",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.10.6.tgz",
-      "integrity": "sha512-BIdolzGpDO9MQ4nu3AUuDwHZZ+KViNm+EZ75Ae55eMXMqLVhDFqEMXxtUe9Qh8hjL+pIna/frs2j6Y2yD5Ua/g==",
+      "version": "4.12.7",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.7.tgz",
+      "integrity": "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"

litellm-js/spend-logs/package.json

@@ -4,7 +4,7 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.10.1",
-    "hono": "^4.10.3"
+    "hono": "^4.12.7"
   },
   "devDependencies": {
     "@types/node": "^20.11.17",

litellm-proxy-extras/litellm_proxy_extras/migrations/20250326162113_baseline/migration.sql

@@ -1,5 +1,5 @@
 -- CreateTable
-CREATE TABLE "LiteLLM_BudgetTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetTable" (
     "budget_id" TEXT NOT NULL,
     "max_budget" DOUBLE PRECISION,
     "soft_budget" DOUBLE PRECISION,
@@ -18,7 +18,7 @@ CREATE TABLE "LiteLLM_BudgetTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_CredentialsTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_CredentialsTable" (
     "credential_id" TEXT NOT NULL,
     "credential_name" TEXT NOT NULL,
     "credential_values" JSONB NOT NULL,
@@ -32,7 +32,7 @@ CREATE TABLE "LiteLLM_CredentialsTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_ProxyModelTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_ProxyModelTable" (
     "model_id" TEXT NOT NULL,
     "model_name" TEXT NOT NULL,
     "litellm_params" JSONB NOT NULL,
@@ -46,7 +46,7 @@ CREATE TABLE "LiteLLM_ProxyModelTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_OrganizationTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationTable" (
     "organization_id" TEXT NOT NULL,
     "organization_alias" TEXT NOT NULL,
     "budget_id" TEXT NOT NULL,
@@ -63,7 +63,7 @@ CREATE TABLE "LiteLLM_OrganizationTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_ModelTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_ModelTable" (
     "id" SERIAL NOT NULL,
     "aliases" JSONB,
     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -75,7 +75,7 @@ CREATE TABLE "LiteLLM_ModelTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_TeamTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_TeamTable" (
     "team_id" TEXT NOT NULL,
     "team_alias" TEXT,
     "organization_id" TEXT,
@@ -102,7 +102,7 @@ CREATE TABLE "LiteLLM_TeamTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_UserTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_UserTable" (
     "user_id" TEXT NOT NULL,
     "user_alias" TEXT,
     "team_id" TEXT,
@@ -131,7 +131,7 @@ CREATE TABLE "LiteLLM_UserTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_VerificationToken" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" (
     "token" TEXT NOT NULL,
     "key_name" TEXT,
     "key_alias" TEXT,
@@ -166,7 +166,7 @@ CREATE TABLE "LiteLLM_VerificationToken" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_EndUserTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_EndUserTable" (
     "user_id" TEXT NOT NULL,
     "alias" TEXT,
     "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
@@ -179,7 +179,7 @@ CREATE TABLE "LiteLLM_EndUserTable" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_Config" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_Config" (
     "param_name" TEXT NOT NULL,
     "param_value" JSONB,
 
@@ -187,7 +187,7 @@ CREATE TABLE "LiteLLM_Config" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_SpendLogs" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogs" (
     "request_id" TEXT NOT NULL,
     "call_type" TEXT NOT NULL,
     "api_key" TEXT NOT NULL DEFAULT '',
@@ -218,7 +218,7 @@ CREATE TABLE "LiteLLM_SpendLogs" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_ErrorLogs" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_ErrorLogs" (
     "request_id" TEXT NOT NULL,
     "startTime" TIMESTAMP(3) NOT NULL,
     "endTime" TIMESTAMP(3) NOT NULL,
@@ -235,7 +235,7 @@ CREATE TABLE "LiteLLM_ErrorLogs" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_UserNotifications" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_UserNotifications" (
     "request_id" TEXT NOT NULL,
     "user_id" TEXT NOT NULL,
     "models" TEXT[],
@@ -246,7 +246,7 @@ CREATE TABLE "LiteLLM_UserNotifications" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_TeamMembership" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_TeamMembership" (
     "user_id" TEXT NOT NULL,
     "team_id" TEXT NOT NULL,
     "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
@@ -256,7 +256,7 @@ CREATE TABLE "LiteLLM_TeamMembership" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_OrganizationMembership" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationMembership" (
     "user_id" TEXT NOT NULL,
     "organization_id" TEXT NOT NULL,
     "user_role" TEXT,
@@ -269,7 +269,7 @@ CREATE TABLE "LiteLLM_OrganizationMembership" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_InvitationLink" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_InvitationLink" (
     "id" TEXT NOT NULL,
     "user_id" TEXT NOT NULL,
     "is_accepted" BOOLEAN NOT NULL DEFAULT false,
@@ -284,7 +284,7 @@ CREATE TABLE "LiteLLM_InvitationLink" (
 );
 
 -- CreateTable
-CREATE TABLE "LiteLLM_AuditLog" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_AuditLog" (
     "id" TEXT NOT NULL,
     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     "changed_by" TEXT NOT NULL DEFAULT '',
@@ -299,62 +299,132 @@ CREATE TABLE "LiteLLM_AuditLog" (
 );
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_CredentialsTable_credential_name_key" ON "LiteLLM_CredentialsTable"("credential_name");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_CredentialsTable_credential_name_key" ON "LiteLLM_CredentialsTable"("credential_name");
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_TeamTable_model_id_key" ON "LiteLLM_TeamTable"("model_id");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_TeamTable_model_id_key" ON "LiteLLM_TeamTable"("model_id");
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_UserTable_sso_user_id_key" ON "LiteLLM_UserTable"("sso_user_id");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_UserTable_sso_user_id_key" ON "LiteLLM_UserTable"("sso_user_id");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_SpendLogs_startTime_idx" ON "LiteLLM_SpendLogs"("startTime");
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_startTime_idx" ON "LiteLLM_SpendLogs"("startTime");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_SpendLogs_end_user_idx" ON "LiteLLM_SpendLogs"("end_user");
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_end_user_idx" ON "LiteLLM_SpendLogs"("end_user");
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_OrganizationMembership_user_id_organization_id_key" ON "LiteLLM_OrganizationMembership"("user_id", "organization_id");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_OrganizationMembership_user_id_organization_id_key" ON "LiteLLM_OrganizationMembership"("user_id", "organization_id");
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_OrganizationTable" ADD CONSTRAINT "LiteLLM_OrganizationTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_OrganizationTable_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_OrganizationTable" ADD CONSTRAINT "LiteLLM_OrganizationTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_TeamTable" ADD CONSTRAINT "LiteLLM_TeamTable_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_TeamTable_organization_id_fkey') THEN
+        ALTER TABLE "LiteLLM_TeamTable" ADD CONSTRAINT "LiteLLM_TeamTable_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_TeamTable" ADD CONSTRAINT "LiteLLM_TeamTable_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "LiteLLM_ModelTable"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_TeamTable_model_id_fkey') THEN
+        ALTER TABLE "LiteLLM_TeamTable" ADD CONSTRAINT "LiteLLM_TeamTable_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "LiteLLM_ModelTable"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_UserTable" ADD CONSTRAINT "LiteLLM_UserTable_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_UserTable_organization_id_fkey') THEN
+        ALTER TABLE "LiteLLM_UserTable" ADD CONSTRAINT "LiteLLM_UserTable_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "LiteLLM_VerificationToken_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_VerificationToken_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "LiteLLM_VerificationToken_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "LiteLLM_VerificationToken_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_VerificationToken_organization_id_fkey') THEN
+        ALTER TABLE "LiteLLM_VerificationToken" ADD CONSTRAINT "LiteLLM_VerificationToken_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_EndUserTable" ADD CONSTRAINT "LiteLLM_EndUserTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_EndUserTable_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_EndUserTable" ADD CONSTRAINT "LiteLLM_EndUserTable_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_TeamMembership" ADD CONSTRAINT "LiteLLM_TeamMembership_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_TeamMembership_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_TeamMembership" ADD CONSTRAINT "LiteLLM_TeamMembership_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_OrganizationMembership_user_id_fkey') THEN
+        ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_OrganizationMembership_organization_id_fkey') THEN
+        ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "LiteLLM_OrganizationTable"("organization_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_OrganizationMembership_budget_id_fkey') THEN
+        ALTER TABLE "LiteLLM_OrganizationMembership" ADD CONSTRAINT "LiteLLM_OrganizationMembership_budget_id_fkey" FOREIGN KEY ("budget_id") REFERENCES "LiteLLM_BudgetTable"("budget_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_InvitationLink_user_id_fkey') THEN
+        ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_InvitationLink_created_by_fkey') THEN
+        ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 
 -- AddForeignKey
-ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_InvitationLink_updated_by_fkey') THEN
+        ALTER TABLE "LiteLLM_InvitationLink" ADD CONSTRAINT "LiteLLM_InvitationLink_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+    END IF;
+END $$;
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20250326171002_add_daily_user_table/migration.sql

@@ -1,5 +1,5 @@
 -- CreateTable
-CREATE TABLE "LiteLLM_DailyUserSpend" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyUserSpend" (
     "id" TEXT NOT NULL,
     "user_id" TEXT NOT NULL,
     "date" TEXT NOT NULL,
@@ -17,17 +17,17 @@ CREATE TABLE "LiteLLM_DailyUserSpend" (
 );
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_DailyUserSpend_date_idx" ON "LiteLLM_DailyUserSpend"("date");
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyUserSpend_date_idx" ON "LiteLLM_DailyUserSpend"("date");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_DailyUserSpend_user_id_idx" ON "LiteLLM_DailyUserSpend"("user_id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyUserSpend_user_id_idx" ON "LiteLLM_DailyUserSpend"("user_id");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_DailyUserSpend_api_key_idx" ON "LiteLLM_DailyUserSpend"("api_key");
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyUserSpend_api_key_idx" ON "LiteLLM_DailyUserSpend"("api_key");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_DailyUserSpend_model_idx" ON "LiteLLM_DailyUserSpend"("model");
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyUserSpend_model_idx" ON "LiteLLM_DailyUserSpend"("model");
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyUserSpend"("user_id", "date", "api_key", "model", "custom_llm_provider");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyUserSpend"("user_id", "date", "api_key", "model", "custom_llm_provider");
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20250327180120_add_api_requests_to_daily_user_table/migration.sql

@@ -1,3 +1,3 @@
 -- AlterTable
-ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN     "api_requests" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "api_requests" INTEGER NOT NULL DEFAULT 0;
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20250329084805_new_cron_job_table/migration.sql

@@ -2,7 +2,7 @@
 CREATE TYPE "JobStatus" AS ENUM ('ACTIVE', 'INACTIVE');
 
 -- CreateTable
-CREATE TABLE "LiteLLM_CronJob" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_CronJob" (
     "cronjob_id" TEXT NOT NULL,
     "pod_id" TEXT NOT NULL,
     "status" "JobStatus" NOT NULL DEFAULT 'INACTIVE',

litellm-proxy-extras/litellm_proxy_extras/migrations/20250331215456_track_success_and_failed_requests_daily_agg_table/migration.sql

@@ -1,4 +1,4 @@
 -- AlterTable
-ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN     "failed_requests" INTEGER NOT NULL DEFAULT 0,
-ADD COLUMN     "successful_requests" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "failed_requests" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN IF NOT EXISTS "successful_requests" INTEGER NOT NULL DEFAULT 0;
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20250411215431_add_managed_file_table/migration.sql

@@ -1,5 +1,5 @@
 -- CreateTable
-CREATE TABLE "LiteLLM_ManagedFileTable" (
+CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedFileTable" (
     "id" TEXT NOT NULL,
     "unified_file_id" TEXT NOT NULL,
     "file_object" JSONB NOT NULL,
@@ -11,8 +11,8 @@ CREATE TABLE "LiteLLM_ManagedFileTable" (
 );
 
 -- CreateIndex
-CREATE UNIQUE INDEX "LiteLLM_ManagedFileTable_unified_file_id_key" ON "LiteLLM_ManagedFileTable"("unified_file_id");
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_unified_file_id_key" ON "LiteLLM_ManagedFileTable"("unified_file_id");
 
 -- CreateIndex
-CREATE INDEX "LiteLLM_ManagedFileTable_unified_file_id_idx" ON "LiteLLM_ManagedFileTable"("unified_file_id");
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_unified_file_id_idx" ON "LiteLLM_ManagedFileTable"("unified_file_id");