scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

merge: resolve conflicts with main

Browse source
This commit is contained in:
Ishaan Jaffer 2026-03-10 16:27:45 -07:00
commit c2c44d993c
1028 changed files with 36914 additions and 10286 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.circleci/config.yml
View file

@ -69,9 +69,11 @@ jobs:
- run:
name: Install Python
command: |
choco install python --version=3.11.0 -y
choco install python --version=3.11.0 -y --no-progress --force
refreshenv
python --version
environment:
CHOCOLATEY_CONFIRM_ALL: "true"
- run:
name: Install Dependencies
command: |

24
.github/workflows/publish_enterprise.yml vendored
View file

@ -19,6 +19,7 @@ jobs:
if: github.repository == 'BerriAI/litellm'
permissions:
contents: write
pull-requests: write
defaults:
run:
working-directory: enterprise
@ -56,14 +57,33 @@ jobs:
- name: Build
run: poetry build
- name: Commit version bump
- name: Commit version bump and create PR
id: create-pr
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
cd ..
BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
git checkout -b "$BRANCH"
git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
git push
git push origin "$BRANCH" --force
gh pr create \
--title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
--body "Version bump for litellm-enterprise. Merge to update main." \
--head "$BRANCH" \
--base main \
|| true
PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ github.token }}
- name: Enable auto-merge
run: |
gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
env:
GH_TOKEN: ${{ github.token }}
- name: Publish to PyPI
env:

2
AGENTS.md
View file

@ -251,9 +251,11 @@ The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
- Black `--check` may report pre-existing formatting issues; this does not block test runs.
- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
### Lint

13
Dockerfile
View file

@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@ -70,7 +70,15 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
# SECURITY FIX: patch npm's own package.json metadata so scanners see the
# actual installed versions instead of the stale declared dependencies.
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
# Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
# no longer visible to image scanners. The globally installed npm@latest
# at /usr/local/lib/node_modules/npm/ remains fully functional.
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -96,6 +104,7 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

2
ci_cd/security_scans.sh
View file

@ -161,6 +161,8 @@ run_grype_scans() {
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
)
# Build JSON array of allowlisted CVE IDs for jq

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -13,6 +13,10 @@ spec:
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
{{- with .Values.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "litellm.selectorLabels" . | nindent 6 }}

8
deploy/charts/litellm-helm/values.yaml
View file

@ -35,6 +35,14 @@ deploymentLabels: {}
podAnnotations: {}
podLabels: {}
# -- Deployment strategy configuration
# Example:
# type: RollingUpdate
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
strategy: {}
terminationGracePeriodSeconds: 90
topologySpreadConstraints:
[]

7
docker/Dockerfile.custom_ui
View file

@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -36,7 +36,10 @@ RUN apt-get update && apt-get upgrade -y \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
apt-get purge -y npm
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

8
docker/Dockerfile.database
View file

@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -67,7 +67,10 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
npm cache clean --force
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
{ apk del --no-cache npm 2>/dev/null || true; }
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -85,6 +88,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

8
docker/Dockerfile.dev
View file

@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -92,7 +92,10 @@ RUN apt-get update && apt-get upgrade -y \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& apt-get purge -y npm
WORKDIR /app
@ -114,6 +117,7 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

8
docker/Dockerfile.non_root
View file

@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
&& npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -123,7 +123,10 @@ RUN for i in 1 2 3; do \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& npm cache clean --force
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& { apk del --no-cache npm 2>/dev/null || true; }
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -169,6 +172,7 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
[ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \

1
docs/my-website/docs/a2a.md
View file

@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
| Logging | ✅ |
| Load Balancing | ✅ |
| Streaming | ✅ |
| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
:::tip

252
docs/my-website/docs/a2a_agent_headers.md Normal file
View file

@ -0,0 +1,252 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# A2A Agent Authentication Headers
Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
## Overview
When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
| Method | Who configures | How it works |
|---|---|---|
| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
All three methods can be combined. **Static headers always win** on key conflicts.
---
## Method 1 — Static Headers
Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"Authorization": "Bearer internal-server-token",
"X-Internal-Service": "litellm-proxy"
}
}'
```
To update an existing agent:
```bash
curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"static_headers": {
"Authorization": "Bearer new-token"
}
}'
```
</TabItem>
</Tabs>
**Client call — no special headers needed:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "Content-Type: application/json" \
-d '{
"jsonrpc": "2.0", "id": "1", "method": "message/send",
"params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
}'
```
The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
---
## Method 2 — Forward Client Headers
Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the LiteLLM dashboard.
2. Create or edit an agent.
3. Open the **Authentication Headers** panel.
4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
</TabItem>
<TabItem value="api" label="REST API">
```bash
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"extra_headers": ["x-api-key", "x-user-token"]
}'
```
</TabItem>
</Tabs>
**Client call — include the forwarded headers:**
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-api-key: user-secret-value" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives `x-api-key: user-secret-value`.
:::note
Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
:::
---
## Method 3 — Convention-Based Forwarding
Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
```
x-a2a-{agent_name_or_id}-{header_name}: value
```
LiteLLM parses these headers automatically and routes them to the matching agent only.
**Examples:**
| Client header sent | Agent name/ID | Forwarded as |
|---|---|---|
| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
```bash
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
:::tip Matches both agent name and agent ID
Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
:::
---
## Merge Precedence
When multiple methods supply the same header name, **static headers win**:
```
dynamic (forwarded/convention) → merged ← static (overlays, wins)
```
Example:
| Source | `Authorization` value |
|---|---|
| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
| Admin-configured `static_headers` | `Bearer server-token` |
| **What the backend agent receives** | **`Bearer server-token`** |
This ensures admin-controlled credentials cannot be overridden by client requests.
---
## Combining All Three Methods
```bash
# Register agent with static + forwarded headers
curl -X POST http://localhost:4000/v1/agents \
-H "Authorization: Bearer sk-admin" \
-H "Content-Type: application/json" \
-d '{
"agent_name": "my-agent",
"agent_card_params": { ... },
"static_headers": {
"X-Internal-Token": "secret123"
},
"extra_headers": ["x-user-id"]
}'
# Client call using all three mechanisms
curl -X POST http://localhost:4000/a2a/my-agent \
-H "Authorization: Bearer sk-client-key" \
-H "x-user-id: user-42" \
-H "x-a2a-my-agent-x-request-id: req-abc" \
-H "Content-Type: application/json" \
-d '{ ... }'
```
The backend agent receives:
```
X-Internal-Token: secret123 ← static header (always)
x-user-id: user-42 ← forwarded (in extra_headers)
x-request-id: req-abc ← convention-based (x-a2a-my-agent-*)
X-LiteLLM-Trace-Id: <uuid> ← LiteLLM internal
X-LiteLLM-Agent-Id: <agent-id> ← LiteLLM internal
```
---
## Header Isolation
Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
---
## API Reference
### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
| Field | Type | Description |
|---|---|---|
| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
| `extra_headers` | `string[]` | Header names to extract from client request and forward |
### Agent Response
Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
```json
{
"agent_id": "...",
"agent_name": "my-agent",
"static_headers": { "X-Internal-Token": "secret123" },
"extra_headers": ["x-user-id"],
...
}
```
:::caution
`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
:::

188
docs/my-website/docs/a2a_iteration_budgets.md Normal file
View file

@ -0,0 +1,188 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Agent Iteration Budgets
Control runaway costs from agentic loops with per-session iteration and budget caps.
## Overview
When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
| Control | Description |
|---------|-------------|
| **Max Iterations** | Hard cap on the number of LLM calls per session |
| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
## Trace-ID Enforcement
LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
| Flag | Description |
|------|-------------|
| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
## Configuring via UI
When creating an agent in the LiteLLM Admin UI:
1. Navigate to the **Agents** tab and click **Add Agent**
2. In the **Agent Settings** step, expand the **Tracing** section
3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
4. Set **Max Iterations** to cap the number of LLM calls per session
5. Set **Max Budget Per Session ($)** to cap spend per session
The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
## Configuring via API
Set trace-id enforcement on the agent itself:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_to_agent": true,
"require_trace_id_on_calls_by_agent": true
}
}'
```
Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true,
"max_iterations": 25,
"max_budget_per_session": 5.00
}
}'
```
## How It Works
### Session Tracking
Callers identify their session by including a `session_id` in one of these ways:
- **Header**: `x-litellm-trace-id: my-session-123`
- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
### Max Iterations
When `max_iterations` is set in agent `litellm_params`:
- Each LLM call for a session increments a counter
- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
### Max Budget Per Session
When `max_budget_per_session` is set in agent `litellm_params`:
- After each successful LLM call, the response cost is accumulated for the session
- Before each call, the accumulated spend is checked against the budget
- When spend exceeds the budget, the request receives a **429 Too Many Requests**
- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
## Example
Create an agent with max 25 iterations and a $5 budget cap:
<Tabs>
<TabItem value="ui" label="Via UI">
1. Go to **Agents** → **Add Agent**
2. Configure your agent (name, model, etc.)
3. In **Agent Settings**, expand the **Tracing** section
4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
5. Set **Max Iterations** to `25`
6. Set **Max Budget Per Session** to `5.00`
7. Proceed to create a new key for the agent
8. Click **Create Agent**
</TabItem>
<TabItem value="api" label="Via API">
```bash
# 1. Create the agent with trace-id enforcement
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent with budget controls",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true
}
}'
# 2. Create a key for the agent
curl -X POST 'http://localhost:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_id": "<agent_id_from_step_1>",
"key_alias": "my-research-agent-key"
}'
```
</TabItem>
</Tabs>
### Making Calls with Session Tracking
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Authorization: Bearer sk-agent-key-xxx' \
-H 'x-litellm-trace-id: session-abc-123' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
After 25 calls or $5 spent within this session, subsequent requests will receive:
```json
{
"error": {
"message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
"type": "budget_exceeded",
"code": 429
}
}
```
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

68
docs/my-website/docs/mcp.md
View file

@ -217,6 +217,7 @@ mcp_servers:
| `bearer_token` | `Authorization: Bearer <auth_value>` |
| `basic` | `Authorization: Basic <auth_value>` |
| `authorization` | `Authorization: <auth_value>` |
| `aws_sigv4` | Per-request AWS SigV4 signature ([details](./mcp_aws_sigv4.md)) |
- **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
- **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server.
@ -257,6 +258,16 @@ mcp_servers:
auth_type: "authorization"
auth_value: "Token example123" # headers={"Authorization": "Token example123"}
# AWS SigV4 for Bedrock AgentCore MCP servers
agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
aws_service_name: bedrock-agentcore
# Example with extra headers forwarding
github_mcp:
url: "https://api.githubcopilot.com/mcp"
@ -704,6 +715,63 @@ asyncio.run(main())
[Learn more about customer management →](./proxy/customers)
## Calling the Proxy's /v1/responses Endpoint
When calling your LiteLLM Proxy's `/v1/responses` endpoint to use MCP tools, **always use `server_url: "litellm_proxy"`** in the tools array. This tells the proxy to use its configured MCP servers.
:::important Do not use the full proxy URL
Using `server_url: "https://your-proxy.com/mcp"` is incorrect when the request is already going to the proxy. The proxy needs the literal value `litellm_proxy` to route to its configured MCP servers.
:::
```bash title="Correct: Using litellm_proxy" showLineNumbers
curl --location 'https://your-proxy.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $LITELLM_API_KEY" \
--data '{
"model": "gpt-4",
"tools": [
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
}
],
"input": "Run available tools",
"tool_choice": "required"
}'
```
### Sending Custom Headers to MCP Servers
To pass custom headers (e.g., API keys, auth tokens) to specific MCP servers, use either:
**Option 1: Request headers** – Add `x-mcp-{server_alias}-{header_name}` to your request headers. The proxy forwards these to the matching MCP server.
```bash
# Send Authorization header to the "weather2" MCP server
--header 'x-mcp-weather2-authorization: Bearer your-token'
# Send custom header to the "github" MCP server
--header 'x-mcp-github-x-api-key: your-api-key'
```
**Option 2: Headers in tool config** – Include a `headers` object in the tool definition. These are merged with request headers.
```json
{
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
"x-mcp-servers": "Zapier_MCP,dev-group",
"x-mcp-weather2-authorization": "Bearer your-weather-api-token"
}
}
```
## Using your MCP with client side credentials
Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

144
docs/my-website/docs/mcp_aws_sigv4.md Normal file
View file

@ -0,0 +1,144 @@
# MCP - AWS SigV4 Auth
Use AWS SigV4 authentication to connect LiteLLM to MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html).
## Why SigV4?
AWS services authenticate requests using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) — a per-request signing protocol that includes the request body in the cryptographic signature. This is fundamentally different from static-header auth types (`api_key`, `bearer_token`, etc.) which send the same header on every request.
LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP request is signed with your AWS credentials before it's sent.
## Quick Start
### 1. Set AWS credentials
```bash
export AWS_ACCESS_KEY_ID="AKIA..."
export AWS_SECRET_ACCESS_KEY="..."
export AWS_REGION_NAME="us-east-1"
```
### 2. Add your AgentCore MCP server to config.yaml
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
:::info URL encoding
The AgentCore runtime ARN must be URL-encoded in the `url` field. For example:
```
arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/my-mcp-server
```
becomes:
```
arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fmy-mcp-server
```
:::
### 3. Start the proxy
```bash
litellm --config config.yaml
```
### 4. Use the MCP tools
Once started, your AgentCore MCP tools are available through LiteLLM like any other MCP server:
```bash title="List available tools"
curl http://localhost:4000/mcp-rest/tools/list \
-H "Authorization: Bearer sk-1234"
```
```bash title="Call a tool"
curl http://localhost:4000/mcp-rest/tools/call \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"name": "my_agentcore_mcp_your_tool_name",
"arguments": {"key": "value"}
}'
```
## Config Reference
| Field | Required | Description |
|-------|----------|-------------|
| `url` | Yes | AgentCore MCP server URL (with URL-encoded ARN) |
| `transport` | Yes | Must be `"http"` |
| `auth_type` | Yes | Must be `"aws_sigv4"` |
| `aws_access_key_id` | No | AWS access key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
| `aws_secret_access_key` | No | AWS secret key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
## How It Works
LiteLLM uses an `httpx.Auth` subclass (`MCPSigV4Auth`) that hooks into the HTTP request lifecycle:
1. For every outgoing MCP request, the auth handler computes a SHA-256 hash of the request body
2. It creates a SigV4 signature using your AWS credentials, the request URL, headers, and body hash
3. The signed `Authorization` and `x-amz-date` headers are added to the request
4. AWS validates the signature and processes the MCP request
This happens transparently — no manual token management required.
## Using Temporary Credentials (STS)
If you use AWS STS temporary credentials (e.g., from IAM roles or SSO), include the session token:
```yaml title="config.yaml with STS credentials" showLineNumbers
mcp_servers:
my_agentcore_mcp:
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
transport: "http"
auth_type: "aws_sigv4"
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_session_token: os.environ/AWS_SESSION_TOKEN
aws_region_name: "us-east-1"
aws_service_name: "bedrock-agentcore"
```
## Troubleshooting
### 403 Forbidden from AWS
- Verify your AWS credentials are valid and not expired
- Check that `aws_region_name` matches the region in your AgentCore URL
- Ensure `aws_service_name` is set to `bedrock-agentcore`
- If using STS credentials, confirm `aws_session_token` is set and not expired
### Health check errors on startup
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
### "botocore not found" error
Install the `botocore` package:
```bash
pip install botocore
```
`botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth.

8
docs/my-website/docs/mcp_control.md
View file

@ -323,7 +323,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@ -335,7 +335,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This example uses URL namespacing to access all servers in the "dev_group" access group.
This example uses the `x-mcp-servers` header to access all servers in the "dev_group" access group. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint—do not use the full proxy URL.
</TabItem>
@ -423,7 +423,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/mcp/",
"server_url": "litellm_proxy",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
@ -436,7 +436,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
}'
```
This configuration restricts the request to only use tools from the specified MCP servers.
This configuration restricts the request to only use tools from the specified MCP servers. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint.
</TabItem>

72
docs/my-website/docs/providers/azure_ai/azure_model_router.md
View file

@ -2,6 +2,32 @@
Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
## Quick Start
**Model pattern**: `azure_ai/model_router/<deployment-name>`
```python
import litellm
response = litellm.completion(
model="azure_ai/model_router/model-router", # Replace with your deployment name
messages=[{"role": "user", "content": "Hello!"}],
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
api_key="your-api-key",
)
```
**Proxy config** (`config.yaml`):
```yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
## Key Features
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
@ -229,19 +255,51 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a fl
## Cost Tracking
LiteLLM automatically handles cost tracking for Azure Model Router by:
LiteLLM automatically handles cost tracking for Azure Model Router. Understanding how this works helps you interpret spend and debug billing.
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
2. **Calculating accurate costs**: Costs are calculated based on:
- The actual model used (e.g., `gpt-4.1-nano` token costs)
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
### How LiteLLM Calculates Cost
When you use Azure Model Router, LiteLLM computes **two cost components**:
| Component | Description | When Applied |
|-----------|-------------|--------------|
| **Model Cost** | Token-based cost for the actual model that handled the request (e.g., `gpt-5-nano`, `gpt-4.1-nano`) | Always, when Azure returns the model in the response |
| **Router Flat Cost** | $0.14 per million input tokens (Azure AI Foundry infrastructure fee) | When the **request** was made via a model router endpoint |
### Cost Calculation Flow
1. **Request model detection**: LiteLLM records the model you requested (e.g., `azure_ai/model_router/model-router`). If it contains `model_router` or `model-router`, the request is treated as a router request.
2. **Response model extraction**: Azure returns the actual model used in the response (e.g., `gpt-5-nano-2025-08-07`). LiteLLM uses this for the model cost lookup.
3. **Model cost**: LiteLLM looks up the response model in its pricing table and computes cost from prompt tokens and completion tokens.
4. **Router flat cost**: Because the original request was to a model router, LiteLLM adds the flat cost ($0.14 per M input tokens) on top of the model cost.
5. **Total cost**: `Total = Model Cost + Router Flat Cost`
### Configuration Requirements
For cost tracking to work correctly:
- **Use the full pattern**: `azure_ai/model_router/<deployment-name>` (e.g., `azure_ai/model_router/model-router`)
- **Proxy config**: When using the LiteLLM proxy, set `model` in `litellm_params` to the full pattern so the request model is correctly identified as a router
```yaml
# proxy_server_config.yaml
model_list:
- model_name: model-router
litellm_params:
model: azure_ai/model_router/model-router # Required for router cost detection
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
api_key: your-api-key
```
### Cost Breakdown
When you use Azure Model Router, the total cost includes:
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-5-nano`, `gpt-4.1-nano`)
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
### Example Response with Cost

2
docs/my-website/docs/providers/bedrock_agentcore.md
View file

@ -13,7 +13,7 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
:::info
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers with LiteLLM, see the [MCP AWS SigV4 Auth](https://docs.litellm.ai/docs/mcp_aws_sigv4) guide for setup instructions.
:::

36
docs/my-website/docs/providers/chatgpt.md
View file

@ -4,12 +4,12 @@ Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow a
| Property | Details |
|-------|-------|
| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
| Description | ChatGPT subscription access (Codex + GPT-5.3/5.4 family) via ChatGPT backend API |
| Provider Route on LiteLLM | `chatgpt/` |
| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
| API Reference | https://chatgpt.com |
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.4`).
Notes:
- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
@ -31,7 +31,7 @@ ChatGPT subscription access uses an OAuth device code flow:
import litellm
response = litellm.responses(
model="chatgpt/gpt-5.2-codex",
model="chatgpt/gpt-5.3-codex",
input="Write a Python hello world"
)
@ -44,7 +44,7 @@ print(response)
import litellm
response = litellm.completion(
model="chatgpt/gpt-5.2",
model="chatgpt/gpt-5.4",
messages=[{"role": "user", "content": "Write a Python hello world"}]
)
@ -55,16 +55,36 @@ print(response)
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.4
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2
- model_name: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4
- model_name: chatgpt/gpt-5.4-pro
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.2-codex
model: chatgpt/gpt-5.4-pro
- model_name: chatgpt/gpt-5.3-codex
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex
- model_name: chatgpt/gpt-5.3-codex-spark
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-codex-spark
- model_name: chatgpt/gpt-5.3-instant
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-instant
- model_name: chatgpt/gpt-5.3-chat-latest
model_info:
mode: responses
litellm_params:
model: chatgpt/gpt-5.3-chat-latest
```
```bash showLineNumbers title="Start LiteLLM Proxy"

21
docs/my-website/docs/providers/openai.md
View file

@ -192,8 +192,12 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5.2-2025-12-11 | `response = completion(model="gpt-5.2-2025-12-11", messages=messages)` |
| gpt-5.2-chat-latest | `response = completion(model="gpt-5.2-chat-latest", messages=messages)` |
| gpt-5.3-chat-latest | `response = completion(model="gpt-5.3-chat-latest", messages=messages)` |
| gpt-5.4 | `response = completion(model="gpt-5.4", messages=messages)` |
| gpt-5.4-2026-03-05 | `response = completion(model="gpt-5.4-2026-03-05", messages=messages)` |
| gpt-5.2-pro | `response = completion(model="gpt-5.2-pro", messages=messages)` |
| gpt-5.2-pro-2025-12-11 | `response = completion(model="gpt-5.2-pro-2025-12-11", messages=messages)` |
| gpt-5.4-pro | `response = completion(model="gpt-5.4-pro", messages=messages)` |
| gpt-5.4-pro-2026-03-05 | `response = completion(model="gpt-5.4-pro-2026-03-05", messages=messages)` |
| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |
| gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` |
| gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` |
@ -628,7 +632,22 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
## OpenAI Chat Completion to Responses API Bridge
Call any Responses API model from OpenAI's `/chat/completions` endpoint.
Call any Responses API model from OpenAI's `/chat/completions` endpoint.
:::tip gpt-5.4 + reasoning_effort + function tools
OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use the responses bridge instead:
```python
response = litellm.completion(
model="openai/responses/gpt-5.4", # routes to /v1/responses
messages=[{"role": "user", "content": "What's the weather?"}],
tools=[...],
reasoning_effort="low",
)
```
:::
<Tabs>
<TabItem value="sdk" label="SDK">

230
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -693,6 +693,236 @@ print(final_response.output)
Set `parallel_tool_calls=False` to ensure zero or one tool is called per turn. [More details](https://platform.openai.com/docs/guides/function-calling#parallel-function-calling).
## Tool Search & Namespaces
Tool search lets models dynamically load tools at runtime instead of sending every tool definition in the prompt. Group functions into **namespaces** and mark them with `defer_loading: true` — the model only loads the schemas it actually needs, saving tokens.
Requires `gpt-5.4` or later. See [OpenAI Tool Search docs](https://developers.openai.com/api/docs/guides/tools-tool-search) for full details.
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Tool Search with Namespaces"
import litellm
# Define namespaces with deferred tools
tools = [
{"type": "tool_search"}, # Enable tool search
{
"type": "namespace",
"name": "crm",
"description": "CRM tools for customer management",
"tools": [
{
"type": "function",
"name": "get_customer",
"description": "Get customer details by ID",
"parameters": {
"type": "object",
"properties": {
"customer_id": {"type": "string"}
},
"required": ["customer_id"],
},
"defer_loading": True,
},
{
"type": "function",
"name": "list_customers",
"description": "List customers with optional filters",
"parameters": {
"type": "object",
"properties": {
"status": {"type": "string", "enum": ["active", "inactive"]},
},
},
"defer_loading": True,
},
],
},
{
"type": "namespace",
"name": "billing",
"description": "Billing and invoicing tools",
"tools": [
{
"type": "function",
"name": "get_invoice",
"description": "Get an invoice by ID",
"parameters": {
"type": "object",
"properties": {
"invoice_id": {"type": "string"}
},
"required": ["invoice_id"],
},
"defer_loading": True,
},
],
},
]
response = litellm.responses(
model="openai/gpt-5.4",
input="Look up invoice INV-2024-001 from the billing system",
tools=tools,
)
# The response contains tool_search_call, tool_search_output, and function_call items
for item in response.output:
if isinstance(item, dict):
if item["type"] == "tool_search_call":
print(f"Searched namespaces: {item['arguments']['paths']}")
elif item["type"] == "tool_search_output":
print(f"Loaded {len(item['tools'])} tool(s)")
elif item["type"] == "function_call":
print(f"Called: {item.get('namespace', '')}.{item['name']}({item['arguments']})")
else:
if item.type == "function_call":
print(f"Called: {item.namespace}.{item.name}({item.arguments})")
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Set up config.yaml
```yaml showLineNumbers title="OpenAI Proxy Configuration"
model_list:
- model_name: openai/gpt-5.4
litellm_params:
model: openai/gpt-5.4
api_key: os.environ/OPENAI_API_KEY
```
2. Start LiteLLM Proxy Server
```bash title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```python showLineNumbers title="Tool Search via OpenAI SDK with LiteLLM Proxy"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-api-key"
)
response = client.responses.create(
model="openai/gpt-5.4",
input="Look up invoice INV-2024-001 from the billing system",
tools=[
{"type": "tool_search"},
{
"type": "namespace",
"name": "billing",
"description": "Billing and invoicing tools",
"tools": [
{
"type": "function",
"name": "get_invoice",
"description": "Get an invoice by ID",
"parameters": {
"type": "object",
"properties": {"invoice_id": {"type": "string"}},
"required": ["invoice_id"],
},
"defer_loading": True,
},
],
},
],
)
print(response.output)
```
</TabItem>
</Tabs>
### Tool Search via Chat Completions Bridge
You can also use tool search through the `/v1/chat/completions` endpoint by prefixing the model with `openai/responses/`. The request is routed through the Responses API but returns a standard chat completions response.
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Tool Search via Chat Completions Bridge"
import litellm
response = litellm.completion(
model="openai/responses/gpt-5.4",
messages=[{"role": "user", "content": "Look up invoice INV-2024-001"}],
tools=[
{"type": "tool_search"},
{
"type": "namespace",
"name": "billing",
"description": "Billing and invoicing tools",
"tools": [
{
"type": "function",
"name": "get_invoice",
"description": "Get an invoice by ID",
"parameters": {
"type": "object",
"properties": {"invoice_id": {"type": "string"}},
"required": ["invoice_id"],
},
"defer_loading": True,
},
],
},
],
)
# Standard chat completions response
for tool_call in response.choices[0].message.tool_calls:
print(f"Called: {tool_call.function.name}({tool_call.function.arguments})")
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```bash showLineNumbers title="Tool Search via /v1/chat/completions"
curl http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "openai/responses/gpt-5.4",
"messages": [{"role": "user", "content": "Look up invoice INV-2024-001"}],
"tools": [
{"type": "tool_search"},
{
"type": "namespace",
"name": "billing",
"description": "Billing and invoicing tools",
"tools": [
{
"type": "function",
"name": "get_invoice",
"description": "Get an invoice by ID",
"parameters": {
"type": "object",
"properties": {"invoice_id": {"type": "string"}},
"required": ["invoice_id"]
},
"defer_loading": true
}
]
}
]
}'
```
</TabItem>
</Tabs>
## Free-form Function Calling
<Tabs>

90
docs/my-website/docs/providers/vertex.md
View file

@ -1472,6 +1472,82 @@ Your WIF credentials JSON file typically looks like this (for AWS federation):
For more details on setting up Workload Identity Federation, see [Google Cloud WIF documentation](https://cloud.google.com/iam/docs/workload-identity-federation).
#### Explicit AWS Credentials for WIF
By default, AWS-based WIF relies on the EC2 instance metadata service to obtain AWS credentials. This works when LiteLLM runs on an EC2 instance or ECS task with an IAM role attached.
If your environment **does not have access to the EC2 metadata service** (e.g., running on-premises, in a container without host networking, or in a different cloud with security restrictions), you can provide explicit AWS credentials directly in the WIF credential JSON file. LiteLLM will use these to authenticate to AWS before performing the GCP token exchange.
Add the `aws_*` keys at the **top level** of your WIF credential JSON (alongside `type`, `audience`, etc.):
```json
{
"type": "external_account",
"audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID",
"subject_token_type": "urn:ietf:params:aws:token-type:aws4_request",
"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SERVICE_ACCOUNT_EMAIL:generateAccessToken",
"token_url": "https://sts.googleapis.com/v1/token",
"credential_source": {
"environment_id": "aws1",
"region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone",
"url": "http://169.254.169.254/latest/meta-data/iam/security-credentials",
"regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15"
},
"aws_role_name": "arn:aws:iam::123456789012:role/MyWifRole",
"aws_region_name": "us-east-1"
}
```
**Supported `aws_*` parameters:**
| Parameter | Required | Description |
|---|---|---|
| `aws_region_name` | Yes | AWS region for credential verification (e.g. `us-east-1`) |
| `aws_role_name` | No | IAM role ARN for STS AssumeRole |
| `aws_access_key_id` | No | Static AWS access key ID |
| `aws_secret_access_key` | No | Static AWS secret access key |
| `aws_session_token` | No | Temporary session token |
| `aws_profile_name` | No | AWS CLI profile name |
| `aws_session_name` | No | Session name for AssumeRole |
| `aws_web_identity_token` | No | Web identity token for STS |
| `aws_sts_endpoint` | No | Custom STS endpoint URL |
| `aws_external_id` | No | External ID for cross-account AssumeRole |
`aws_region_name` is always required when using explicit AWS credentials. The other parameters follow the same authentication flows as [Bedrock AWS auth](/docs/providers/bedrock#authentication) -- you can use role assumption, static keys, profiles, or web identity tokens.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-1.5-pro",
messages=[{"role": "user", "content": "Hello!"}],
vertex_credentials="/path/to/wif-credentials-with-aws.json", # WIF JSON with aws_* keys
vertex_project="your-gcp-project-id",
vertex_location="us-central1"
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gemini-model
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: your-gcp-project-id
vertex_location: us-central1
vertex_credentials: /path/to/wif-credentials-with-aws.json # WIF JSON with aws_* keys
```
</TabItem>
</Tabs>
When `aws_*` keys are present in the JSON, LiteLLM automatically uses explicit AWS authentication instead of the EC2 metadata service. When they are absent, the standard metadata-based flow is used unchanged.
### **Environment Variables**
You can set:
@ -1687,6 +1763,20 @@ litellm.vertex_location = "us-central1 # Your Location
| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
| gemini-3.1-flash-lite-preview | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` |
## PayGo / Priority Cost Tracking
LiteLLM automatically tracks spend for Vertex AI Gemini models using the correct pricing tier based on the response's `usageMetadata.trafficType`:
| Vertex AI `trafficType` | LiteLLM `service_tier` | Pricing applied |
|-------------------------|-------------------------|-----------------|
| `ON_DEMAND_PRIORITY` | `priority` | PayGo / priority pricing (`input_cost_per_token_priority`, `output_cost_per_token_priority`) |
| `ON_DEMAND` | standard | Default on-demand pricing |
| `FLEX` / `BATCH` | `flex` | Batch/flex pricing |
When you use [Vertex AI PayGo](https://cloud.google.com/vertex-ai/generative-ai/pricing) (on-demand priority) or batch workloads, LiteLLM reads `trafficType` from the response and applies the matching cost per token from the [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). No configuration is required — spend tracking works out of the box for both standard and PayGo requests.
See [Spend Tracking](../proxy/cost_tracking.md) for general cost tracking setup.
## Private Service Connect (PSC) Endpoints
LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments.

72
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -41,12 +41,38 @@ After creating the app, copy your **Client ID** and **Client Secret** from the a
Ensure users are assigned to the app in the **Assignments** tab. If Federation Broker Mode is enabled, you may need to disable it to assign users manually.
#### Step 3: Configure Authorization Server Access Policy
#### Step 3: Set Environment Variables
:::warning Important
This step is required. Without an Access Policy for your app, users will get a `no_matching_policy` error when attempting to log in.
Set the following environment variables. The only difference between the two Okta authorization servers is the endpoint URLs:
**Org Authorization Server** (available on all Okta plans, no additional SKU required):
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/v1/userinfo"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
**Custom Authorization Server** (requires the Okta API Access Management SKU):
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
:::tip
You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
:::
#### Step 3a: Configure Access Policy (Custom Authorization Server only)
If you are using the Custom Authorization Server, you must configure an Access Policy. Without it, users will get a `no_matching_policy` error. Skip this step if you are using the Org Authorization Server.
1. Go to **Security** → **API**
<Image img={require('../../img/okta_security_api.png')} />
@ -62,21 +88,21 @@ This step is required. Without an Access Policy for your app, users will get a `
See [Okta's Access Policy documentation](https://help.okta.com/en-us/content/topics/security/api-access-management/access-policies.htm) for more details.
#### Step 4: Configure LiteLLM Environment Variables
#### Step 4: Configure Okta Security Settings
**GENERIC_CLIENT_STATE** is recommended for Okta to prevent CSRF attacks:
```bash
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
GENERIC_CLIENT_STATE="random-string"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
:::tip
You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
:::
**PKCE (Proof Key for Code Exchange)** — If your Okta application is configured to require PKCE, enable it by setting:
```bash
GENERIC_CLIENT_USE_PKCE="true"
```
LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
#### Step 5: Test the SSO Flow
@ -91,7 +117,7 @@ You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/open
|-------|-------|----------|
| `redirect_uri` error | Redirect URI not configured | Add `<proxy_base_url>/sso/callback` to Sign-in redirect URIs in Okta |
| `access_denied` | User not assigned to app | Assign the user in the Assignments tab |
| `no_matching_policy` | Missing Access Policy | Create an Access Policy in the Authorization Server (see Step 3) |
| `no_matching_policy` | Missing Access Policy (Custom Authorization Server only) | Create an Access Policy in the Authorization Server (see Step 3a) |
</TabItem>
<TabItem value="google" label="Google SSO">
@ -456,23 +482,9 @@ PROXY_BASE_URL=http://litellm.platform.com
PROXY_BASE_URL=litellm.platform.com
```
**2. For Okta specifically, ensure GENERIC_CLIENT_STATE is set**
**2. For Okta specifically, ensure `GENERIC_CLIENT_STATE` is set and PKCE is configured if required**
Okta requires the `GENERIC_CLIENT_STATE` parameter:
```bash
GENERIC_CLIENT_STATE="random-string" # Required for Okta
```
### Okta PKCE
If your Okta application is configured to require PKCE (Proof Key for Code Exchange), enable it by setting:
```bash
GENERIC_CLIENT_USE_PKCE="true"
```
This is required when your Okta app settings enforce PKCE for enhanced security. LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
See [Okta SSO — Step 4: Configure Okta Security Settings](#step-4-configure-okta-security-settings) for details on `GENERIC_CLIENT_STATE` and PKCE configuration.
### Common Configuration Issues

7
docs/my-website/docs/proxy/config_settings.md
View file

@ -199,6 +199,7 @@ router_settings:
| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
### general_settings - Reference
@ -354,7 +355,7 @@ router_settings:
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
| retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
| provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. **Required** for `model_info.max_input_tokens` enforcement. Default: false. [More information here](reliability) |
| model_group_retry_policy | Dict[str, RetryPolicy] | [SDK-only arg] Set retry policy for model groups. |
| context_window_fallbacks | List[Dict[str, List[str]]] | Fallback models for context window violations. |
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
@ -803,6 +804,7 @@ router_settings:
| PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
| PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour)
| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
| LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
@ -815,6 +817,7 @@ router_settings:
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages`
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_WORKER_STARTUP_HOOKS | Comma-separated list of `module.path:function_name` callables to run in each worker process during startup. Runs early in the worker lifecycle (before config/DB loading). Useful for re-initializing per-process state like [gflags](https://github.com/google/python-gflags). See [Worker Startup Hooks](/proxy/worker_startup_hooks) for details
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
@ -918,6 +921,7 @@ router_settings:
| PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS | Interval in seconds for Prisma health watchdog probes. Default is 30
| PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS | Timeout in seconds for each Prisma health probe. Default is 5.0
| PRISMA_RECONNECT_COOLDOWN_SECONDS | Cooldown in seconds between Prisma reconnection attempts. Default is 15
| PRISMA_RECONNECT_ESCALATION_THRESHOLD | Number of consecutive reconnect failures before escalating the reconnection strategy. Default is 3
| PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma watchdog-initiated reconnection. Default is 30.0
| PREDIBASE_API_BASE | Base URL for Predibase API
| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
@ -940,6 +944,7 @@ router_settings:
| QDRANT_URL | Connection URL for Qdrant database
| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: '[{"host": "node1", "port": 6379}]'
| REDIS_HOST | Hostname for Redis server
| REDIS_PASSWORD | Password for Redis service
| REDIS_PORT | Port number for Redis server

2
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -8,6 +8,8 @@ Track spend for keys, users, and teams across 100+ LLMs.
LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../providers/vertex.md#paygo--priority-cost-tracking), [Bedrock service tiers](../providers/bedrock.md#usage---service-tier), [Azure base model mapping](./custom_pricing.md#set-base_model-for-cost-tracking-eg-azure-deployments)) is applied automatically when the response includes tier metadata.
:::tip Keep Pricing Data Updated
[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
:::

9
docs/my-website/docs/proxy/custom_pricing.md
View file

@ -104,9 +104,18 @@ There are other keys you can use to specify costs for different scenarios and mo
- `input_cost_per_video_per_second` - Cost per second of video input
- `input_cost_per_video_per_second_above_128k_tokens` - Video cost for large contexts
- `input_cost_per_character` - Character-based pricing for some providers
- `input_cost_per_token_priority` / `output_cost_per_token_priority` - Priority/PayGo pricing (Vertex AI Gemini, Bedrock)
- `input_cost_per_token_flex` / `output_cost_per_token_flex` - Batch/flex pricing
These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
### Service Tier / PayGo Pricing (Vertex AI, Bedrock)
For providers that support multiple pricing tiers (e.g., Vertex AI PayGo, Bedrock service tiers), LiteLLM automatically applies the correct cost based on the response:
- **Vertex AI Gemini**: Uses `usageMetadata.trafficType` (`ON_DEMAND_PRIORITY` → priority, `FLEX`/`BATCH` → flex). See [Vertex AI - PayGo / Priority Cost Tracking](../providers/vertex.md#paygo--priority-cost-tracking).
- **Bedrock**: Uses `serviceTier` from the response. See [Bedrock - Usage - Service Tier](../providers/bedrock.md#usage---service-tier).
## Zero-Cost Models (Bypass Budget Checks)
**Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.

16
docs/my-website/docs/proxy/custom_sso.md
View file

@ -121,15 +121,14 @@ Use this if you want to run your own code **after** a user signs on to the LiteL
Make sure the response type follows the `SSOUserDefinedValues` pydantic object. This is used for logging the user into the Admin UI:
```python
from fastapi import Request
from fastapi_sso.sso.base import OpenID
from litellm.proxy._types import LitellmUserRoles, SSOUserDefinedValues
from litellm.proxy.management_endpoints.internal_user_endpoints import (
new_user,
user_info,
)
from litellm.proxy.management_endpoints.team_endpoints import add_new_member
from litellm.proxy import proxy_server
# These imports are available if you need to create users or manage team membership:
# from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
# from litellm.proxy.management_endpoints.team_endpoints import add_new_member
async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
@ -158,8 +157,9 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
#################################################
# Run your custom code / logic here
# check if user exists in litellm proxy DB
_user_info = await user_info(user_id=userIDPInfo.id)
print("_user_info from litellm DB ", _user_info) # noqa
if proxy_server.prisma_client is not None:
_user_info = await proxy_server.prisma_client.get_data(user_id=userIDPInfo.id)
print("_user_info from litellm DB ", _user_info) # noqa
#################################################
return SSOUserDefinedValues(

2
docs/my-website/docs/proxy/dynamic_rate_limit.md
View file

@ -3,6 +3,8 @@
Prevent projects from gobbling too much tpm/rpm.
**See Also:** [Request Prioritization](../scheduler.md) - Prioritize LLM API requests in high-traffic by adding them to a priority queue.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
## Quick Start Usage

2
docs/my-website/docs/proxy/forward_client_headers.md
View file

@ -112,6 +112,8 @@ general_settings:
forward_llm_provider_auth_headers: true # Enable BYOK
```
For **Claude Code** with `/login` and your own Anthropic key, see [Claude Code BYOK](../tutorials/claude_code_byok.md). Use `ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345"` to pass your LiteLLM key while your Anthropic key (from `/login`) is forwarded as `x-api-key`.
Client request:
```bash
curl -X POST "http://localhost:4000/v1/messages" \

33
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -497,7 +497,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Run guardrails based on the user-agent header. This is useful for running pre-call checks on OpenWebUI but only masking in logs for Claude CLI.
`default` can be a single mode string or a list of modes.
Both `default` and tag values can be a single mode string or a list of modes.
<Tabs>
<TabItem value="single" label="Single Default Mode">
@ -545,6 +545,29 @@ guardrails:
default_on: true
```
</TabItem>
<TabItem value="tag-list" label="Multiple Tag Modes">
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "guardrails_ai-guard"
litellm_params:
guardrail: guardrails_ai
guard_name: "pii_detect"
mode:
tags:
"User-Agent: claude-cli": ["pre_call", "post_call"] # Run both pre and post call for claude-cli
default: "logging_only" # Default to logging only when no tags match
api_base: os.environ/GUARDRAILS_AI_API_BASE
default_on: true
```
</TabItem>
</Tabs>
@ -669,7 +692,7 @@ guardrails:
Mode Specification
`default` accepts either a single string or a list of strings.
Both `default` and tag values accept either a single string or a list of strings.
```python
from litellm.types.guardrails import Mode
@ -685,6 +708,12 @@ mode = Mode(
tags={"User-Agent: claude-cli": "logging_only"},
default=["pre_call", "post_call"]
)
# Multiple modes on a tag value
mode = Mode(
tags={"User-Agent: claude-cli": ["pre_call", "post_call"]},
default="logging_only"
)
```
### `guardrails` Request Parameter

28
docs/my-website/docs/proxy/reliability.md
View file

@ -713,6 +713,34 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
[**See Code**](https://github.com/BerriAI/litellm/blob/c9e6b05cfb20dfb17272218e2555d6b496c47f6f/litellm/router.py#L2163)
:::important
**`enable_pre_call_checks` is required** for context-window enforcement. Without it, requests are sent to the provider regardless of input token count. Set `enable_pre_call_checks: true` in `router_settings` in your config.
:::
#### Custom max_input_tokens per deployment
You can override the default context limit for a deployment by setting `max_input_tokens` in `model_info`. This is useful for testing, rate-limiting long prompts, or enforcing stricter limits than the provider's default.
**Both** of the following are required:
1. **`router_settings.enable_pre_call_checks: true`** — enables pre-call checks
2. **`model_info.max_input_tokens`** on the deployment — overrides the limit for that model
```yaml
router_settings:
enable_pre_call_checks: true # Required for enforcement
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
model_info:
max_input_tokens: 10 # Override: reject prompts > 10 tokens
```
If a request exceeds the limit, LiteLLM raises `ContextWindowExceededError` with details like `Model=gpt-4o, Max Input Tokens=10, Got=306`.
**1. Setup config**
For azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with azure/.

130
docs/my-website/docs/proxy/users.md
View file

@ -10,6 +10,8 @@ import TabItem from '@theme/TabItem';
**Team member budgets**: Set individual spending limits within the team's shared budget
**Agent budgets**: Set rate limits (tpm/rpm) and session-level caps (iterations, dollar budget) on agents [**Jump**](#agents)
***If a key belongs to a team, the team budget is applied, not the user's personal budget.***
:::
@ -420,6 +422,109 @@ Expected response on failure
</Tabs>
### Agents
Set budgets and rate limits on agents registered with LiteLLM's [Agent Gateway](../a2a.md). You can control:
- **Per-agent rate limits**: `tpm_limit` and `rpm_limit` on the agent itself
- **Per-session rate limits**: `session_tpm_limit` and `session_rpm_limit` applied per session
- **Per-session iteration cap**: `max_iterations` in agent `litellm_params`
- **Per-session budget cap**: `max_budget_per_session` in agent `litellm_params`
<Tabs>
<TabItem value="agent-rate-limits" label="Agent Rate Limits">
Set `tpm_limit` and `rpm_limit` on the agent to cap total throughput across all sessions.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"tpm_limit": 100000,
"rpm_limit": 100
}'
```
</TabItem>
<TabItem value="session-rate-limits" label="Session Rate Limits">
Set `session_tpm_limit` and `session_rpm_limit` to cap throughput per individual session.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"session_tpm_limit": 50000,
"session_rpm_limit": 50
}'
```
</TabItem>
<TabItem value="session-budgets" label="Session Budgets">
Set `max_iterations` and `max_budget_per_session` in agent `litellm_params` to cap individual sessions. Requires `require_trace_id_on_calls_by_agent` so LiteLLM can track calls per session.
```bash
curl -X POST 'http://localhost:4000/v1/agents' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"agent_name": "my-research-agent",
"agent_card_params": {
"name": "my-research-agent",
"description": "A research agent",
"url": "http://my-agent:8080",
"version": "1.0.0"
},
"litellm_params": {
"require_trace_id_on_calls_by_agent": true,
"max_iterations": 25,
"max_budget_per_session": 5.00
}
}'
```
When a session exceeds the limit, requests receive a **429 Too Many Requests** response.
See the [Agent Iteration Budgets](../a2a_iteration_budgets) guide for full details.
</TabItem>
</Tabs>
:::info
You can also update rate limits on existing agents using `PATCH /v1/agents/{agent_id}`:
```bash
curl -X PATCH 'http://localhost:4000/v1/agents/<agent_id>' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"tpm_limit": 200000,
"rpm_limit": 200,
"session_tpm_limit": 50000,
"session_rpm_limit": 50
}'
```
:::
### Customers
Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user**
@ -685,6 +790,31 @@ These headers indicate:
- 1 request remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
- 179 tokens remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
</TabItem>
<TabItem value="per-agent" label="Per Agent">
Set rate limits on agents registered with the [Agent Gateway](../a2a.md).
**Agent-level limits** cap total throughput across all sessions:
```shell
curl -X POST 'http://0.0.0.0:4000/v1/agents' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "tpm_limit": 100000, "rpm_limit": 100}'
```
**Session-level limits** cap throughput per individual session:
```shell
curl -X POST 'http://0.0.0.0:4000/v1/agents' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "session_tpm_limit": 50000, "session_rpm_limit": 50}'
```
You can also set **max_iterations** (call count cap) and **max_budget_per_session** (dollar cap) per session via `litellm_params`. See [Agent Iteration Budgets](../a2a_iteration_budgets) for details.
</TabItem>
<TabItem value="per-end-user" label="For customers">

155
docs/my-website/docs/proxy/worker_startup_hooks.md Normal file
View file

@ -0,0 +1,155 @@
# Worker Startup Hooks
Use `LITELLM_WORKER_STARTUP_HOOKS` to run custom initialization functions in **each worker process** during proxy startup. This is essential when using multi-worker deployments (`--num_workers > 1`) with libraries that require per-process initialization, such as [gflags](https://github.com/google/python-gflags).
## The Problem
When running the LiteLLM proxy with multiple workers:
```bash
litellm --config config.yaml --num_workers 4
```
Each worker is a **separate process** spawned by uvicorn or gunicorn. Any in-process state initialized in the master process (before `run_server()`) is **not available** in worker processes. This includes:
- [python-gflags](https://github.com/google/python-gflags) (`gflags.FLAGS`)
- [absl-py flags](https://abseil.io/docs/python/guides/flags) (`absl.flags.FLAGS`)
- Custom singleton registries or connection pools
- Any module-level state that requires explicit initialization
## Usage
Set the `LITELLM_WORKER_STARTUP_HOOKS` environment variable to a comma-separated list of `module.path:function_name` callables:
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:my_init_function"
```
Each hook is called **early** in the worker startup lifecycle — before config loading, database setup, or any request handling. Both sync and async functions are supported.
## Example: gflags Initialization
### 1. Define your wrapper module
```python title="my_litellm_wrapper.py"
import gflags
import json
import os
import sys
from typing import Optional, List, Any
def init_gflags(
usage: Optional[Any] = None,
raw_args: Optional[List[str]] = None,
known_only: bool = False,
) -> List[str]:
"""Initialize gflags from command-line arguments."""
try:
gflags.FLAGS.set_gnu_getopt(True)
if raw_args is None:
raw_args = sys.argv
argv = gflags.FLAGS(raw_args, known_only=known_only)
except gflags.Error as e:
if usage is None:
print("%s\nUsage: %s ARGS\n%s" % (e, sys.argv[0], gflags.FLAGS))
else:
print(usage % dict(cmd=sys.argv[0], flags=gflags.FLAGS))
sys.exit(1)
return argv
def init_gflags_for_worker():
"""Re-initialize gflags in each worker process.
Reads the original sys.argv from the GFLAGS_ARGV env var
(set by the master process before starting the proxy).
"""
raw_args = json.loads(os.environ.get("GFLAGS_ARGV", "[]")) or sys.argv
init_gflags(raw_args=raw_args, known_only=True)
```
### 2. Start the proxy
```python title="start_proxy.py"
import json
import os
import sys
from my_litellm_wrapper import init_gflags
# Store sys.argv so workers can re-parse the same flags
os.environ["GFLAGS_ARGV"] = json.dumps(sys.argv)
# Tell LiteLLM to call our hook in each worker
os.environ["LITELLM_WORKER_STARTUP_HOOKS"] = "my_litellm_wrapper:init_gflags_for_worker"
# Initialize gflags in the master process
init_gflags()
# Start the proxy (programmatic invocation)
from litellm.proxy.proxy_cli import run_server
run_server(
["--config", "config.yaml", "--num_workers", "4"],
standalone_mode=False,
)
```
Or via shell:
```bash
export GFLAGS_ARGV='["my_app", "--my_flag=value", "--batch_size=32"]'
export LITELLM_WORKER_STARTUP_HOOKS="my_litellm_wrapper:init_gflags_for_worker"
litellm --config config.yaml --num_workers 4
```
## How It Works
```
Master Process Worker Process (×N)
───────────────── ──────────────────────
1. init_gflags() 3. proxy_startup_event():
2. run_server() → Read LITELLM_WORKER_STARTUP_HOOKS
→ sets env vars → Import & call each hook
→ uvicorn.run(workers=N) (gflags.FLAGS re-initialized ✓)
→ spawns workers ──────────────────► → Continue with config/DB setup
→ Ready to serve requests
```
- Hooks run at the **very beginning** of `proxy_startup_event` (the FastAPI lifespan), before config loading, database connections, or any other initialization.
- Environment variables set in the master process are **inherited** by worker processes (standard Unix fork/spawn behavior).
- If a hook **raises an exception**, the worker fails to start — this is intentional, since missing initialization (e.g., uninitialized gflags) would cause downstream errors.
## Multiple Hooks
Separate multiple hooks with commas:
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:init_gflags,my_module:init_metrics,my_module:init_connections"
```
Hooks are executed **in order**, left to right.
## Async Hooks
Async functions are also supported — they are automatically awaited:
```python
async def init_async_connections():
"""Example async hook for initializing async resources."""
await setup_async_connection_pool()
```
```bash
export LITELLM_WORKER_STARTUP_HOOKS="my_module:init_async_connections"
```
## Reference
| Environment Variable | Description |
|---|---|
| `LITELLM_WORKER_STARTUP_HOOKS` | Comma-separated `module.path:function_name` callables to run in each worker on startup |
The hook format follows the standard Python entry point syntax: `module.path:function_name`, where `module.path` is a dotted Python import path and `function_name` is the name of the callable within that module.

6
docs/my-website/docs/reasoning_content.md
View file

@ -592,6 +592,12 @@ Expected Response
</TabItem>
</Tabs>
:::tip gpt-5.4: reasoning_effort + function tools
OpenAI does not support `reasoning_effort` with function tools for `gpt-5.4` in `/v1/chat/completions`. Use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
:::
## OpenAI Responses API - Auto-Summary Control
When using OpenAI Responses API models (like `gpt-5`) via `/chat/completions` with `reasoning_effort`, you can control whether `summary="detailed"` is automatically added to the reasoning parameter.

5
docs/my-website/docs/search/index.md
View file

@ -2,7 +2,7 @@
| Feature | Supported |
|---------|-----------|
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi` |
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi`, `serper` |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Load Balancing | ❌ |
@ -210,7 +210,7 @@ See the [official Perplexity Search documentation](https://docs.perplexity.ai/ap
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `query` | string or array | Yes | Search query. Can be a single string or array of strings |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, or `"searchapi"` |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, `"searchapi"`, or `"serper"` |
| `search_tool_name` | string | Yes (Proxy) | Name of the search tool configured in `config.yaml` |
| `max_results` | integer | No | Maximum number of results to return (1-20). Default: 10 |
| `search_domain_filter` | array | No | List of domains to filter results (max 20 domains) |
@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
| Serper | `SERPER_API_KEY` | `serper` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
| SearchAPI.io | `SEARCHAPI_API_KEY` | `searchapi` |

77
docs/my-website/docs/search/serper.md Normal file
View file

@ -0,0 +1,77 @@
# Serper Search
**Get API Key:** [https://serper.dev](https://serper.dev)
## LiteLLM Python SDK
```python showLineNumbers title="Serper Search"
import os
from litellm import search
os.environ["SERPER_API_KEY"] = "your-api-key"
response = search(
query="latest AI developments",
search_provider="serper",
max_results=5
)
```
## LiteLLM AI Gateway
### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-5
litellm_params:
model: gpt-5
api_key: os.environ/OPENAI_API_KEY
search_tools:
- search_tool_name: serper-search
litellm_params:
search_provider: serper
api_key: os.environ/SERPER_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test the search endpoint
```bash showLineNumbers title="Test Request"
curl http://0.0.0.0:4000/v1/search/serper-search \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "latest AI developments",
"max_results": 5
}'
```
## Provider-specific Parameters
```python showLineNumbers title="Serper Search with Provider-specific Parameters"
import os
from litellm import search
os.environ["SERPER_API_KEY"] = "your-api-key"
response = search(
query="latest tech news",
search_provider="serper",
max_results=10,
# Serper-specific parameters
gl="us", # Country/geolocation code
hl="en", # Language code
autocorrect=False, # Disable autocorrect
tbs="qdr:d", # Time filter: past day ('qdr:h' hour, 'qdr:w' week, 'qdr:m' month)
page=2 # Page number
)
```

121
docs/my-website/docs/troubleshoot/pip_venv_upgrade.md Normal file
View file

@ -0,0 +1,121 @@
# Upgrading LiteLLM Proxy (pip/venv)
Guide for upgrading LiteLLM Proxy when installed via pip in a virtual environment.
:::info Important
Always activate your virtual environment before running any `litellm` or `prisma` commands. All commands in this guide assume you're working inside an activated venv.
:::
## How pip/venv Upgrades Work
There are two pieces that need to stay in sync:
1. **Prisma client** - Generated Python code that talks to the DB
2. **DB schema** - Tables/columns in PostgreSQL
When you upgrade via pip, the `litellm-proxy-extras` package ships with a new `schema.prisma` and a `migrations/` directory. But unlike the Docker image, pip install does NOT automatically regenerate the Prisma client or run migrations. You have to do both manually.
## Upgrade Workflow (pip/venv)
### 1. Stop the proxy
Stop your running LiteLLM proxy instance.
### 2. (Optional) Back up your DB
```bash
pg_dump -h <host> -U <user> -d <db> -F c -f backup_$(date +%Y%m%d).dump
```
### 3. Upgrade the package
```bash
pip install 'litellm[proxy]==<version>'
```
### 4. Regenerate the Prisma client
```bash
prisma generate --schema <venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma
```
Replace `<venv>` with your virtual environment path and `<version>` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`).
### 5. Apply DB migrations
You have two options:
**Option A: Just start the proxy** (simplest)
The proxy automatically runs `prisma migrate deploy` on startup, which applies any new migrations.
First, activate your virtual environment:
```bash
source <venv>/bin/activate
```
Then start the proxy:
```bash
litellm --config your_config.yaml --port 4000
```
**Option B: Run manually before starting**
Activate your virtual environment first:
```bash
source <venv>/bin/activate
```
Then run the migration with the explicit schema path:
```bash
prisma migrate deploy --schema <venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma
```
Replace `<venv>` with your virtual environment path and `<version>` with your Python version (e.g., `python3.11`, `python3.12`, `python3.13`).
### 6. Start the proxy
If you used Option B above, now start the proxy (with venv still activated):
```bash
litellm --config your_config.yaml --port 4000
```
## How to Verify Migrations
> **Note:** `<schema-path>` = `<venv>/lib/python<version>/site-packages/litellm_proxy_extras/schema.prisma`
### Before applying migrations: Preview what will change
Run `pip install 'litellm[proxy]==<version>'` first (Step 3) so the new `schema.prisma` is available.
```bash
prisma migrate diff \
--from-url $DATABASE_URL \
--to-schema-datamodel <schema-path> \
--script
```
### After applying migrations: Check status
```bash
prisma migrate status --schema <schema-path>
```
All migrations should have a `finished_at` timestamp and no `rolled_back_at`.
## Key Things to Know
- **`DISABLE_SCHEMA_UPDATE=true`** env var prevents auto-migration on startup - useful if you want full manual control
- **`prisma db push`** is the nuclear option: force-syncs the DB to match the schema, bypassing migration history. Safe when all changes are additive (new columns/tables), but always have a backup.
- **The `schema.prisma` inside `litellm_proxy_extras` is the source of truth** - always use that one, not one from a different version or from the git repo
## Troubleshooting
If you encounter migration errors, see the [Prisma Migration Troubleshooting Guide](./prisma_migrations).

123
docs/my-website/docs/tutorials/claude_code_byok.md Normal file
View file

@ -0,0 +1,123 @@
# Claude Code with Bring Your Own Key (BYOK)
Use Claude Code with your own Anthropic API key through the LiteLLM proxy. When you use Claude's `/login` with your Anthropic account, your API key is sent as `x-api-key`. With BYOK enabled, LiteLLM forwards your key to Anthropic instead of using proxy-configured keys — so you pay Anthropic directly while still benefiting from LiteLLM's routing, logging, and guardrails.
## How It Works
1. **Claude Code `/login`** — You sign in with your Anthropic account; Claude Code sends your Anthropic API key as `x-api-key`.
2. **LiteLLM authentication** — You pass your LiteLLM proxy key via `ANTHROPIC_CUSTOM_HEADERS` so the proxy can authenticate and track your usage.
3. **Key forwarding** — With `forward_llm_provider_auth_headers: true`, LiteLLM forwards your `x-api-key` to Anthropic, giving it precedence over any proxy-configured keys.
## Prerequisites
- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
- Anthropic API key (from [console.anthropic.com](https://console.anthropic.com))
- LiteLLM proxy with a virtual key for authentication
## Step 1: Configure LiteLLM Proxy
Enable forwarding of LLM provider auth headers so your Anthropic key takes precedence:
```yaml title="config.yaml"
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: anthropic/claude-sonnet-4-5
# No api_key needed — client's key will be used
litellm_settings:
forward_llm_provider_auth_headers: true # Required for BYOK
```
:::info Why `forward_llm_provider_auth_headers`?
By default, LiteLLM strips `x-api-key` from client requests for security. Setting this to `true` allows client-provided provider keys (like your Anthropic key from `/login`) to be forwarded to Anthropic, overriding any proxy-configured keys.
:::
## Step 2: Create a LiteLLM Virtual Key
Create a virtual key in the LiteLLM UI or via API.
```bash
# Example: Create key via API
curl -X POST "http://localhost:4000/key/generate" \
-H "Authorization: Bearer sk-your-master-key" \
-H "Content-Type: application/json" \
-d '{"key_alias": "claude-code-byok", "models": ["claude-sonnet-4-5"]}'
```
## Step 3: Configure Claude Code
Set environment variables so Claude Code uses LiteLLM and sends your LiteLLM key for proxy auth:
```bash
# Point Claude Code to your LiteLLM proxy
export ANTHROPIC_BASE_URL="http://localhost:4000"
# Model name from your config
export ANTHROPIC_MODEL="claude-sonnet-4-5"
# LiteLLM proxy auth — this is added to every request
# Use x-litellm-api-key so the proxy authenticates you; your Anthropic key goes via x-api-key from /login
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345"
```
Replace `sk-12345` with your actual LiteLLM virtual key.
:::tip Multiple headers
For multiple headers, use newline-separated values:
```bash
export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: sk-12345
x-litellm-user-id: my-user-id"
```
:::
## Step 4: Sign In with Claude Code
1. Launch Claude Code:
```bash
claude
```
2. Use **`/login`** and sign in with your Anthropic account (or use your API key directly).
3. Claude Code will send:
- `x-api-key`: Your Anthropic API key (from `/login`)
- `x-litellm-api-key`: Your LiteLLM key (from `ANTHROPIC_CUSTOM_HEADERS`)
4. LiteLLM authenticates you via `x-litellm-api-key`, then forwards `x-api-key` to Anthropic. Your Anthropic key takes precedence over any proxy-configured key.
## Summary
| Header | Source | Purpose |
|--------|--------|---------|
| `x-api-key` | Claude Code `/login` (Anthropic key) | Sent to Anthropic for API calls |
| `x-litellm-api-key` | `ANTHROPIC_CUSTOM_HEADERS` | Proxy authentication, tracking, rate limits |
## Troubleshooting
### Requests fail with "invalid x-api-key"
- Ensure `forward_llm_provider_auth_headers: true` is set in `litellm_settings` (or `general_settings`).
- Restart the LiteLLM proxy after changing the config.
- Verify you completed `/login` in Claude Code so your Anthropic key is being sent.
### Proxy returns 401
- Check that `ANTHROPIC_CUSTOM_HEADERS` includes `x-litellm-api-key: <your-key>`.
- Ensure the LiteLLM key is valid and has access to the model.
### Proxy key is used instead of my Anthropic key
- Confirm `forward_llm_provider_auth_headers: true` is in your config.
- The setting can be in `litellm_settings` or `general_settings` depending on your config structure.
- Enable debug logging: `LITELLM_LOG=DEBUG` to see which key is being forwarded.
## Related
- [Forward Client Headers](./../proxy/forward_client_headers.md) — Full BYOK and header forwarding docs
- [Claude Code Max Subscription](./claude_code_max_subscription.md) — Using Claude Code with OAuth/Max subscription through LiteLLM

BIN
docs/my-website/img/claude_code_byok_screenshot.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 99 KiB

67
docs/my-website/package-lock.json generated
View file

@ -7449,15 +7449,6 @@
"tslib": "^2.6.2"
}
},
"node_modules/@trysound/sax": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz",
"integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==",
"license": "ISC",
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/@types/body-parser": {
"version": "1.19.6",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
@ -10340,13 +10331,13 @@
}
},
"node_modules/css-tree": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.3.1.tgz",
"integrity": "sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==",
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
"integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==",
"license": "MIT",
"dependencies": {
"mdn-data": "2.0.30",
"source-map-js": "^1.0.1"
"mdn-data": "2.27.1",
"source-map-js": "^1.2.1"
},
"engines": {
"node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0"
@ -11363,10 +11354,13 @@
}
},
"node_modules/dompurify": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.0.tgz",
"integrity": "sha512-r+f6MYR1gGN1eJv0TVQbhA7if/U7P87cdPl3HN5rikqaBSBxLiCb/b9O+2eG0cxz0ghyU+mU1QkbsOwERMYlWQ==",
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.2.tgz",
"integrity": "sha512-6obghkliLdmKa56xdbLOpUZ43pAR6xFy1uOrxBaIDjT+yaRuuybLjGS9eVBoSR/UPU5fq3OXClEHLJNGvbxKpQ==",
"license": "(MPL-2.0 OR Apache-2.0)",
"engines": {
"node": ">=20"
},
"optionalDependencies": {
"@types/trusted-types": "^2.0.7"
}
@ -14704,9 +14698,9 @@
}
},
"node_modules/mdn-data": {
"version": "2.0.30",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz",
"integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==",
"version": "2.27.1",
"resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz",
"integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==",
"license": "CC0-1.0"
},
"node_modules/media-typer": {
@ -20409,6 +20403,13 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",
@ -21381,24 +21382,24 @@
"license": "MIT"
},
"node_modules/svgo": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/svgo/-/svgo-3.3.2.tgz",
"integrity": "sha512-OoohrmuUlBs8B8o6MB2Aevn+pRIH9zDALSR+6hhqVfa6fRwG/Qw9VUMSMW9VNg2CFc/MTIfabtdOVl9ODIJjpw==",
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/svgo/-/svgo-4.0.1.tgz",
"integrity": "sha512-XDpWUOPC6FEibaLzjfe0ucaV0YrOjYotGJO1WpF0Zd+n6ZGEQUsSugaoLq9QkEZtAfQIxT42UChcssDVPP3+/w==",
"license": "MIT",
"dependencies": {
"@trysound/sax": "0.2.0",
"commander": "^7.2.0",
"commander": "^11.1.0",
"css-select": "^5.1.0",
"css-tree": "^2.3.1",
"css-tree": "^3.0.1",
"css-what": "^6.1.0",
"csso": "^5.0.5",
"picocolors": "^1.0.0"
"picocolors": "^1.1.1",
"sax": "^1.5.0"
},
"bin": {
"svgo": "bin/svgo"
"svgo": "bin/svgo.js"
},
"engines": {
"node": ">=14.0.0"
"node": ">=16"
},
"funding": {
"type": "opencollective",
@ -21406,12 +21407,12 @@
}
},
"node_modules/svgo/node_modules/commander": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
"integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==",
"version": "11.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz",
"integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==",
"license": "MIT",
"engines": {
"node": ">= 10"
"node": ">=16"
}
},
"node_modules/tailwind-merge": {

6
docs/my-website/package.json
View file

@ -61,7 +61,7 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.8",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
@ -93,6 +93,8 @@
"axios": ">=0.30.2",
"webpack": ">=5.94.0",
"serve-static": ">=1.16.0",
"path-to-regexp": ">=0.1.12"
"path-to-regexp": ">=0.1.12",
"dompurify": ">=3.3.2",
"svgo": ">=3.3.3"
}
}

9
docs/my-website/sidebars.js
View file

@ -154,6 +154,7 @@ const sidebars = {
items: [
"tutorials/claude_responses_api",
"tutorials/claude_code_max_subscription",
"tutorials/claude_code_byok",
"tutorials/claude_code_customer_tracking",
"tutorials/claude_code_prompt_cache_routing",
"tutorials/claude_code_websearch",
@ -310,6 +311,7 @@ const sidebars = {
"proxy/master_key_rotations",
"proxy/model_management",
"proxy/prod",
"proxy/worker_startup_hooks",
"proxy/release_cycle",
],
},
@ -538,8 +540,10 @@ const sidebars = {
items: [
"a2a",
"a2a_invoking_agents",
"a2a_agent_headers",
"a2a_cost_tracking",
"a2a_agent_permissions"
"a2a_agent_permissions",
"a2a_iteration_budgets"
],
},
"assistants",
@ -610,6 +614,7 @@ const sidebars = {
"mcp_usage",
"mcp_openapi",
"mcp_oauth",
"mcp_aws_sigv4",
"mcp_public_internet",
"mcp_semantic_filter",
"mcp_control",
@ -680,6 +685,7 @@ const sidebars = {
"search/firecrawl",
"search/searxng",
"search/linkup",
"search/serper",
]
},
"skills",
@ -1151,6 +1157,7 @@ const sidebars = {
"troubleshoot/prisma_migrations",
],
},
"troubleshoot/pip_venv_upgrade",
"troubleshoot/rollback",
"troubleshoot",
],

4
enterprise/litellm_enterprise/integrations/custom_guardrail.py
View file

@ -50,8 +50,10 @@ class EnterpriseCustomGuardrailHelper:
break
if matched_mode is not None:
# Tag matched: only run if event_type matches the tag's mode value
# Tag matched: only run if event_type matches the tag's mode value(s)
if event_type is not None:
if isinstance(matched_mode, list):
return event_type.value in matched_mode
return event_type.value == matched_mode
return True

22
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -78,8 +78,6 @@ class CheckBatchCost:
"status": {"not_in": ["failed", "expired", "cancelled"]}
}
)
completed_jobs = []
for job in jobs:
# get the model from the job
unified_object_id = job.unified_object_id
@ -237,10 +235,16 @@ class CheckBatchCost:
)
# mark the job as complete
completed_jobs.append(job)
if len(completed_jobs) > 0:
await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"batch_processed": True, "status": "complete"},
)
try:
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data={
"batch_processed": True,
"status": "complete",
"file_object": response.model_dump_json(),
},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)

2
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.33"
version = "0.1.34"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"

4
litellm-js/spend-logs/package.json
View file

@ -12,8 +12,8 @@
},
"overrides": {
"glob": ">=11.1.0",
"tar": ">=7.5.8",
"minimatch": ">=10.2.1",
"tar": ">=7.5.10",
"minimatch": ">=10.2.4",
"diff": ">=8.0.3",
"@isaacs/brace-expansion": ">=5.0.1",
"@babel/traverse": ">=7.23.2",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.51-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.51.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.52-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.52.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.53-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.53.tar.gz vendored Normal file
View file

Binary file not shown.

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260304175016_add_spend_to_agent_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260305000000_add_agent_headers/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- Add static_headers and extra_headers to LiteLLM_AgentsTable
ALTER TABLE "LiteLLM_AgentsTable"
ADD COLUMN IF NOT EXISTS "static_headers" JSONB DEFAULT '{}',
ADD COLUMN IF NOT EXISTS "extra_headers" TEXT[] DEFAULT ARRAY[]::TEXT[];

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260305000000_add_rate_limits_to_agents/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "tpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "rpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_tpm_limit" INTEGER;
ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_rpm_limit" INTEGER;

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260306175056_add_configs_override_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "spec_path" TEXT;

57
litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql Normal file
View file

@ -0,0 +1,57 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "byok_api_key_help_url" TEXT,
ADD COLUMN "byok_description" TEXT[] DEFAULT ARRAY[]::TEXT[],
ADD COLUMN "is_byok" BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN "tool_name_to_description" JSONB DEFAULT '{}',
ADD COLUMN "tool_name_to_display_name" JSONB DEFAULT '{}';
-- CreateTable
CREATE TABLE "LiteLLM_MCPUserCredentials" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"server_id" TEXT NOT NULL,
"credential_b64" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_MCPUserCredentials_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LiteLLM_JWTKeyMapping" (
"id" TEXT NOT NULL,
"jwt_claim_name" TEXT NOT NULL,
"jwt_claim_value" TEXT NOT NULL,
"token" TEXT NOT NULL,
"description" TEXT,
"is_active" BOOLEAN NOT NULL DEFAULT true,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_JWTKeyMapping_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "LiteLLM_ConfigOverrides" (
"config_type" TEXT NOT NULL,
"config_value" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LiteLLM_ConfigOverrides_pkey" PRIMARY KEY ("config_type")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_MCPUserCredentials_user_id_server_id_key" ON "LiteLLM_MCPUserCredentials"("user_id", "server_id");
-- CreateIndex
CREATE INDEX "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx" ON "LiteLLM_JWTKeyMapping"("jwt_claim_name", "jwt_claim_value", "is_active");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key" ON "LiteLLM_JWTKeyMapping"("jwt_claim_name", "jwt_claim_value");
-- AddForeignKey
ALTER TABLE "LiteLLM_JWTKeyMapping" ADD CONSTRAINT "LiteLLM_JWTKeyMapping_token_fkey" FOREIGN KEY ("token") REFERENCES "LiteLLM_VerificationToken"("token") ON DELETE RESTRICT ON UPDATE CASCADE;

11
litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000000_add_mcp_approval_status/migration.sql Normal file
View file

@ -0,0 +1,11 @@
-- AlterTable: Add BYOM approval workflow fields to LiteLLM_MCPServerTable
ALTER TABLE "LiteLLM_MCPServerTable"
ADD COLUMN IF NOT EXISTS "approval_status" TEXT DEFAULT 'active',
ADD COLUMN IF NOT EXISTS "submitted_by" TEXT,
ADD COLUMN IF NOT EXISTS "submitted_at" TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS "reviewed_at" TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS "review_notes" TEXT;
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_MCPServerTable_approval_status_idx"
ON "LiteLLM_MCPServerTable"("approval_status");

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260309000001_add_mcp_source_url/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable: Add source_url field to LiteLLM_MCPServerTable for GitHub/docs link
ALTER TABLE "LiteLLM_MCPServerTable"
ADD COLUMN IF NOT EXISTS "source_url" TEXT;

85
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -63,9 +63,16 @@ model LiteLLM_AgentsTable {
agent_name String @unique
litellm_params Json?
agent_card_params Json
static_headers Json? @default("{}")
extra_headers String[] @default([])
agent_access_groups String[] @default([])
object_permission_id String?
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
spend Float @default(0.0)
tpm_limit Int?
rpm_limit Int?
session_tpm_limit Int?
session_rpm_limit Int?
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@ -288,6 +295,8 @@ model LiteLLM_MCPServerTable {
mcp_info Json? @default("{}")
mcp_access_groups String[]
allowed_tools String[] @default([])
tool_name_to_display_name Json? @default("{}")
tool_name_to_description Json? @default("{}")
extra_headers String[] @default([])
static_headers Json? @default("{}")
// Health check status
@ -303,6 +312,21 @@ model LiteLLM_MCPServerTable {
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(true)
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?
}
// Per-user BYOK credentials for MCP servers
model LiteLLM_MCPUserCredentials {
id String @id @default(uuid())
user_id String
server_id String
credential_b64 String
created_at DateTime @default(now()) @map("created_at")
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@@unique([user_id, server_id])
}
// Generate Tokens for Proxy
@ -353,6 +377,7 @@ model LiteLLM_VerificationToken {
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
jwt_key_mappings LiteLLM_JWTKeyMapping[]
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@ -365,6 +390,24 @@ model LiteLLM_VerificationToken {
@@index([budget_reset_at, expires])
}
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
description String?
is_active Boolean @default(true)
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token])
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
model LiteLLM_DeprecatedVerificationToken {
id String @id @default(uuid())
@ -1019,6 +1062,14 @@ model LiteLLM_UISettings {
updated_at DateTime @updatedAt
}
// Generic config overrides table - one row per config_type
model LiteLLM_ConfigOverrides {
config_type String @id
config_value Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
@ -1077,24 +1128,24 @@ model LiteLLM_PolicyAttachmentTable {
updated_by String?
}
// Global tool registry - auto-discovered from LLM responses; admins set input_policy/output_policy here
// Global tool registry - auto-discovered from LLM responses; admins set input/output policies here
model LiteLLM_ToolTable {
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([input_policy])
@@index([output_policy])

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.50"
version = "0.4.53"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.50"
version = "0.4.53"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

3
litellm/__init__.py
View file

@ -305,6 +305,9 @@ return_response_headers: bool = (
False # get response headers from LLM Api providers - example x-remaining-requests,
)
enable_json_schema_validation: bool = False
enable_key_alias_format_validation: bool = (
False # opt-in validation of key_alias format on /key/generate and /key/update
)
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None

48
litellm/a2a_protocol/main.py
View file

@ -212,6 +212,7 @@ async def asend_message(
api_base: Optional[str] = None,
litellm_params: Optional[Dict[str, Any]] = None,
agent_id: Optional[str] = None,
agent_extra_headers: Optional[Dict[str, str]] = None,
**kwargs: Any,
) -> LiteLLMSendMessageResponse:
"""
@ -293,9 +294,12 @@ async def asend_message(
"Either a2a_client or api_base is required for standard A2A flow"
)
trace_id = trace_id or str(uuid.uuid4())
extra_headers = {"X-LiteLLM-Trace-Id": trace_id}
extra_headers: Dict[str, str] = {"X-LiteLLM-Trace-Id": trace_id}
if agent_id:
extra_headers["X-LiteLLM-Agent-Id"] = agent_id
# Overlay agent-level headers (agent headers take precedence over LiteLLM internal ones)
if agent_extra_headers:
extra_headers.update(agent_extra_headers)
a2a_client = await create_a2a_client(
base_url=api_base, extra_headers=extra_headers
)
@ -434,7 +438,7 @@ def _build_streaming_logging_obj(
return logging_obj
async def asend_message_streaming(
async def asend_message_streaming( # noqa: PLR0915
a2a_client: Optional["A2AClientType"] = None,
request: Optional["SendStreamingMessageRequest"] = None,
api_base: Optional[str] = None,
@ -442,6 +446,7 @@ async def asend_message_streaming(
agent_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
proxy_server_request: Optional[Dict[str, Any]] = None,
agent_extra_headers: Optional[Dict[str, str]] = None,
) -> AsyncIterator[Any]:
"""
Async: Send a streaming message to an A2A agent.
@ -523,7 +528,17 @@ async def asend_message_streaming(
raise ValueError(
"Either a2a_client or api_base is required for standard A2A flow"
)
a2a_client = await create_a2a_client(base_url=api_base)
# Mirror the non-streaming path: always include trace and agent-id headers
streaming_extra_headers: Dict[str, str] = {
"X-LiteLLM-Trace-Id": str(request.id),
}
if agent_id:
streaming_extra_headers["X-LiteLLM-Agent-Id"] = agent_id
if agent_extra_headers:
streaming_extra_headers.update(agent_extra_headers)
a2a_client = await create_a2a_client(
base_url=api_base, extra_headers=streaming_extra_headers
)
# Type assertion: a2a_client is guaranteed to be non-None here
assert a2a_client is not None
@ -637,17 +652,30 @@ async def create_a2a_client(
verbose_logger.info(f"Creating A2A client for {base_url}")
# Use LiteLLM's cached httpx client
http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.A2A,
params={"timeout": timeout},
# Use get_async_httpx_client with per-agent params so that different agents
# (with different extra_headers) get separate cached clients. The params
# dict is hashed into the cache key, keeping agent auth isolated while
# still reusing connections within the same agent.
#
# Only pass params that AsyncHTTPHandler.__init__ accepts (e.g. timeout).
# Use "disable_aiohttp_transport" key for cache-key-only data (it's
# filtered out before reaching the constructor).
_client_params: dict = {"timeout": timeout}
if extra_headers:
# Encode headers into a cache-key-only param so each unique header
# set produces a distinct cache key.
_client_params["disable_aiohttp_transport"] = str(
sorted(extra_headers.items())
)
_async_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.A2AProvider,
params=_client_params,
)
httpx_client = http_handler.client
httpx_client = _async_handler.client
if extra_headers:
httpx_client.headers.update(extra_headers)
verbose_proxy_logger.debug(
f"A2A client created with extra_headers={extra_headers}"
f"A2A client created with extra_headers={list(extra_headers.keys())}"
)
# Resolve agent card

7
litellm/batches/batch_utils.py
View file

@ -198,9 +198,8 @@ async def _get_batch_output_file_content_as_dictionary(
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
from litellm.proxy.openai_files_endpoints.common_utils import \
_is_base64_encoded_unified_file_id
if custom_llm_provider == "vertex_ai":
raise ValueError("Vertex AI does not support file content retrieval")
@ -227,7 +226,7 @@ async def _get_batch_output_file_content_as_dictionary(
credentials = _extract_file_access_credentials(litellm_params)
file_content_kwargs.update(credentials)
_file_content = await afile_content(**file_content_kwargs)
_file_content = await afile_content(**file_content_kwargs) # type: ignore[reportArgumentType]
return _get_file_content_as_dictionary(_file_content.content)

8
litellm/caching/caching.py
View file

@ -166,6 +166,14 @@ class Cache:
None. Cache is set as a litellm param
"""
if type == LiteLLMCacheType.REDIS:
# Check REDIS_CLUSTER_NODES env var if no explicit startup nodes
if not redis_startup_nodes:
_env_cluster_nodes = litellm.get_secret("REDIS_CLUSTER_NODES")
if _env_cluster_nodes is not None and isinstance(
_env_cluster_nodes, str
):
redis_startup_nodes = json.loads(_env_cluster_nodes)
if redis_startup_nodes:
# Only pass GCP parameters if they are provided
cluster_kwargs = {

4
litellm/caching/dual_cache.py
View file

@ -346,6 +346,8 @@ class DualCache(BaseCache):
)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
kwargs["ttl"] = self.default_in_memory_ttl
await self.in_memory_cache.async_set_cache(key, value, **kwargs)
if self.redis_cache is not None and local_only is False:
@ -367,6 +369,8 @@ class DualCache(BaseCache):
)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
kwargs["ttl"] = self.default_in_memory_ttl
await self.in_memory_cache.async_set_cache_pipeline(
cache_list=cache_list, **kwargs
)

22
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -390,6 +390,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
ResponseOutputMessage,
ResponseReasoningItem,
)
from openai.types.responses.response_output_item import ResponseApplyPatchToolCall
from litellm.types.utils import Choices, Message
@ -448,6 +449,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, ResponseApplyPatchToolCall):
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_apply_patch_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=tool_call_index,
)
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, dict) and handle_raw_dict_callback is not None:
# Handle raw dict responses (e.g., from GPT-5 Codex)
choice, index = handle_raw_dict_callback(item=item, index=index)
@ -1095,6 +1108,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
finish_reason = "tool_calls" if has_function_calls else "stop"
usage = None
if response_data.get("usage"):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
response_data.get("usage")
)
return ModelResponseStream(
choices=[
StreamingChoices(
@ -1102,7 +1121,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
delta=Delta(content=""),
finish_reason=finish_reason,
)
]
],
usage=usage
)
else:
pass

5
litellm/constants.py
View file

@ -1242,6 +1242,11 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "
"Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
"To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
)
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################

10
litellm/cost_calculator.py
View file

@ -272,6 +272,8 @@ def cost_per_token( # noqa: PLR0915
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
response: Optional[Any] = None,
### REQUEST MODEL ###
request_model: Optional[str] = None, # original request model for router detection
) -> Tuple[float, float]: # type: ignore
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -520,7 +522,7 @@ def cost_per_token( # noqa: PLR0915
return dashscope_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "azure_ai":
return azure_ai_cost_per_token(
model=model, usage=usage_block, response_time_ms=response_time_ms
model=model, usage=usage_block, response_time_ms=response_time_ms, request_model=request_model
)
else:
model_info = _cached_get_model_info_helper(
@ -1457,6 +1459,11 @@ def completion_cost( # noqa: PLR0915
text=completion_string
)
# Get the original request model for router detection
request_model_for_cost = None
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1479,6 +1486,7 @@ def completion_cost( # noqa: PLR0915
rerank_billed_units=rerank_billed_units,
service_tier=service_tier,
response=completion_response,
request_model=request_model_for_cost,
)
# Get additional costs from provider (e.g., routing fees, infrastructure costs)

97
litellm/experimental_mcp_client/client.py
View file

@ -4,7 +4,7 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
import asyncio
import base64
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, TypeVar, Union
from typing import Any, Awaitable, Callable, Dict, Generator, List, Optional, Tuple, TypeVar, Union
import httpx
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
@ -50,6 +50,86 @@ def to_basic_auth(auth_value: str) -> str:
TSessionResult = TypeVar("TSessionResult")
class MCPSigV4Auth(httpx.Auth):
"""
httpx Auth class that signs each request with AWS SigV4.
This is used for MCP servers that require AWS SigV4 authentication,
such as AWS Bedrock AgentCore MCP servers. httpx calls auth_flow()
for every outgoing request, enabling per-request signature computation.
"""
requires_request_body = True
def __init__(
self,
aws_access_key_id: Optional[str] = None,
aws_secret_access_key: Optional[str] = None,
aws_session_token: Optional[str] = None,
aws_region_name: Optional[str] = None,
aws_service_name: Optional[str] = None,
):
try:
from botocore.credentials import Credentials
except ImportError:
raise ImportError(
"Missing botocore to use AWS SigV4 authentication. "
"Run 'pip install boto3'."
)
self.service_name = aws_service_name or "bedrock-agentcore"
self.region_name = aws_region_name or "us-east-1"
# Note: os.environ/ prefixed values are already resolved by
# ProxyConfig._check_for_os_environ_vars() at config load time.
# Values arrive here as plain strings.
if aws_access_key_id and aws_secret_access_key:
self.credentials = Credentials(
access_key=aws_access_key_id,
secret_key=aws_secret_access_key,
token=aws_session_token,
)
else:
# Fall back to default boto3 credential chain
import botocore.session
session = botocore.session.get_session()
self.credentials = session.get_credentials()
if self.credentials is None:
raise ValueError(
"No AWS credentials found. Provide aws_access_key_id and "
"aws_secret_access_key, or configure default credentials "
"(env vars, ~/.aws/credentials, instance profile)."
)
def auth_flow(
self, request: httpx.Request
) -> Generator[httpx.Request, httpx.Response, None]:
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
# Build AWSRequest from the httpx Request.
# Pass all request headers so the canonical SigV4 signature covers them.
aws_request = AWSRequest(
method=request.method,
url=str(request.url),
data=request.content,
headers=dict(request.headers),
)
# Sign the request — SigV4Auth.add_auth() adds Authorization,
# X-Amz-Date, and X-Amz-Security-Token (if session token present).
# Host header is derived automatically from the URL.
sigv4 = SigV4Auth(self.credentials, self.service_name, self.region_name)
sigv4.add_auth(aws_request)
# Copy SigV4 headers back to the httpx request
for header_name, header_value in aws_request.headers.items():
request.headers[header_name] = header_value
yield request
class MCPClient:
"""
MCP Client supporting:
@ -68,6 +148,7 @@ class MCPClient:
stdio_config: Optional[MCPStdioConfig] = None,
extra_headers: Optional[Dict[str, str]] = None,
ssl_verify: Optional[VerifyTypes] = None,
aws_auth: Optional[httpx.Auth] = None,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@ -77,6 +158,7 @@ class MCPClient:
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.extra_headers: Optional[Dict[str, str]] = extra_headers
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
self._aws_auth: Optional[httpx.Auth] = aws_auth
# handle the basic auth value if provided
if auth_value:
self.update_auth_value(auth_value)
@ -212,8 +294,13 @@ class MCPClient:
headers["Authorization"] = self._mcp_auth_value
elif self.auth_type == MCPAuth.oauth2:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
# signing (including the body hash), so it uses httpx.Auth flow instead
# of static headers. See MCPSigV4Auth and _create_httpx_client_factory().
# update the headers with the extra headers
if self.extra_headers:
@ -246,10 +333,16 @@ class MCPClient:
f"MCP client using SSL configuration: {type(ssl_config).__name__}"
)
# Use SigV4 auth if configured and no explicit auth provided.
# The MCP SDK's sse_client and streamable_http_client call this
# factory without passing auth=, so self._aws_auth is used.
# For non-SigV4 clients, self._aws_auth is None — no behavior change.
effective_auth = auth if auth is not None else self._aws_auth
return httpx.AsyncClient(
headers=headers,
timeout=timeout,
auth=auth,
auth=effective_auth,
verify=ssl_config,
follow_redirects=True,
)

57
litellm/fine_tuning/main.py
View file

@ -126,6 +126,18 @@ async def acreate_fine_tuning_job(
raise e
def _build_fine_tuning_job_data(model, training_file, hyperparameters, suffix, validation_file, integrations, seed):
return FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
def _resolve_fine_tuning_timeout(
timeout: Any,
custom_llm_provider: str,
@ -206,19 +218,9 @@ def create_fine_tuning_job(
or os.getenv("OPENAI_API_KEY")
)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
exclude_none=True
)
create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
).model_dump(exclude_none=True)
response = openai_fine_tuning_apis_instance.create_fine_tuning_job(
api_base=api_base,
@ -260,20 +262,10 @@ def create_fine_tuning_job(
# Prepare Azure-specific parameters for extra_body
extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
).model_dump(exclude_none=True)
create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
exclude_none=True
)
# Add extra_body if it has Azure-specific parameters
if extra_body:
create_fine_tuning_job_data_dict["extra_body"] = extra_body
@ -303,18 +295,11 @@ def create_fine_tuning_job(
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
create_fine_tuning_job_data = FineTuningJobCreate(
model=model,
training_file=training_file,
hyperparameters=_oai_hyperparameters,
suffix=suffix,
validation_file=validation_file,
integrations=integrations,
seed=seed,
)
response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
_is_async=_is_async,
create_fine_tuning_job_data=create_fine_tuning_job_data,
create_fine_tuning_job_data=_build_fine_tuning_job_data(
model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
),
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,

4
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -82,8 +82,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
_targetted_index: Optional[Union[int, str]] = point.get("index", None)
targetted_index: Optional[int] = None
if isinstance(_targetted_index, str):
if _targetted_index.isdigit():
try:
targetted_index = int(_targetted_index)
except ValueError:
pass
else:
targetted_index = _targetted_index

26
litellm/integrations/custom_guardrail.py
View file

@ -231,8 +231,14 @@ class CustomGuardrail(CustomLogger):
event_hook, supported_event_hooks
)
elif isinstance(event_hook, Mode):
tag_values_flat: list = []
for v in event_hook.tags.values():
if isinstance(v, list):
tag_values_flat.extend(v)
else:
tag_values_flat.append(v)
_validate_event_hook_list_is_in_supported_event_hooks(
list(event_hook.tags.values()), supported_event_hooks
tag_values_flat, supported_event_hooks
)
if event_hook.default:
default_list = (
@ -466,8 +472,12 @@ class CustomGuardrail(CustomLogger):
if isinstance(self.event_hook, list):
return event_type.value in self.event_hook
if isinstance(self.event_hook, Mode):
if event_type.value in self.event_hook.tags.values():
return True
for tag_value in self.event_hook.tags.values():
if isinstance(tag_value, list):
if event_type.value in tag_value:
return True
elif event_type.value == tag_value:
return True
if self.event_hook.default:
default_list = (
self.event_hook.default
@ -579,6 +589,16 @@ class CustomGuardrail(CustomLogger):
guardrail_json_response
)
# Strip secret_fields to prevent plaintext Authorization headers from
# being persisted to spend logs, OTEL traces, or other logging backends.
# This matches the pattern used by Langfuse and Arize integrations.
if isinstance(clean_guardrail_response, dict):
clean_guardrail_response.pop("secret_fields", None)
elif isinstance(clean_guardrail_response, list):
for item in clean_guardrail_response:
if isinstance(item, dict):
item.pop("secret_fields", None)
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,

68
litellm/integrations/opentelemetry.py
View file

@ -735,13 +735,10 @@ class OpenTelemetry(CustomLogger):
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, span
)
# Ensure proxy-request parent span is annotated with the actual operation kind
if (
parent_span is not None
and hasattr(parent_span, "name")
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
self.set_attributes(parent_span, kwargs, response_obj)
# Do NOT duplicate attributes onto the parent proxy-request span.
# The child litellm_request span already carries all attributes;
# copying them to the parent doubles storage and complicates
# search (Issue #4).
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
from opentelemetry.trace import Status, StatusCode
@ -757,8 +754,12 @@ class OpenTelemetry(CustomLogger):
kwargs, response_obj, start_time, end_time, parent_span
)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
# 3. Guardrail span — ensure guardrails are always parented to an
# existing span so they never become orphaned root spans (Issue #5).
guardrail_ctx = self._resolve_guardrail_context(
span=span, parent_span=parent_span, fallback_ctx=ctx
)
self._create_guardrail_span(kwargs=kwargs, context=guardrail_ctx)
# 4. Metrics & cost recording
self._record_metrics(kwargs, response_obj, start_time, end_time)
@ -1145,6 +1146,27 @@ class OpenTelemetry(CustomLogger):
)
otel_logger.emit(log_record)
@staticmethod
def _resolve_guardrail_context(
span: Optional[Any],
parent_span: Optional[Any],
fallback_ctx: Optional[Any],
) -> Optional[Any]:
"""
Return a valid OTEL context for guardrail child spans so they are
never orphaned (Issue #5). Priority:
1. The litellm_request span that was just created
2. The parent proxy-request span
3. The original fallback context (may be None — last resort)
"""
from opentelemetry import trace as _trace
if span is not None:
return _trace.set_span_in_context(span)
if parent_span is not None:
return _trace.set_span_in_context(parent_span)
return fallback_ctx
def _create_guardrail_span(
self, kwargs: Optional[dict], context: Optional[Context]
):
@ -1250,6 +1272,7 @@ class OpenTelemetry(CustomLogger):
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
span = None
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
@ -1275,8 +1298,11 @@ class OpenTelemetry(CustomLogger):
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
# Create span for guardrail information
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
# Create span for guardrail information — ensure proper parenting (Issue #5)
guardrail_ctx = self._resolve_guardrail_context(
span=span, parent_span=parent_otel_span, fallback_ctx=_parent_context
)
self._create_guardrail_span(kwargs=kwargs, context=guardrail_ctx)
# Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
@ -1579,12 +1605,20 @@ class OpenTelemetry(CustomLogger):
value=optional_params.get("user"),
)
# The unique identifier for the completion.
if response_obj and response_obj.get("id"):
# The unique identifier for the LLM call.
# Completions have a provider response ID (e.g. "chatcmpl-xxx"),
# but Embeddings and Image-gen responses do not. Fall back to
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (
(response_obj.get("id") if response_obj else None)
or standard_logging_payload.get("id")
)
if response_id:
self.safe_set_attribute(
span=span,
key="gen_ai.response.id",
value=response_obj.get("id"),
value=response_id,
)
# The model used to generate the response.
@ -1808,8 +1842,10 @@ class OpenTelemetry(CustomLogger):
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
self.set_attributes(span, kwargs, response_obj)
kwargs.get("optional_params", {})
# Only set provider-specific raw payload attributes on this span.
# The parent litellm_request span already carries the standard
# gen_ai.* / metadata.* attributes — duplicating them here doubles
# storage and adds noise (Issue #3).
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")

10
litellm/litellm_core_utils/duration_parser.py
View file

@ -64,12 +64,10 @@ def duration_in_seconds(duration: str) -> int:
now = time.time()
current_time = datetime.fromtimestamp(now)
if current_time.month == 12:
target_year = current_time.year + 1
target_month = 1
else:
target_year = current_time.year
target_month = current_time.month + value
# Calculate target month and year, handling overflow past December
total_months = current_time.month - 1 + value # 0-indexed months
target_year = current_time.year + total_months // 12
target_month = total_months % 12 + 1 # back to 1-indexed
# Determine the day to set for next month
target_day = current_time.day

27
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -2,7 +2,7 @@ import asyncio
import json
import time
import traceback
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast
import litellm
from litellm._logging import verbose_logger
@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.types.llms.databricks import DatabricksTool
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
ImageURLListItem,
OpenAIModerationResponse,
)
from litellm.types.utils import (
@ -26,13 +27,13 @@ from litellm.types.utils import (
Function,
HiddenParams,
ImageResponse,
PromptTokensDetailsWrapper,
)
from litellm.types.utils import Logprobs as TextCompletionLogprobs
from litellm.types.utils import (
Message,
ModelResponse,
ModelResponseStream,
PromptTokensDetailsWrapper,
RerankResponse,
StreamingChoices,
TextChoices,
@ -52,6 +53,24 @@ _MODEL_RESPONSE_FIELDS: frozenset = frozenset(ModelResponse.model_fields.keys())
}
def _normalize_images_for_message(
images: Optional[List[dict]],
) -> Optional[List[ImageURLListItem]]:
"""
Ensure each image has an 'index' field, as required by ImageURLListItem.
Some providers (e.g. OpenRouter) return images without index.
"""
if not images:
return cast(Optional[List[ImageURLListItem]], images)
normalized: List[ImageURLListItem] = []
for i, img in enumerate(images):
if isinstance(img, dict) and "index" not in img:
normalized.append(cast(ImageURLListItem, {**img, "index": i}))
else:
normalized.append(cast(ImageURLListItem, img))
return normalized
def _safe_convert_created_field(created_value) -> int:
"""
Safely convert a 'created' field value to an integer.
@ -591,7 +610,9 @@ def convert_to_model_response_object( # noqa: PLR0915
reasoning_content=reasoning_content,
thinking_blocks=thinking_blocks,
annotations=choice["message"].get("annotations", None),
images=choice["message"].get("images", None),
images=_normalize_images_for_message(
choice["message"].get("images", None)
),
)
finish_reason = choice.get("finish_reason", None)
if finish_reason is None:

70
litellm/litellm_core_utils/redact_messages.py
View file

@ -73,6 +73,53 @@ def _redact_responses_api_output(output_items):
summary_item.text = "redacted-by-litellm"
def _redact_standard_logging_object(model_call_details: dict):
"""Redact messages and response inside standard_logging_object if present."""
standard_logging_object = model_call_details.get("standard_logging_object")
if standard_logging_object is None:
return
redacted_str = "redacted-by-litellm"
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [
{"role": "user", "content": redacted_str}
]
response = standard_logging_object.get("response")
if response is not None:
if isinstance(response, dict) and "output" in response:
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
for output_item in response["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
for choice in response["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
elif isinstance(response, str):
standard_logging_object["response"] = redacted_str
else:
# For other formats (empty dict, None, etc.), use simple text format
standard_logging_object["response"] = {"text": redacted_str}
def perform_redaction(model_call_details: dict, result):
"""
Performs the actual redaction on the logging object and result.
@ -114,6 +161,29 @@ def perform_redaction(model_call_details: dict, result):
if hasattr(_result, "choices") and _result.choices is not None:
for choice in _result.choices:
_redact_choice_content(choice)
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
for choice in _result["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["message"]:
choice["message"]["reasoning_content"] = "redacted-by-litellm"
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["delta"]:
choice["delta"]["reasoning_content"] = "redacted-by-litellm"
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)

6
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -476,13 +476,15 @@ class ChunkProcessor:
"prompt_tokens_details": prompt_tokens_details,
}
def count_reasoning_tokens(self, response: ModelResponse) -> int:
reasoning_tokens = 0
def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]:
reasoning_tokens: Optional[int] = None
for choice in response.choices:
if (
hasattr(cast(Choices, choice).message, "reasoning_content")
and cast(Choices, choice).message.reasoning_content is not None
):
if reasoning_tokens is None:
reasoning_tokens = 0
reasoning_tokens += token_counter(
text=cast(Choices, choice).message.reasoning_content,
count_response_tokens=True,

18
litellm/llms/anthropic/chat/transformation.py
View file

@ -317,6 +317,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else:
result[key] = value
# Anthropic requires additionalProperties=false for object schemas
# See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
if result.get("type") == "object" and "additionalProperties" not in result:
result["additionalProperties"] = False
return result
def get_json_schema_from_pydantic_object(
@ -770,6 +775,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if json_schema is None:
return None
# Resolve $ref/$defs before filtering — Anthropic doesn't support
# external schema references (e.g., /$defs/CalendarEvent).
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema = self.filter_anthropic_output_schema(json_schema)

4
litellm/llms/anthropic/skills/transformation.py
View file

@ -77,8 +77,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
api_base = AnthropicModelInfo.get_api_base()
if skill_id:
return f"{api_base}/v1/skills/{skill_id}?beta=true"
return f"{api_base}/v1/{endpoint}?beta=true"
return f"{api_base}/v1/skills/{skill_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_skill_request(
self,

45
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -4,7 +4,10 @@ from typing import List
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.chat.gpt_5_transformation import (
OpenAIGPT5Config,
_get_effort_level,
)
from litellm.types.llms.openai import AllMessageValues
from .gpt_transformation import AzureOpenAIConfig
@ -15,6 +18,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Override to handle gpt5_series/ prefix used for Azure routing.
The parent class calls ``_supports_factory(model, custom_llm_provider=None)``
which fails to resolve ``gpt5_series/gpt-5.1`` to the correct Azure model
entry. Strip the prefix and prepend ``azure/`` so the lookup finds
``azure/gpt-5.1`` in model_prices_and_context_window.json.
"""
if model.startswith(cls.GPT5_SERIES_ROUTE):
model = "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
elif not model.startswith("azure/"):
model = "azure/" + model
return super()._supports_reasoning_effort_level(model, level)
@classmethod
def is_model_gpt_5_model(cls, model: str) -> bool:
"""Check if the Azure model string refers to a gpt-5 variant.
@ -46,7 +64,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
# Only gpt-5.2+ has been verified to support logprobs on Azure.
# The base OpenAI class includes logprobs for gpt-5.1+, but Azure
# hasn't verified support for gpt-5.1, so remove them unless gpt-5.2/5.4+.
if self.is_model_gpt_5_1_model(model) and not self.is_model_gpt_5_2_model(model):
if self._supports_reasoning_effort_level(model, "none") and not self.is_model_gpt_5_2_model(model):
params = [p for p in params if p not in ["logprobs", "top_logprobs"]]
elif self.is_model_gpt_5_2_model(model):
azure_supported_params = ["logprobs", "top_logprobs"]
@ -66,20 +84,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
effective_effort = _get_effort_level(reasoning_effort_value)
# gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't
# See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
is_gpt_5_1 = self.is_model_gpt_5_1_model(model)
supports_none = self._supports_reasoning_effort_level(model, "none")
if reasoning_effort_value == "none" and not is_gpt_5_1:
if effective_effort == "none" and not supports_none:
if litellm.drop_params is True or (
drop_params is not None and drop_params is True
):
non_default_params = non_default_params.copy()
optional_params = optional_params.copy()
if non_default_params.get("reasoning_effort") == "none":
if _get_effort_level(non_default_params.get("reasoning_effort")) == "none":
non_default_params.pop("reasoning_effort")
if optional_params.get("reasoning_effort") == "none":
if _get_effort_level(optional_params.get("reasoning_effort")) == "none":
optional_params.pop("reasoning_effort")
else:
raise UnsupportedParamsError(
@ -101,10 +120,20 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
drop_params=drop_params,
)
# Only drop reasoning_effort='none' for non-gpt-5.1/5.2/5.4 models
if result.get("reasoning_effort") == "none" and not is_gpt_5_1:
# Only drop reasoning_effort='none' for models that don't support it
result_effort = _get_effort_level(result.get("reasoning_effort"))
if result_effort == "none" and not supports_none:
result.pop("reasoning_effort")
# Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together.
# Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not).
if self.is_model_gpt_5_4_plus_model(model):
has_tools = bool(
non_default_params.get("tools") or optional_params.get("tools")
)
if has_tools and result_effort not in (None, "none"):
result.pop("reasoning_effort", None)
return result
def transform_request(

30
litellm/llms/azure_ai/cost_calculator.py
View file

@ -61,7 +61,10 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
def cost_per_token(
model: str, usage: Usage, response_time_ms: Optional[float] = 0.0
model: str,
usage: Usage,
response_time_ms: Optional[float] = 0.0,
request_model: Optional[str] = None,
) -> Tuple[float, float]:
"""
Calculate the cost per token for Azure AI models.
@ -71,9 +74,10 @@ def cost_per_token(
- Plus the cost of the actual model used (handled by generic_cost_per_token)
Args:
model: str, the model name without provider prefix
model: str, the model name without provider prefix (from response)
usage: LiteLLM Usage block
response_time_ms: Optional response time in milliseconds
request_model: Optional[str], the original request model name (to detect router usage)
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
@ -84,7 +88,13 @@ def cost_per_token(
"""
prompt_cost = 0.0
completion_cost = 0.0
# Determine if this was a model router request
# Check both the response model and the request model
is_router_request = _is_azure_model_router(model) or (
request_model is not None and _is_azure_model_router(request_model)
)
# Calculate base cost using generic cost calculator
# This may raise an exception if the model is not in the cost map
try:
@ -103,19 +113,21 @@ def cost_per_token(
verbose_logger.debug(
f"Azure AI Model Router: model '{model}' not in cost map, calculating routing flat cost only. Error: {e}"
)
# Add flat cost for Azure Model Router
# The flat cost is defined in model_prices_and_context_window.json for azure_ai/model_router
if _is_azure_model_router(model):
router_flat_cost = calculate_azure_model_router_flat_cost(model, usage.prompt_tokens)
if is_router_request:
# Use the request model for flat cost calculation if available, otherwise use response model
router_model_for_calc = request_model if request_model else model
router_flat_cost = calculate_azure_model_router_flat_cost(router_model_for_calc, usage.prompt_tokens)
if router_flat_cost > 0:
verbose_logger.debug(
f"Azure AI Model Router flat cost: ${router_flat_cost:.6f} "
f"({usage.prompt_tokens} tokens × ${router_flat_cost / usage.prompt_tokens:.9f}/token)"
)
# Add flat cost to prompt cost
prompt_cost += router_flat_cost
return prompt_cost, completion_cost

187
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -334,24 +334,67 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
"""
Parse direct JSON response (non-streaming).
JSON response structure:
{
"result": {
"role": "assistant",
"content": [{"text": "..."}]
}
}
Supports multiple agent response schemas:
1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
2. {"response": [{"text": "..."}]} - Strands agent format
3. {"result": "plain text"} or {"response": "plain text"} - simple string
4. Fallback: raw JSON as content string
"""
result = response_json.get("result", {})
# Guard: if json.loads() returned a non-dict (e.g. array or primitive),
# skip strategy matching and fall back to raw JSON string
if not isinstance(response_json, dict):
verbose_logger.warning(
"AgentCore: JSON response is not a dict. "
"Returning raw JSON as content."
)
return AgentCoreParsedResponse(
content=json.dumps(response_json),
usage=None,
final_message=None,
)
# Extract content using the same helper as SSE parsing
content = self._extract_content_from_message(result) # type: ignore
# Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
if "result" in response_json and isinstance(response_json["result"], dict):
result = response_json["result"]
content = self._extract_content_from_message(result) # type: ignore
return AgentCoreParsedResponse(
content=content,
usage=None,
final_message=result, # type: ignore
)
# JSON responses don't include usage data
# Strategy 2: {"response": [{"text": "..."}]} - Strands agent content blocks
if "response" in response_json and isinstance(
response_json["response"], list
):
content = self._extract_content_from_message(
{"content": response_json["response"]} # type: ignore
)
return AgentCoreParsedResponse(
content=content,
usage=None,
final_message=None,
)
# Strategy 3: string values - {"result": "text"} or {"response": "text"}
for key in ("result", "response"):
val = response_json.get(key)
if isinstance(val, str):
return AgentCoreParsedResponse(
content=val,
usage=None,
final_message=None,
)
# Strategy 4: fallback - return raw JSON as content
verbose_logger.warning(
f"AgentCore: Could not extract content from JSON response keys "
f"{list(response_json.keys())}. Returning raw JSON as content."
)
return AgentCoreParsedResponse(
content=content,
content=json.dumps(response_json),
usage=None,
final_message=result, # type: ignore
final_message=None,
)
def _get_parsed_response(
@ -589,7 +632,64 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
additional_args={"complete_input_dict": data},
)
# Wrap the generator in CustomStreamWrapper
# Check if response is JSON (agent used sync return) instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
verbose_logger.debug(
"AgentCore streaming: received JSON response instead of SSE, "
"converting to single-chunk stream"
)
try:
body = response.read()
response_json = json.loads(body)
except (json.JSONDecodeError, Exception) as e:
raise BedrockError(
status_code=response.status_code,
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
)
parsed = self._parse_json_response(response_json)
def _json_as_sync_stream():
# Content chunk
content_chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
object="chat.completion.chunk",
)
content_chunk.choices = [
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content=parsed["content"], role="assistant"),
)
]
yield content_chunk
# Stop sentinel chunk (matches SSE path convention)
stop_chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
object="chat.completion.chunk",
)
stop_chunk.choices = [
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(),
)
]
yield stop_chunk
return CustomStreamWrapper(
completion_stream=_json_as_sync_stream(),
model=model,
custom_llm_provider="bedrock",
logging_obj=logging_obj,
)
# SSE stream (text/event-stream or default) - use existing SSE parser
return CustomStreamWrapper(
completion_stream=self._stream_agentcore_response_sync(response, model),
model=model,
@ -746,7 +846,64 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
additional_args={"complete_input_dict": data},
)
# Wrap the async generator in CustomStreamWrapper
# Check if response is JSON (agent used sync return) instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
verbose_logger.debug(
"AgentCore streaming: received JSON response instead of SSE, "
"converting to single-chunk stream"
)
try:
body = await response.aread()
response_json = json.loads(body)
except (json.JSONDecodeError, Exception) as e:
raise BedrockError(
status_code=response.status_code,
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
)
parsed = self._parse_json_response(response_json)
async def _json_as_async_stream() -> AsyncGenerator[ModelResponseStream, None]:
# Content chunk
content_chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
object="chat.completion.chunk",
)
content_chunk.choices = [
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content=parsed["content"], role="assistant"),
)
]
yield content_chunk
# Stop sentinel chunk (matches SSE path convention)
stop_chunk = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
object="chat.completion.chunk",
)
stop_chunk.choices = [
StreamingChoices(
finish_reason="stop",
index=0,
delta=Delta(),
)
]
yield stop_chunk
return CustomStreamWrapper(
completion_stream=_json_as_async_stream(),
model=model,
custom_llm_provider="bedrock",
logging_obj=logging_obj,
)
# SSE stream (text/event-stream or default) - use existing SSE parser
return CustomStreamWrapper(
completion_stream=self._stream_agentcore_response(response, model),
model=model,

28
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -51,6 +51,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
CompletionTokensDetailsWrapper,
Function,
Message,
ModelResponse,
@ -63,6 +64,7 @@ from litellm.utils import (
has_tool_call_blocks,
last_assistant_with_tool_calls_has_no_thinking_blocks,
supports_reasoning,
token_counter,
)
from ..common_utils import (
@ -1206,6 +1208,7 @@ class AmazonConverseConfig(BaseConfig):
self._validate_request_metadata(request_metadata)
output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None)
inference_params.pop("output_config", None) # Bedrock Converse doesn't support it
# keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
additional_request_params = {
@ -1620,7 +1623,11 @@ class AmazonConverseConfig(BaseConfig):
thinking_blocks_list.append(_redacted_block)
return thinking_blocks_list
def _transform_usage(self, usage: ConverseTokenUsageBlock) -> Usage:
def _transform_usage(
self,
usage: ConverseTokenUsageBlock,
reasoning_content: Optional[str] = None,
) -> Usage:
input_tokens = usage["inputTokens"]
output_tokens = usage["outputTokens"]
total_tokens = usage["totalTokens"]
@ -1637,6 +1644,19 @@ class AmazonConverseConfig(BaseConfig):
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens
)
reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else 0
)
completion_tokens_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens,
text_tokens=(
output_tokens - reasoning_tokens
if reasoning_tokens > 0
else output_tokens
),
)
openai_usage = Usage(
prompt_tokens=input_tokens,
completion_tokens=output_tokens,
@ -1644,6 +1664,7 @@ class AmazonConverseConfig(BaseConfig):
prompt_tokens_details=prompt_tokens_details,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
completion_tokens_details=completion_tokens_details,
)
return openai_usage
@ -1980,7 +2001,10 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message["tool_calls"] = filtered_tools
## CALCULATING USAGE - bedrock returns usage in the headers
usage = self._transform_usage(completion_response["usage"])
usage = self._transform_usage(
completion_response["usage"],
reasoning_content=chat_completion_message.get("reasoning_content"),
)
## HANDLE TOOL CALLS
_message = Message(**chat_completion_message)

3
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -108,6 +108,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
_anthropic_request.pop("stream", None)
# Bedrock Invoke doesn't support output_format parameter
_anthropic_request.pop("output_format", None)
# Bedrock Invoke doesn't support output_config parameter
# Fixes: https://github.com/BerriAI/litellm/issues/22797
_anthropic_request.pop("output_config", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version

4
litellm/llms/bedrock/image_edit/stability_transformation.py
View file

@ -22,7 +22,6 @@ API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parame
"""
import base64
import json
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
import httpx
@ -285,8 +284,6 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
"""
try:
response_data = raw_response.json()
with open("response_data.json", "w") as f:
json.dump(response_data, f)
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Bedrock Stability response: {e}",
@ -396,4 +393,3 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
headers["Content-Type"] = "application/json"
return headers

4
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -419,6 +419,10 @@ class AmazonAnthropicClaudeMessagesConfig(
anthropic_messages_request=anthropic_messages_request,
)
# 5b. Strip `output_config` — Bedrock Invoke doesn't support it
# Fixes: https://github.com/BerriAI/litellm/issues/22797
anthropic_messages_request.pop("output_config", None)
# 5a. Remove `custom` field from tools (Bedrock doesn't support it)
# Claude Code sends `custom: {defer_loading: true}` on tool definitions,
# which causes Bedrock to reject the request with "Extra inputs are not permitted"

5
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -426,8 +426,11 @@ class FireworksAIConfig(OpenAIGPTConfig):
"FIREWORKS_ACCOUNT_ID is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint."
)
base = api_base.rstrip("/")
if base.endswith("/v1"):
base = base[: -len("/v1")]
response = litellm.module_level_client.get(
url=f"{api_base}/v1/accounts/{account_id}/models",
url=f"{base}/v1/accounts/{account_id}/models",
headers={"Authorization": f"Bearer {api_key}"},
)

167
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -1,12 +1,46 @@
"""Support for OpenAI gpt-5 model family."""
from typing import Optional
from typing import Optional, Union
import litellm
from litellm.utils import _supports_factory
from .gpt_transformation import OpenAIGPTConfig
def _normalize_reasoning_effort_for_chat_completion(
value: Union[str, dict, None],
) -> Optional[str]:
"""Convert reasoning_effort to the string format expected by OpenAI chat completion API.
The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'.
Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
"""
if value is None:
return None
if isinstance(value, str):
return value
if isinstance(value, dict) and "effort" in value:
return value["effort"]
return None
def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]:
"""Extract the effective effort level from reasoning_effort (string or dict).
Use this for guards that compare effort level (e.g. xhigh validation, "none" checks).
Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly
treated as effort="none" for validation purposes.
"""
if value is None:
return None
if isinstance(value, str):
return value
if isinstance(value, dict) and "effort" in value:
return value["effort"]
return None
class OpenAIGPT5Config(OpenAIGPTConfig):
"""Configuration for gpt-5 models including GPT-5-Codex variants.
@ -40,47 +74,45 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
"""Check if the model is specifically a GPT-5 Codex variant."""
return "gpt-5-codex" in model
@classmethod
def is_model_gpt_5_1_codex_max_model(cls, model: str) -> bool:
"""Check if the model is the gpt-5.1-codex-max variant."""
model_name = model.split("/")[-1] # handle provider prefixes
return model_name == "gpt-5.1-codex-max"
@classmethod
def is_model_gpt_5_1_model(cls, model: str) -> bool:
"""Check if the model is a gpt-5.1, gpt-5.2, or gpt-5.4 chat variant.
gpt-5.1/5.2/5.4 support temperature when reasoning_effort="none",
unlike base gpt-5 which only supports temperature=1. Excludes
pro variants which keep stricter knobs and chat-only variants
which only support temperature=1.
"""
model_name = model.split("/")[-1]
is_gpt_5_1 = model_name.startswith("gpt-5.1")
is_gpt_5_2 = (
model_name.startswith("gpt-5.2")
and "pro" not in model_name
and not model_name.startswith("gpt-5.2-chat")
)
is_gpt_5_4 = (
model_name.startswith("gpt-5.4")
and "pro" not in model_name
and not model_name.startswith("gpt-5.4-chat")
)
return is_gpt_5_1 or is_gpt_5_2 or is_gpt_5_4
@classmethod
def is_model_gpt_5_2_pro_model(cls, model: str) -> bool:
"""Check if the model is the gpt-5.2-pro snapshot/alias."""
model_name = model.split("/")[-1]
return model_name.startswith("gpt-5.2-pro")
@classmethod
def is_model_gpt_5_2_model(cls, model: str) -> bool:
"""Check if the model is a gpt-5.2 variant (including pro)."""
model_name = model.split("/")[-1]
return model_name.startswith("gpt-5.2") or model_name.startswith("gpt-5.4")
@classmethod
def is_model_gpt_5_4_model(cls, model: str) -> bool:
"""Check if the model is a gpt-5.4 variant (including pro)."""
model_name = model.split("/")[-1]
return model_name.startswith("gpt-5.4")
@classmethod
def is_model_gpt_5_4_plus_model(cls, model: str) -> bool:
"""Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro)."""
model_name = model.split("/")[-1]
if not model_name.startswith("gpt-5."):
return False
try:
version_str = model_name.replace("gpt-5.", "").split("-")[0]
major = version_str.split(".")[0]
return int(major) >= 4
except (ValueError, IndexError):
return False
@classmethod
def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
"""Check if the model supports a specific reasoning_effort level.
Looks up ``supports_{level}_reasoning_effort`` in the model map via
the shared ``_supports_factory`` helper.
Returns False for unknown models (safe fallback).
"""
return _supports_factory(
model=model,
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",
)
def get_supported_openai_params(self, model: str) -> list:
if self.is_model_gpt_5_search_model(model):
return [
@ -118,8 +150,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
"web_search_options",
]
# gpt-5.1/5.2/5.4 support logprobs, top_p, top_logprobs when reasoning_effort="none"
if not self.is_model_gpt_5_1_model(model):
# gpt-5.1/5.2 support logprobs, top_p, top_logprobs when reasoning_effort="none"
if not self._supports_reasoning_effort_level(model, "none"):
non_supported_params.extend(["logprobs", "top_p", "top_logprobs"])
return [
@ -147,15 +179,33 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
drop_params=drop_params,
)
reasoning_effort = (
# Get raw reasoning_effort and effective effort level for all guards.
# Use effective_effort (extracted string) for xhigh validation, "none" checks, and
# tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"}
# must be treated as effort="none" to avoid incorrect tool-drop or sampling errors.
raw_reasoning_effort = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
if reasoning_effort is not None and reasoning_effort == "xhigh":
if not (
self.is_model_gpt_5_1_codex_max_model(model)
or self.is_model_gpt_5_2_model(model)
):
effective_effort = _get_effort_level(raw_reasoning_effort)
# Normalize to string for Chat Completions API when dict has only "effort".
# Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API.
if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}:
normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort)
if normalized is not None:
if "reasoning_effort" in non_default_params:
non_default_params["reasoning_effort"] = normalized
if "reasoning_effort" in optional_params:
optional_params["reasoning_effort"] = normalized
reasoning_effort = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
or raw_reasoning_effort
)
if effective_effort is not None and effective_effort == "xhigh":
if not self._supports_reasoning_effort_level(model, "xhigh"):
if litellm.drop_params or drop_params:
non_default_params.pop("reasoning_effort", None)
else:
@ -175,11 +225,26 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
"max_tokens"
)
# gpt-5.1/5.2/5.4 support logprobs, top_p, top_logprobs only when reasoning_effort="none"
if self.is_model_gpt_5_1_model(model):
# gpt-5.4: function calls not supported when reasoning_effort != "none"
# Drop reasoning_effort when tools are present (small minority of volume)
if self.is_model_gpt_5_4_model(model):
has_tools = bool(
non_default_params.get("tools") or optional_params.get("tools")
)
if has_tools and effective_effort not in (None, "none"):
# Check if this will be routed to Responses API
# If so, keep reasoning_effort; otherwise drop it for chat completions API
if not self.is_model_gpt_5_4_plus_model(model):
non_default_params.pop("reasoning_effort", None)
optional_params.pop("reasoning_effort", None)
reasoning_effort = None
# gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none"
supports_none = self._supports_reasoning_effort_level(model, "none")
if supports_none:
sampling_params = ["logprobs", "top_logprobs", "top_p"]
has_sampling = any(p in non_default_params for p in sampling_params)
if has_sampling and reasoning_effort not in (None, "none"):
if has_sampling and effective_effort not in (None, "none"):
if litellm.drop_params or drop_params:
for p in sampling_params:
non_default_params.pop(p, None)
@ -189,17 +254,15 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
"gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when "
"reasoning_effort='none'. Current reasoning_effort='{}'. "
"To drop unsupported params set `litellm.drop_params = True`"
).format(reasoning_effort),
).format(effective_effort),
status_code=400,
)
if "temperature" in non_default_params:
temperature_value: Optional[float] = non_default_params.pop("temperature")
if temperature_value is not None:
is_gpt_5_1 = self.is_model_gpt_5_1_model(model)
# gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none")
if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None):
# models supporting reasoning_effort="none" also support flexible temperature
if supports_none and (effective_effort == "none" or effective_effort is None):
optional_params["temperature"] = temperature_value
elif temperature_value == 1:
optional_params["temperature"] = temperature_value

5
litellm/llms/openai/chat/o_series_transformation.py
View file

@ -131,7 +131,10 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
def is_model_o_series_model(self, model: str) -> bool:
model = model.split("/")[-1] # could be "openai/o3" or "o3"
return model.startswith(("o1", "o3", "o4")) and model in litellm.open_ai_chat_completion_models
return (
len(model) > 1 and model[0] == "o" and model[1].isdigit()
and model in litellm.open_ai_chat_completion_models
)
@overload
def _transform_messages(

7
litellm/llms/openai_like/providers.json
View file

@ -94,5 +94,12 @@
"assemblyai": {
"base_url": "https://llm-gateway.assemblyai.com/v1",
"api_key_env": "ASSEMBLYAI_API_KEY"
},
"charity_engine": {
"base_url": "https://api.charityengine.services/remotejobs/v2/inference",
"api_key_env": "CHARITY_ENGINE_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
}
}

56
litellm/llms/sagemaker/completion/handler.py
View file

@ -583,35 +583,17 @@ class SagemakerLLM(BaseAWSLLM):
### BOTO3 INIT
import boto3
# pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them
aws_secret_access_key = optional_params.pop("aws_secret_access_key", None)
aws_access_key_id = optional_params.pop("aws_access_key_id", None)
aws_region_name = optional_params.pop("aws_region_name", None)
# Use _load_credentials to support role assumption (aws_role_name, aws_session_name)
credentials, aws_region_name = self._load_credentials(optional_params)
if aws_access_key_id is not None:
# uses auth params passed to completion
# aws_access_key_id is not None, assume user is trying to auth using litellm.completion
client = boto3.client(
service_name="sagemaker-runtime",
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region_name,
)
else:
# aws_access_key_id is None, assume user is trying to auth using env variables
# boto3 automaticaly reads env variables
# we need to read region name from env
# I assume majority of users use .env for auth
region_name = (
get_secret("AWS_REGION_NAME")
or aws_region_name # get region from config file if specified
or "us-west-2" # default to us-west-2 if region not specified
)
client = boto3.client(
service_name="sagemaker-runtime",
region_name=region_name,
)
# Create boto3 session with the loaded credentials
session = boto3.Session(
aws_access_key_id=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
aws_session_token=credentials.token,
region_name=aws_region_name,
)
client = session.client(service_name="sagemaker-runtime")
# pop streaming if it's in the optional params as 'stream' raises an error with sagemaker
inference_params = deepcopy(optional_params)
@ -628,7 +610,9 @@ class SagemakerLLM(BaseAWSLLM):
#### EMBEDDING LOGIC
# Transform request based on model type
provider_config = SagemakerEmbeddingConfig.get_model_config(model)
request_data = provider_config.transform_embedding_request(model, input, optional_params, {})
request_data = provider_config.transform_embedding_request(
model, input, optional_params, {}
)
data = json.dumps(request_data).encode("utf-8")
## LOGGING
@ -673,19 +657,19 @@ class SagemakerLLM(BaseAWSLLM):
)
print_verbose(f"raw model_response: {response}")
# Transform response based on model type
from httpx import Response as HttpxResponse
# Create a mock httpx Response object for the transformation
mock_response = HttpxResponse(
status_code=200,
content=json.dumps(response).encode('utf-8'),
headers={"content-type": "application/json"}
content=json.dumps(response).encode("utf-8"),
headers={"content-type": "application/json"},
)
model_response = EmbeddingResponse()
# Use the request_data that was already transformed above
return provider_config.transform_embedding_response(
model=model,
@ -695,5 +679,5 @@ class SagemakerLLM(BaseAWSLLM):
api_key=None,
request_data=request_data,
optional_params=optional_params,
litellm_params=litellm_params or {}
litellm_params=litellm_params or {},
)

2
litellm/llms/searchapi/search/transformation.py
View file

@ -159,7 +159,7 @@ class SearchAPIConfig(BaseSearchConfig):
domains = optional_params["search_domain_filter"]
if isinstance(domains, list) and len(domains) > 0:
result_data["q"] = self._append_domain_filters(
result_data["q"], domains
str(result_data["q"]), domains
)
if "country" in optional_params:

6
litellm/llms/serper/search/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""
Serper Search API module.
"""
from litellm.llms.serper.search.transformation import SerperSearchConfig
__all__ = ["SerperSearchConfig"]

167
litellm/llms/serper/search/transformation.py Normal file
View file

@ -0,0 +1,167 @@
"""
Calls Serper's /search endpoint to search Google.
Serper API Reference: https://serper.dev
"""
from typing import Dict, List, Optional, TypedDict, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
class _SerperSearchRequestRequired(TypedDict):
"""Required fields for Serper Search API request."""
q: str # Required - search query
class SerperSearchRequest(_SerperSearchRequestRequired, total=False):
"""
Serper Search API request format.
Based on: https://serper.dev
"""
num: int # Optional - number of results to return, default 10
page: int # Optional - page number (default 1)
gl: str # Optional - country/geolocation code (e.g., "us", "gb")
hl: str # Optional - language code (e.g., "en", "de")
location: str # Optional - specific location for search targeting
autocorrect: bool # Optional - enable autocorrect (default True)
tbs: str # Optional - time-based search filter (e.g., "qdr:h", "qdr:d", "qdr:w")
class SerperSearchConfig(BaseSearchConfig):
SERPER_API_BASE = "https://google.serper.dev"
@staticmethod
def ui_friendly_name() -> str:
return "Serper"
def validate_environment(
self,
headers: Dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
**kwargs,
) -> Dict:
"""
Validate environment and return headers.
"""
api_key = api_key or get_secret_str("SERPER_API_KEY")
if not api_key:
raise ValueError("SERPER_API_KEY is not set. Set `SERPER_API_KEY` environment variable.")
headers["X-API-KEY"] = api_key
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(
self,
api_base: Optional[str],
optional_params: dict,
data: Optional[Union[Dict, List[Dict]]] = None,
**kwargs,
) -> str:
"""
Get complete URL for Search endpoint.
"""
api_base = api_base or get_secret_str("SERPER_API_BASE") or self.SERPER_API_BASE
api_base = api_base.rstrip("/")
if not api_base.endswith("/search"):
api_base = f"{api_base}/search"
return api_base
def transform_search_request(
self,
query: Union[str, List[str]],
optional_params: dict,
**kwargs,
) -> Dict:
"""
Transform Search request to Serper API format.
Args:
query: Search query (string or list of strings). Serper only supports single string queries.
optional_params: Optional parameters for the request
- max_results: Maximum number of search results -> maps to `num`
- search_domain_filter: List of domains -> appended as site: clauses to `q`
- country: Country code filter (e.g., 'US', 'GB') -> maps to `gl` (lowercased)
Returns:
Dict with typed request data following SerperSearchRequest spec
"""
if isinstance(query, list):
query = " ".join(query)
request_data: SerperSearchRequest = {
"q": query,
}
if "max_results" in optional_params:
request_data["num"] = optional_params["max_results"]
if "country" in optional_params:
request_data["gl"] = optional_params["country"].lower()
if "search_domain_filter" in optional_params:
domains = optional_params["search_domain_filter"]
if isinstance(domains, list) and len(domains) > 0:
domain_clauses = " OR ".join(f"site:{d}" for d in domains)
request_data["q"] = f"({request_data['q']}) ({domain_clauses})"
# Convert to dict before dynamic key assignments
result_data = dict(request_data)
# pass through all other parameters as-is
for param, value in optional_params.items():
if param not in self.get_supported_perplexity_optional_params() and param not in result_data:
result_data[param] = value
return result_data
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> SearchResponse:
"""
Transform Serper API response to LiteLLM unified SearchResponse format.
Serper -> LiteLLM mappings:
- organic[].title -> SearchResult.title
- organic[].link -> SearchResult.url
- organic[].snippet -> SearchResult.snippet
- organic[].date -> SearchResult.date (optional, not always present)
Args:
raw_response: Raw httpx response from Serper API
logging_obj: Logging object for tracking
Returns:
SearchResponse with standardized format
"""
response_json = raw_response.json()
results = []
for result in response_json.get("organic", []):
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("link", ""),
snippet=result.get("snippet", ""),
date=result.get("date"),
last_updated=None,
)
results.append(search_result)
return SearchResponse(
results=results,
object="search",
)

52
litellm/llms/vertex_ai/aws_credentials_supplier.py Normal file
View file

@ -0,0 +1,52 @@
"""
Custom AWS Security Credentials Supplier for Vertex AI WIF.
Wraps boto3/botocore credentials so that google-auth can use them
for the AWS-to-GCP Workload Identity Federation token exchange
without hitting the EC2 instance metadata service.
Requires google-auth >= 2.29.0.
"""
from typing import Callable
from google.auth import aws
class AwsCredentialsSupplier(aws.AwsSecurityCredentialsSupplier):
"""
Supplies AWS credentials to google-auth's aws.Credentials for WIF
token exchange.
This bypasses the default metadata-based credential retrieval,
allowing WIF to work in environments where EC2 metadata is blocked.
Accepts a credentials_provider callable that is invoked on every
get_aws_security_credentials() call, so that refreshed/rotated
credentials are picked up automatically (important for temporary
STS tokens).
"""
def __init__(self, credentials_provider: Callable, aws_region: str):
"""
Args:
credentials_provider: A zero-arg callable that returns a
botocore.credentials.Credentials object (with access_key,
secret_key, and token attributes).
aws_region: The AWS region string (e.g. "us-east-1").
"""
self._credentials_provider = credentials_provider
self._region = aws_region
def get_aws_security_credentials(self, context, request):
"""Return current AWS credentials for the GCP token exchange."""
current = self._credentials_provider()
return aws.AwsSecurityCredentials(
access_key_id=current.access_key,
secret_access_key=current.secret_key,
session_token=current.token,
)
def get_aws_region(self, context, request):
"""Return the AWS region for credential verification."""
return self._region

36
litellm/llms/vertex_ai/common_utils.py
View file

@ -571,38 +571,14 @@ def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
return schema_dict
def _is_any_type_schema(schema: dict) -> bool:
"""
Detect schemas that represent "any JSON value" (no type constraints).
In JSON Schema, an empty schema {} means "any value is valid".
Schemas with only metadata keys (title, description, default, examples)
but no type-constraining keywords also represent "any type".
Gemini's Schema proto uses TYPE_UNSPECIFIED (0) as default,
so omitting the type field is valid and means "any type".
"""
type_constraining_keys = {
"type",
"properties",
"items",
"anyOf",
"oneOf",
"allOf",
"enum",
"required",
"$ref",
"$schema",
}
return not any(key in type_constraining_keys for key in schema.keys())
def process_items(schema, depth=0):
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
)
if isinstance(schema, dict):
if "items" in schema and schema["items"] == {}:
schema["items"] = {"type": "object"}
for key, value in schema.items():
if isinstance(value, dict):
process_items(value, depth + 1)
@ -701,8 +677,9 @@ def convert_anyof_null_to_nullable(schema, depth=0):
# remove null type
anyof.remove(atype)
contains_null = True
elif isinstance(atype, dict) and _is_any_type_schema(atype):
pass # preserve "any type" semantics — don't coerce to object
elif "type" not in atype and len(atype) == 0:
# Handle empty object case
atype["type"] = "object"
if len(anyof) == 0:
# Edge case: response schema with only null type present is invalid in Vertex AI
@ -737,8 +714,7 @@ def add_object_type(schema):
# Gemini requires all function parameters to be type OBJECT
# Handle case where schema has no properties and no type (e.g. tools with no arguments)
if "type" not in schema and "anyOf" not in schema and "oneOf" not in schema and "allOf" not in schema:
if not _is_any_type_schema(schema):
schema["type"] = "object"
schema["type"] = "object"
properties = schema.get("properties", None)
if properties is not None:

6
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -529,12 +529,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
raise e
# Keys that LiteLLM consumes internally and must never be forwarded to the
_LITELLM_INTERNAL_EXTRA_BODY_KEYS: frozenset = frozenset({"cache", "tags"})
def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None:
"""Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values."""
extra_body: Optional[dict] = optional_params.pop("extra_body", None)
if extra_body is not None:
data_dict: dict = data # type: ignore[assignment]
for k, v in extra_body.items():
if k in _LITELLM_INTERNAL_EXTRA_BODY_KEYS:
continue
if k in data_dict and isinstance(data_dict[k], dict) and isinstance(v, dict):
data_dict[k].update(v)
else:

5
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -800,9 +800,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
GeminiThinkingConfig with thinkingLevel and includeThoughts
"""
# Check if this is gemini-3-flash which supports MINIMAL thinking level
# Covers gemini-3-flash, gemini-3-flash-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview, etc.
is_gemini3flash = model and (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
"gemini-3-flash" in model.lower()
or "gemini-3.1-flash" in model.lower()
)
is_gemini31pro = model and (
"gemini-3.1-pro-preview" in model.lower()

Some files were not shown because too many files have changed in this diff Show more