scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

Merge branch 'fix-spend-logs' of https://github.com/OrionCodeDev/litellm into fix-spend-logs

Browse source
This commit is contained in:
OrionCodeDev 2026-02-12 03:28:14 +01:00
commit d68970c705
280 changed files with 18431 additions and 6281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.circleci/config.yml
View file

@ -2277,6 +2277,7 @@ jobs:
- run: python ./tests/code_coverage_tests/router_code_coverage.py
- run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
- run: python ./tests/code_coverage_tests/info_log_check.py
- run: python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
- run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
- run: python ./tests/code_coverage_tests/code_qa_check_tests.py
- run: python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
@ -3801,7 +3802,6 @@ jobs:
- run:
name: Get new version
command: |
cd litellm-proxy-extras
NEW_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
@ -3826,7 +3826,6 @@ jobs:
- run:
name: Publish to PyPI
command: |
cd litellm-proxy-extras
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
python -m pip install --upgrade pip build twine setuptools wheel
rm -rf build dist

2
.dockerignore
View file

@ -48,7 +48,7 @@ dist/
build/
*.egg-info/
.DS_Store
node_modules/
**/node_modules
*.log
.env
.env.local

31
Dockerfile
View file

@ -49,7 +49,22 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
# We must find and replace EVERY copy inside npm's directory.
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -71,6 +86,20 @@ RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/nul
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable
RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

33
ci_cd/.grype.yaml
View file

@ -1,3 +1,36 @@
ignore:
- vulnerability: CVE-2026-22184
reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
# Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
- vulnerability: CVE-2025-55130
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59465
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55131
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-59466
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2026-21637
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: CVE-2025-55132
reason: Node in Wolfi apk; only used for Admin UI build/prisma
- vulnerability: GHSA-hx9q-6w63-j58v
reason: orjson dumps recursion; allowlisted
- vulnerability: GHSA-73rr-hh4g-fpgx
reason: diff npm transitive dep; override in package.json, allowlisted
- vulnerability: CVE-2026-0865
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15282
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-0672
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15366
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-15367
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-11468
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2025-12781
reason: Python 3.13 in Wolfi base; no fixed apk build yet
- vulnerability: CVE-2026-1299
reason: Python 3.13 in Wolfi base; no fixed apk build yet

19
ci_cd/security_scans.sh
View file

@ -140,12 +140,14 @@ run_grype_scans() {
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55131" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59466" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-55130" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59467" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2026-21637" # We do not use Node in application runtime, only used for building Admin UI
"CVE-2025-59465" # Node only used for Admin UI build/prisma
"CVE-2025-55131" # Node only used for Admin UI build/prisma
"CVE-2025-59466" # Node only used for Admin UI build/prisma
"CVE-2025-55130" # Node only used for Admin UI build/prisma
"CVE-2025-59467" # Node only used for Admin UI build/prisma
"CVE-2026-21637" # Node only used for Admin UI build/prisma
"CVE-2025-55132" # Node only used for Admin UI build/prisma
"GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
"CVE-2025-15281" # No fix available yet
"CVE-2026-0865" # No fix available yet
"CVE-2025-15282" # No fix available yet
@ -155,10 +157,7 @@ run_grype_scans() {
"CVE-2025-12781" # No fix available yet
"CVE-2025-11468" # No fix available yet
"CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
"GHSA-7h2j-956f-4vf2" # @isaacs/brace-expansion ReDoS - npm tooling dependency, not used in application runtime
"GHSA-hx9q-6w63-j58v" # orjson deep recursion - no fix available yet
"GHSA-8qq5-rm4j-mr97" # node-tar symlink poisoning - npm tooling dependency, tar CLI not exposed in application code
"GHSA-29xp-372q-xqph" # node-tar race condition - npm tooling dependency, tar CLI not exposed in application code
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
)
# Build JSON array of allowlisted CVE IDs for jq

13
docker/Dockerfile.custom_ui
View file

@ -6,7 +6,18 @@ WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get install -y nodejs npm && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

29
docker/Dockerfile.database
View file

@ -50,7 +50,18 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@latest
npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
npm cache clean --force
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -64,9 +75,19 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Install semantic_router and aurelio-sdk using script
# Convert Windows line endings to Unix and make executable

27
docker/Dockerfile.dev
View file

@ -62,7 +62,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@latest
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
WORKDIR /app
@ -80,6 +91,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
rm -f *.whl && \
rm -rf /wheels
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Generate prisma client and set permissions
# Convert Windows line endings to Unix for entrypoint scripts
RUN prisma generate && \

29
docker/Dockerfile.non_root
View file

@ -104,7 +104,18 @@ RUN for i in 1 2 3; do \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& npm install -g npm@latest tar@latest
&& npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& npm cache clean --force
# Copy artifacts from builder
COPY --from=builder /app/requirements.txt /app/requirements.txt
@ -146,9 +157,19 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
fi; \
fi
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
# Patch every copy of tar, glob, and brace-expansion inside that tree.
RUN GLOBAL="$(npm root -g)" && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done
# Permissions, cleanup, and Prisma prep
# Convert Windows line endings to Unix for entrypoint scripts

330
docs/my-website/blog/claude_opus_4_6/index.md
View file

@ -223,11 +223,16 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</TabItem>
</Tabs>
## Compaction
## Advanced Features
### Compaction
<Tabs>
<TabItem value="completions" label="/chat/completions">
Litellm supports enabling compaction for the new claude-opus-4-6.
### Enabling Compaction
**Enabling Compaction**
To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
@ -255,8 +260,43 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
```
All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
</TabItem>
<TabItem value="messages" label="/v1/messages">
### Response with Compaction Block
Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled.
:::info
**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs).
:::
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "Hi"
}
],
"context_management": {
"edits": [
{
"type": "compact_20260112"
}
]
}
}'
```
</TabItem>
</Tabs>
**Response with Compaction Block**
The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
@ -292,7 +332,7 @@ The response will include the compaction summary in `provider_specific_fields.co
}
```
### Using Compaction Blocks in Follow-up Requests
**Using Compaction Blocks in Follow-up Requests**
To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
@ -340,15 +380,17 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
### Streaming Support
**Streaming Support**
Compaction blocks are also supported in streaming mode. You'll receive:
- `compaction_start` event when a compaction block begins
- `compaction_delta` events with the compaction content
- The accumulated `compaction_blocks` in `provider_specific_fields`
### Adaptive Thinking
## Adaptive Thinking
<Tabs>
<TabItem value="completions" label="/chat/completions">
LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
@ -368,7 +410,37 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
## Effort Levels
</TabItem>
<TabItem value="messages" label="/v1/messages">
Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 16000,
"thinking": {
"type": "adaptive"
},
"messages": [
{
"role": "user",
"content": "Explain why the sum of two even numbers is always even."
}
]
}'
```
</TabItem>
</Tabs>
### Effort Levels
<Tabs>
<TabItem value="completions" label="/chat/completions">
Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
@ -387,17 +459,253 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"output_config": {
"effort": "medium"
}
}'
```
You can use reasoning effort plus output_config to have more control on the model.
## 1M Token Context (Beta)
</TabItem>
<TabItem value="messages" label="/v1/messages">
Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "medium"
}
}'
```
</TabItem>
</Tabs>
### 1M Token Context (Beta)
Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
## US-Only Inference
<Tabs>
<TabItem value="completions" label="/chat/completions">
Available at 1.1× token pricing. LiteLLM supports this pricing model.
To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
**Step 1: Enable header forwarding in your config**
```yaml
general_settings:
forward_client_headers_to_llm_api: true
```
**Step 2: Send requests with the beta header**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--header 'anthropic-beta: context-1m-2025-08-07' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Analyze this large document..."
}
]
}'
```
</TabItem>
<TabItem value="messages" label="/v1/messages">
To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
**Step 1: Enable header forwarding in your config**
```yaml
general_settings:
forward_client_headers_to_llm_api: true
```
**Step 2: Send requests with the beta header**
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'anthropic-beta: context-1m-2025-08-07' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 16000,
"messages": [
{
"role": "user",
"content": "Analyze this large document..."
}
]
}'
```
:::tip
You can combine multiple beta headers by separating them with commas:
```bash
--header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12'
```
:::
</TabItem>
</Tabs>
### US-Only Inference
Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference.
<Tabs>
<TabItem value="completions" label="/chat/completions">
Use the `inference_geo` parameter to specify US-only inference:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"inference_geo": "us"
}'
```
LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
</TabItem>
<TabItem value="messages" label="/v1/messages">
Use the `inference_geo` parameter to specify US-only inference:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
],
"inference_geo": "us"
}'
```
LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
</TabItem>
</Tabs>
### Fast Mode
:::info
Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock.
:::
**Pricing:**
- Standard: $5 input / $25 output per MTok
- Fast: $30 input / $150 output per MTok (6× premium)
<Tabs>
<TabItem value="completions" label="/chat/completions">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-6",
"messages": [
{
"role": "user",
"content": "Refactor this module..."
}
],
"max_tokens": 4096,
"speed": "fast"
}'
```
**Using OpenAI SDK:**
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-opus-4-6",
messages=[{"role": "user", "content": "Refactor this module..."}],
max_tokens=4096,
extra_body={"speed": "fast"}
)
```
**Using LiteLLM SDK:**
```python
from litellm import completion
response = completion(
model="anthropic/claude-opus-4-6",
messages=[{"role": "user", "content": "Refactor this module..."}],
max_tokens=4096,
speed="fast"
)
```
LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations.
</TabItem>
<TabItem value="messages" label="/v1/messages">
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-6",
"max_tokens": 4096,
"speed": "fast",
"messages": [
{
"role": "user",
"content": "Refactor this module..."
}
]
}'
```
LiteLLM automatically:
- Adds the `fast-mode-2026-02-01` beta header
- Tracks the 6× premium pricing in cost calculations
</TabItem>
</Tabs>

95
docs/my-website/blog/model_cost_map_incident/index.md Normal file
View file

@ -0,0 +1,95 @@
---
slug: model-cost-map-incident
title: "Incident Report: Invalid model cost map on main"
date: 2026-02-10T10:00:00
authors:
- name: Ishaan Jaffer
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/ishaanjaffer/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [incident-report, stability]
hide_table_of_contents: false
---
**Date:** January 27, 2026
**Duration:** ~20 minutes
**Severity:** Low
**Status:** Resolved
## Summary
A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
- **LLM calls and proxy routing:** No impact.
- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
{/* truncate */}
---
## Background
The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
```mermaid
flowchart TD
A["1. litellm.completion() receives request
litellm/main.py"] --> B["2. Route to provider
litellm/litellm_core_utils/get_llm_provider_logic.py"]
B --> C["3. LLM returns response
litellm/main.py"]
C --> D["4. Post-call: look up model in cost map
litellm/cost_calculator.py"]
D -->|"found"| E["5a. Attach cost to response"]
D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
E --> G["6. Return response to caller"]
F --> G
style D fill:#fff3cd,stroke:#ffc107
style F fill:#fff3cd,stroke:#ffc107
style E fill:#d4edda,stroke:#28a745
style G fill:#d4edda,stroke:#28a745
```
Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
---
## Root cause
LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
**Timeline:**
1. Malformed JSON merged to `main`
2. LiteLLM installations fall back to local backup on next import
3. Users report `"This model isn't mapped yet"` for newer models
4. Bad commit identified and reverted (~20 minutes)
---
## Remediation
| # | Action | Status | Code |
|---|---|---|---|
| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
---
## Other dependencies on external resources
| Dependency | Impact if unavailable | Fallback |
|---|---|---|
| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
| JWT public keys (IDP/SSO) | Auth fails | None |
| OIDC UserInfo (IDP/SSO) | Auth fails | None |
| HuggingFace model API | HF provider calls fail | None |
| Ollama tags (localhost) | Ollama model list stale | Static list |

411
docs/my-website/docs/integrations/websearch_interception.md Normal file
View file

@ -0,0 +1,411 @@
# Web Search Integration
Enable transparent server-side web search execution for any LLM provider. LiteLLM automatically intercepts web search tool calls and executes them using your configured search provider (Perplexity, Tavily, etc.).
## Quick Start
### 1. Configure Web Search Interception
Add to your `config.yaml`:
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- openai
- minimax
- anthropic
search_tool_name: perplexity-search # Optional
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
### 2. Use with Any Provider
```python
import litellm
response = await litellm.acompletion(
model="gpt-4o",
messages=[
{"role": "user", "content": "What's the weather in San Francisco today?"}
],
tools=[
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web for information",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query"}
},
"required": ["query"]
}
}
}
]
)
# Response includes search results automatically!
print(response.choices[0].message.content)
```
## How It Works
When a model makes a web search tool call, LiteLLM:
1. **Detects** the `litellm_web_search` tool call in the response
2. **Executes** the search using your configured search provider
3. **Makes a follow-up request** with the search results
4. **Returns** the final answer to the user
```mermaid
sequenceDiagram
participant User
participant LiteLLM
participant LLM as LLM Provider
participant Search as Search Provider
User->>LiteLLM: Request with web_search tool
LiteLLM->>LLM: Forward request
LLM-->>LiteLLM: Response with tool_call
Note over LiteLLM: Detect web search<br/>tool call
LiteLLM->>Search: Execute search
Search-->>LiteLLM: Search results
LiteLLM->>LLM: Follow-up with results
LLM-->>LiteLLM: Final answer
LiteLLM-->>User: Final answer with search results
```
**Result**: One API call from user → Complete answer with search results
## Supported Providers
Web search integration works with **all providers** that use:
- ✅ **Base HTTP Handler** (`BaseLLMHTTPHandler`)
- ✅ **OpenAI Completion Handler** (`OpenAIChatCompletion`)
### Providers Using Base HTTP Handler
| Provider | Status | Notes |
|----------|--------|-------|
| **OpenAI** | ✅ Supported | GPT-4, GPT-3.5, etc. |
| **Anthropic** | ✅ Supported | Claude models via HTTP handler |
| **MiniMax** | ✅ Supported | All MiniMax models |
| **Mistral** | ✅ Supported | Mistral AI models |
| **Cohere** | ✅ Supported | Command models |
| **Fireworks AI** | ✅ Supported | All Fireworks models |
| **Together AI** | ✅ Supported | All Together AI models |
| **Groq** | ✅ Supported | All Groq models |
| **Perplexity** | ✅ Supported | Perplexity models |
| **DeepSeek** | ✅ Supported | DeepSeek models |
| **xAI** | ✅ Supported | Grok models |
| **Hugging Face** | ✅ Supported | Inference API models |
| **OCI** | ✅ Supported | Oracle Cloud models |
| **Vertex AI** | ✅ Supported | Google Vertex AI models |
| **Bedrock** | ✅ Supported | AWS Bedrock models (converse_like route) |
| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI models |
| **Sagemaker** | ✅ Supported | AWS Sagemaker models |
| **Databricks** | ✅ Supported | Databricks models |
| **DataRobot** | ✅ Supported | DataRobot models |
| **Hosted VLLM** | ✅ Supported | Self-hosted VLLM |
| **Heroku** | ✅ Supported | Heroku-hosted models |
| **RAGFlow** | ✅ Supported | RAGFlow models |
| **Compactif** | ✅ Supported | Compactif models |
| **Cometapi** | ✅ Supported | Comet API models |
| **A2A** | ✅ Supported | Agent-to-Agent models |
| **Bytez** | ✅ Supported | Bytez models |
### Providers Using OpenAI Handler
| Provider | Status | Notes |
|----------|--------|-------|
| **OpenAI** | ✅ Supported | Native OpenAI API |
| **Azure OpenAI** | ✅ Supported | Azure-hosted OpenAI |
| **OpenAI-Compatible** | ✅ Supported | Any OpenAI-compatible API |
## Configuration
### WebSearch Interception Parameters
| Parameter | Type | Required | Description | Example |
|-----------|------|----------|-------------|---------|
| `enabled_providers` | List[String] | Yes | List of providers to enable web search for | `[openai, minimax, anthropic]` |
| `search_tool_name` | String | No | Specific search tool from `search_tools` config. If not set, uses first available. | `perplexity-search` |
### Provider Values
Use these values in `enabled_providers`:
| Provider | Value | Provider | Value |
|----------|-------|----------|-------|
| OpenAI | `openai` | Anthropic | `anthropic` |
| MiniMax | `minimax` | Mistral | `mistral` |
| Cohere | `cohere` | Fireworks AI | `fireworks_ai` |
| Together AI | `together_ai` | Groq | `groq` |
| Perplexity | `perplexity` | DeepSeek | `deepseek` |
| xAI | `xai` | Hugging Face | `huggingface` |
| OCI | `oci` | Vertex AI | `vertex_ai` |
| Bedrock | `bedrock` | Azure | `azure` |
| Sagemaker | `sagemaker_chat` | Databricks | `databricks` |
| DataRobot | `datarobot` | VLLM | `hosted_vllm` |
| Heroku | `heroku` | RAGFlow | `ragflow` |
| Compactif | `compactif` | Cometapi | `cometapi` |
| A2A | `a2a` | Bytez | `bytez` |
## Search Providers
Configure which search provider to use. LiteLLM supports multiple search providers:
| Provider | `search_provider` Value | Environment Variable |
|----------|------------------------|----------------------|
| **Perplexity AI** | `perplexity` | `PERPLEXITYAI_API_KEY` |
| **Tavily** | `tavily` | `TAVILY_API_KEY` |
| **Exa AI** | `exa_ai` | `EXA_API_KEY` |
| **Parallel AI** | `parallel_ai` | `PARALLEL_AI_API_KEY` |
| **Google PSE** | `google_pse` | `GOOGLE_PSE_API_KEY`, `GOOGLE_PSE_ENGINE_ID` |
| **DataForSEO** | `dataforseo` | `DATAFORSEO_LOGIN`, `DATAFORSEO_PASSWORD` |
| **Firecrawl** | `firecrawl` | `FIRECRAWL_API_KEY` |
| **SearXNG** | `searxng` | `SEARXNG_API_BASE` (required) |
| **Linkup** | `linkup` | `LINKUP_API_KEY` |
See [Search Providers Documentation](../search/index.md) for detailed setup instructions.
## Complete Configuration Example
```yaml
model_list:
# OpenAI
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
# MiniMax
- model_name: minimax
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
# Anthropic
- model_name: claude
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: os.environ/ANTHROPIC_API_KEY
# Azure OpenAI
- model_name: azure-gpt4
litellm_params:
model: azure/gpt-4
api_base: https://my-azure.openai.azure.com
api_key: os.environ/AZURE_API_KEY
litellm_settings:
callbacks:
- websearch_interception:
enabled_providers:
- openai
- minimax
- anthropic
- azure
search_tool_name: perplexity-search
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
- search_tool_name: tavily-search
litellm_params:
search_provider: tavily
api_key: os.environ/TAVILY_API_KEY
```
## Usage Examples
### Python SDK
```python
import litellm
# Configure callbacks
litellm.callbacks = ["websearch_interception"]
# Make completion with web search tool
response = await litellm.acompletion(
model="gpt-4o",
messages=[
{"role": "user", "content": "What are the latest AI news?"}
],
tools=[
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web for current information",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
}
},
"required": ["query"]
}
}
}
]
)
print(response.choices[0].message.content)
```
### Proxy Server
```bash
# Start proxy with config
litellm --config config.yaml
# Make request
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "What is the weather in San Francisco?"}
],
"tools": [
{
"type": "function",
"function": {
"name": "litellm_web_search",
"description": "Search the web",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"}
},
"required": ["query"]
}
}
}
]
}'
```
## How Search Tool Selection Works
1. **If `search_tool_name` is specified** → Uses that specific search tool
2. **If `search_tool_name` is not specified** → Uses first search tool in `search_tools` list
```yaml
search_tools:
- search_tool_name: perplexity-search # ← This will be used if no search_tool_name specified
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
- search_tool_name: tavily-search
litellm_params:
search_provider: tavily
api_key: os.environ/TAVILY_API_KEY
```
## Troubleshooting
### Web Search Not Working
1. **Check provider is enabled**:
```yaml
enabled_providers:
- openai # Make sure your provider is in this list
```
2. **Verify search tool is configured**:
```yaml
search_tools:
- search_tool_name: perplexity-search
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITY_API_KEY
```
3. **Check API keys are set**:
```bash
export PERPLEXITY_API_KEY=your-key
```
4. **Enable debug logging**:
```python
litellm.set_verbose = True
```
### Common Issues
**Issue**: Model returns tool_calls instead of final answer
- **Cause**: Provider not in `enabled_providers` list
- **Solution**: Add provider to `enabled_providers`
**Issue**: "No search tool configured" error
- **Cause**: No search tools in `search_tools` config
- **Solution**: Add at least one search tool configuration
**Issue**: "Invalid function arguments json string" error (MiniMax)
- **Cause**: Fixed in latest version - arguments weren't properly JSON serialized
- **Solution**: Update to latest LiteLLM version
## Related Documentation
- [Search Providers](../search/index.md) - Detailed search provider setup
- [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code
- [Tool Calling](../completion/function_call.md) - General tool calling documentation
- [Callbacks](./custom_callback.md) - Custom callback documentation
## Technical Details
### Architecture
Web search integration is implemented as a custom callback (`WebSearchInterceptionLogger`) that:
1. **Pre-request Hook**: Converts native web search tools to LiteLLM standard format
2. **Post-response Hook**: Detects web search tool calls in responses
3. **Agentic Loop**: Executes searches and makes follow-up requests automatically
### Supported APIs
- ✅ **Chat Completions API** (OpenAI format)
- ✅ **Anthropic Messages API** (Anthropic format)
- ✅ **Streaming** (automatically converted)
- ✅ **Non-streaming**
### Response Format Detection
The handler automatically detects response format:
- **OpenAI format**: `tool_calls` in assistant message
- **Anthropic format**: `tool_use` blocks in content
### Performance
- **Latency**: Adds one additional LLM call (follow-up request with search results)
- **Caching**: Search results can be cached (depends on search provider)
- **Parallel Searches**: Multiple search queries executed in parallel
## Contributing
Found a bug or want to add support for a new provider? See our [Contributing Guide](https://github.com/BerriAI/litellm/blob/main/CONTRIBUTING.md).

13
docs/my-website/docs/mcp.md
View file

@ -506,7 +506,14 @@ Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
## MCP Oauth
## MCP OAuth
LiteLLM supports OAuth 2.0 for MCP servers -- both interactive (PKCE) flows for user-facing clients and machine-to-machine (M2M) `client_credentials` for backend services.
See the **[MCP OAuth guide](./mcp_oauth.md)** for setup instructions, sequence diagrams, and a test server.
<details>
<summary>Detailed OAuth reference (click to expand)</summary>
LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
@ -588,6 +595,8 @@ sequenceDiagram
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
</details>
## Forwarding Custom Headers to MCP Servers
@ -1486,7 +1495,7 @@ async with stdio_client(server_params) as (read, write):
**Q: How do I use OAuth2 client_credentials (machine-to-machine) with MCP servers behind LiteLLM?**
At the moment LiteLLM only forwards whatever `Authorization` header/value you configure for the MCP server; it does not issue OAuth2 tokens by itself. If your MCP requires the Client Credentials grant, obtain the access token directly from the authorization server and set that bearer token as the MCP server’s Authorization header value. LiteLLM does not yet fetch or refresh those machine-to-machine tokens on your behalf, but we plan to add first-class client_credentials support in a future release so the proxy can manage those tokens automatically.
LiteLLM supports automatic token management for the `client_credentials` grant. Configure `client_id`, `client_secret`, and `token_url` on your MCP server and LiteLLM will fetch, cache, and refresh tokens automatically. See the [MCP OAuth M2M guide](./mcp_oauth.md#machine-to-machine-m2m-auth) for setup instructions.
**Q: When I fetch an OAuth token from the LiteLLM UI, where is it stored?**

244
docs/my-website/docs/mcp_oauth.md Normal file
View file

@ -0,0 +1,244 @@
# MCP OAuth
LiteLLM supports two OAuth 2.0 flows for MCP servers:
| Flow | Use Case | How It Works |
|------|----------|--------------|
| **Interactive (PKCE)** | User-facing apps (Claude Code, Cursor) | Browser-based consent, per-user tokens |
| **Machine-to-Machine (M2M)** | Backend services, CI/CD, automated agents | `client_credentials` grant, proxy-managed tokens |
## Interactive OAuth (PKCE)
For user-facing MCP clients (Claude Code, Cursor), LiteLLM supports the full OAuth 2.0 authorization code flow with PKCE.
### Setup
```yaml title="config.yaml" showLineNumbers
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
```
[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
### How It Works
```mermaid
sequenceDiagram
participant Browser as User-Agent (Browser)
participant Client as Client
participant LiteLLM as LiteLLM Proxy
participant MCP as MCP Server (Resource Server)
participant Auth as Authorization Server
Note over Client,LiteLLM: Step 1 – Resource discovery
Client->>LiteLLM: GET /.well-known/oauth-protected-resource/{mcp_server_name}/mcp
LiteLLM->>Client: Return resource metadata
Note over Client,LiteLLM: Step 2 – Authorization server discovery
Client->>LiteLLM: GET /.well-known/oauth-authorization-server/{mcp_server_name}
LiteLLM->>Client: Return authorization server metadata
Note over Client,Auth: Step 3 – Dynamic client registration
Client->>LiteLLM: POST /{mcp_server_name}/register
LiteLLM->>Auth: Forward registration request
Auth->>LiteLLM: Issue client credentials
LiteLLM->>Client: Return client credentials
Note over Client,Browser: Step 4 – User authorization (PKCE)
Client->>Browser: Open authorization URL + code_challenge + resource
Browser->>Auth: Authorization request
Note over Auth: User authorizes
Auth->>Browser: Redirect with authorization code
Browser->>LiteLLM: Callback to LiteLLM with code
LiteLLM->>Browser: Redirect back with authorization code
Browser->>Client: Callback with authorization code
Note over Client,Auth: Step 5 – Token exchange
Client->>LiteLLM: Token request + code_verifier + resource
LiteLLM->>Auth: Forward token request
Auth->>LiteLLM: Access (and refresh) token
LiteLLM->>Client: Return tokens
Note over Client,MCP: Step 6 – Authenticated MCP call
Client->>LiteLLM: MCP request with access token + LiteLLM API key
LiteLLM->>MCP: MCP request with Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: Return MCP response
```
**Participants**
- **Client** -- The MCP-capable AI agent (e.g., Claude Code, Cursor, or another IDE/agent) that initiates OAuth discovery, authorization, and tool invocations on behalf of the user.
- **LiteLLM Proxy** -- Mediates all OAuth discovery, registration, token exchange, and MCP traffic while protecting stored credentials.
- **Authorization Server** -- Issues OAuth 2.0 tokens via dynamic client registration, PKCE authorization, and token endpoints.
- **MCP Server (Resource Server)** -- The protected MCP endpoint that receives LiteLLM's authenticated JSON-RPC requests.
- **User-Agent (Browser)** -- Temporarily involved so the end user can grant consent during the authorization step.
**Flow Steps**
1. **Resource Discovery**: The client fetches MCP resource metadata from LiteLLM's `.well-known/oauth-protected-resource` endpoint to understand scopes and capabilities.
2. **Authorization Server Discovery**: The client retrieves the OAuth server metadata (token endpoint, authorization endpoint, supported PKCE methods) through LiteLLM's `.well-known/oauth-authorization-server` endpoint.
3. **Dynamic Client Registration**: The client registers through LiteLLM, which forwards the request to the authorization server (RFC 7591). If the provider doesn't support dynamic registration, you can pre-store `client_id`/`client_secret` in LiteLLM (e.g., GitHub MCP) and the flow proceeds the same way.
4. **User Authorization**: The client launches a browser session (with code challenge and resource hints). The user approves access, the authorization server sends the code through LiteLLM back to the client.
5. **Token Exchange**: The client calls LiteLLM with the authorization code, code verifier, and resource. LiteLLM exchanges them with the authorization server and returns the issued access/refresh tokens.
6. **MCP Invocation**: With a valid token, the client sends the MCP JSON-RPC request (plus LiteLLM API key) to LiteLLM, which forwards it to the MCP server and relays the tool response.
See the official [MCP Authorization Flow](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization#authorization-flow-steps) for additional reference.
## Machine-to-Machine (M2M) Auth
LiteLLM automatically fetches, caches, and refreshes OAuth2 tokens using the `client_credentials` grant. No manual token management required.
### Setup
You can configure M2M OAuth via the LiteLLM UI or `config.yaml`.
### UI Setup
Navigate to the **MCP Servers** page and click **+ Add New MCP Server**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/d1f1e89c-a789-4975-8846-b15d9821984a/ascreenshot_630800e00a2e4b598baabfc25efbabd3_text_export.jpeg)
Enter a name for your server and select **HTTP** as the transport type.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/2008c9d6-6093-4121-beab-1e52c71376aa/ascreenshot_516ffd6c7b524465a253a56048c3d228_text_export.jpeg)
Paste the MCP server URL.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/b0ee8b7d-6de8-492b-8962-287987feec29/ascreenshot_b3efca82078a4c6bb1453c58161909f9_text_export.jpeg)
Under **Authentication**, select **OAuth**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/e1597814-ff8e-40b9-9d7b-864dcdbe0910/ascreenshot_2097612712264d8f9e553f7ca9175fb0_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/f6ea5694-f28a-4bc3-9c9a-bb79f199bd65/ascreenshot_9be839f55b1b4f96bfe24030ba2c7f8d_text_export.jpeg)
Choose **Machine-to-Machine (M2M)** as the OAuth flow type. This is for server-to-server authentication using the `client_credentials` grant — no browser interaction required.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/9853310c-1d86-4628-bad1-7a391eca0e4d/ascreenshot_f302a286fa264fdd8d56db53b8f9395c_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/df64dc65-ef86-475d-adaf-12e227d5e873/ascreenshot_9e2f41d43a76435f918a00b52ffcc639_text_export.jpeg)
Fill in the **Client ID** and **Client Secret** provided by your OAuth provider.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0de5a7bd-9898-4fc7-8843-b23dd5aac47f/ascreenshot_b9087aaa81a14b5b9c199929efc4a563_text_export.jpeg)
Enter the **Token URL** — this is the endpoint LiteLLM will call to fetch access tokens using `client_credentials`.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0aea70f1-558c-4dca-91bc-1175fe1ddc89/ascreenshot_b3fcf8a1287e4e2d9a3d67c4a29f7bff_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/e842ef09-1fd7-47a6-909b-252d389f0abc/ascreenshot_2a87dad3624847e7ac370591d1d1aedd_text_export.jpeg)
Scroll down and review the server URL and all fields, then click **Create MCP Server**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/0857712b-4b53-40f8-8c1f-a4c72edaa644/ascreenshot_47be3fcd5de64ed391f70c1fb74a8bfc_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/9d961765-955f-4905-a3dc-1a446aa3b2cc/ascreenshot_43fd39d014224564bc6b35aced1fb6d3_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/3825d5fa-8fd1-4e71-b090-77ff0259c3f6/ascreenshot_2509a7ebd9bf421eb0e82f2553566745_text_export.jpeg)
Once created, open the server and navigate to the **MCP Tools** tab to verify that LiteLLM can connect and list available tools.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/8107e27b-5072-4675-8fd6-89b47692b1bd/ascreenshot_f774bc76138f430d808fb4482ebfcdca_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/ce94bb7b-c81b-4396-9939-178efb2cdfce/ascreenshot_28b838ab6ae34c76858454555c4c1d79_text_export.jpeg)
Select a tool (e.g. **echo**) to test it. Fill in the required parameters and click **Call Tool**.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/c459c1d3-ec29-4211-9c28-37fbe7783bbc/ascreenshot_e9b138b3c2cc4440bb1a6f42ac7ae861_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/5438ac60-e0ac-4a79-bf6f-5594f160d3b5/ascreenshot_9133a17d26204c46bce497e74685c483_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/a8f6821b-3982-4b4d-9b25-70c8aff5ac31/ascreenshot_28d474d0e62545a482cff6128527883a_text_export.jpeg)
LiteLLM automatically fetches an OAuth token behind the scenes and calls the tool. The result confirms the M2M OAuth flow is working end-to-end.
![](https://colony-recorder.s3.amazonaws.com/files/2026-02-10/c3924549-a949-48d1-ac67-ab4c30475859/ascreenshot_8f6eca9d717f45478d50a881bd244bb3_text_export.jpeg)
### Config.yaml Setup
```yaml title="config.yaml" showLineNumbers
mcp_servers:
my_mcp_server:
url: "https://my-mcp-server.com/mcp"
auth_type: oauth2
client_id: os.environ/MCP_CLIENT_ID
client_secret: os.environ/MCP_CLIENT_SECRET
token_url: "https://auth.example.com/oauth/token"
scopes: ["mcp:read", "mcp:write"] # optional
```
### How It Works
1. On first MCP request, LiteLLM POSTs to `token_url` with `grant_type=client_credentials`
2. The access token is cached in-memory with TTL = `expires_in - 60s`
3. Subsequent requests reuse the cached token
4. When the token expires, LiteLLM fetches a new one automatically
```mermaid
sequenceDiagram
participant Client as Client
participant LiteLLM as LiteLLM Proxy
participant Auth as Authorization Server
participant MCP as MCP Server
Client->>LiteLLM: MCP request + LiteLLM API key
LiteLLM->>Auth: POST /oauth/token (client_credentials)
Auth->>LiteLLM: access_token (expires_in: 3600)
LiteLLM->>MCP: MCP request + Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: MCP response
Note over LiteLLM: Token cached for subsequent requests
Client->>LiteLLM: Next MCP request
LiteLLM->>MCP: MCP request + cached Bearer token
MCP-->>LiteLLM: MCP response
LiteLLM-->>Client: MCP response
```
### Test with Mock Server
Use [BerriAI/mock-oauth2-mcp-server](https://github.com/BerriAI/mock-oauth2-mcp-server) to test locally:
```bash title="Terminal 1 - Start mock server" showLineNumbers
pip install fastapi uvicorn
python mock_oauth2_mcp_server.py # starts on :8765
```
```yaml title="config.yaml" showLineNumbers
mcp_servers:
test_oauth2:
url: "http://localhost:8765/mcp"
auth_type: oauth2
client_id: "test-client"
client_secret: "test-secret"
token_url: "http://localhost:8765/oauth/token"
```
```bash title="Terminal 2 - Start proxy and test" showLineNumbers
litellm --config config.yaml --port 4000
# List tools
curl http://localhost:4000/mcp-rest/tools/list \
-H "Authorization: Bearer sk-1234"
# Call a tool
curl http://localhost:4000/mcp-rest/tools/call \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{"name": "echo", "arguments": {"message": "hello"}}'
```
### Config Reference
| Field | Required | Description |
|-------|----------|-------------|
| `auth_type` | Yes | Must be `oauth2` |
| `client_id` | Yes | OAuth2 client ID. Supports `os.environ/VAR_NAME` |
| `client_secret` | Yes | OAuth2 client secret. Supports `os.environ/VAR_NAME` |
| `token_url` | Yes | Token endpoint URL |
| `scopes` | No | List of scopes to request |

144
docs/my-website/docs/pass_through/bedrock.md
View file

@ -556,3 +556,147 @@ for event in response.get("completion"):
print(completion)
```
## Using LangChain AWS SDK with LiteLLM
You can use the [LangChain AWS SDK](https://python.langchain.com/docs/integrations/chat/bedrock/) with LiteLLM Proxy to get cost tracking, load balancing, and other LiteLLM features.
### Quick Start
**1. Install LangChain AWS**:
```bash showLineNumbers
pip install langchain-aws
```
**2. Setup LiteLLM Proxy**:
Create a `config.yaml`:
```yaml showLineNumbers
model_list:
- model_name: claude-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
```
Start the proxy:
```bash showLineNumbers
export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
**3. Use LangChain with LiteLLM**:
```python showLineNumbers
from langchain_aws import ChatBedrockConverse
from langchain_core.messages import HumanMessage
# Your LiteLLM API key
API_KEY = "Bearer sk-1234"
# Initialize ChatBedrockConverse pointing to LiteLLM proxy
llm = ChatBedrockConverse(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
endpoint_url="http://localhost:4000/bedrock",
region_name="us-east-1",
aws_access_key_id=API_KEY,
aws_secret_access_key="bedrock" # Any non-empty value works
)
# Invoke the model
messages = [HumanMessage(content="Hello, how are you?")]
response = llm.invoke(messages)
print(response.content)
```
### Advanced Example: PDF Document Processing with Citations
LangChain AWS SDK supports Bedrock's document processing features. Here's how to use it with LiteLLM:
```python showLineNumbers
import os
import json
from langchain_aws import ChatBedrockConverse
from langchain_core.messages import HumanMessage
# Your LiteLLM API key
API_KEY = "Bearer sk-1234"
def get_llm() -> ChatBedrockConverse:
"""Initialize LLM pointing to LiteLLM proxy"""
llm = ChatBedrockConverse(
model_id="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
base_model_id="anthropic.claude-3-7-sonnet-20250219-v1:0",
endpoint_url="http://localhost:4000/bedrock",
region_name="us-east-1",
aws_access_key_id=API_KEY,
aws_secret_access_key="bedrock"
)
return llm
if __name__ == "__main__":
# Initialize the LLM
llm = get_llm()
# Read PDF file as bytes (Converse API requires raw bytes)
with open("your-document.pdf", "rb") as file:
file_bytes = file.read()
# Prepare messages with document attachment
messages = [
HumanMessage(content=[
{"text": "What is the policy number in this document?"},
{
"document": {
"format": "pdf",
"name": "PolicyDocument",
"source": {"bytes": file_bytes},
"citations": {"enabled": True}
}
}
])
]
# Invoke the LLM
response = llm.invoke(messages)
# Print response with citations
print(json.dumps(response.content, indent=4))
```
### Supported LangChain Features
All LangChain AWS features work with LiteLLM:
| Feature | Supported | Notes |
|---------|-----------|-------|
| Text Generation | ✅ | Full support |
| Streaming | ✅ | Use `stream()` method |
| Document Processing | ✅ | PDF, images, etc. |
| Citations | ✅ | Enable in document config |
| Tool Use | ✅ | Function calling support |
| Multi-modal | ✅ | Text + images + documents |
### Troubleshooting
**Issue**: `UnknownOperationException` error
**Solution**: Make sure you're using the correct endpoint URL format:
- ✅ Correct: `http://localhost:4000/bedrock`
- ❌ Wrong: `http://localhost:4000/bedrock/v2`
**Issue**: Authentication errors
**Solution**: Ensure your API key is in the correct format:
```python
aws_access_key_id="Bearer sk-1234" # Include "Bearer " prefix
```

22
docs/my-website/docs/providers/litellm_proxy.md
View file

@ -227,6 +227,28 @@ response = litellm.completion(
)
```
## OAuth2/JWT Authentication
If your LiteLLM Proxy requires OAuth2/JWT authentication (e.g., Azure AD, Keycloak, Okta), the SDK can automatically obtain and refresh tokens for you.
```python
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(),
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
[Learn more about SDK Proxy Authentication (OAuth2/JWT Auto-Refresh) →](../proxy_auth)
## Sending `tags` to LiteLLM Proxy
Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter.

287
docs/my-website/docs/providers/perplexity.md
View file

@ -120,6 +120,293 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
## Agentic Research API (Responses API)
Requires v1.72.6+
### Using Presets
Presets provide optimized defaults for specific use cases. Start with a preset for quick setup:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
# Using the pro-search preset
response = responses(
model="perplexity/preset/pro-search",
input="What are the latest developments in AI?",
custom_llm_provider="perplexity",
)
print(response.output)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: perplexity-pro-search
litellm_params:
model: perplexity/preset/pro-search
api_key: os.environ/PERPLEXITY_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/responses \
-H "Content-Type: application/json" \
-H "Authorization: Bearer anything" \
-d '{
"model": "perplexity-pro-search",
"input": "What are the latest developments in AI?"
}'
```
</TabItem>
</Tabs>
### Using Third-Party Models
Access models from OpenAI, Anthropic, Google, xAI, and other providers through Perplexity's unified API:
<Tabs>
<TabItem value="openai" label="OpenAI">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="Explain quantum computing in simple terms",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
input="Write a short story about a robot learning to paint",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="google" label="Google">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/google/gemini-2.0-flash-exp",
input="Explain the concept of neural networks",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/xai/grok-2-1212",
input="What makes a good AI assistant?",
custom_llm_provider="perplexity",
max_output_tokens=500,
)
print(response.output)
```
</TabItem>
</Tabs>
### Web Search Tool
Enable web search capabilities to access real-time information:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="What's the weather in San Francisco today?",
custom_llm_provider="perplexity",
tools=[{"type": "web_search"}],
instructions="You have access to a web_search tool. Use it for questions about current events.",
)
print(response.output)
```
### Reasoning Effort (Responses API)
Control the reasoning effort level for reasoning-capable models:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-5.2",
input="Solve this complex problem step by step",
custom_llm_provider="perplexity",
reasoning={"effort": "high"}, # Options: low, medium, high
max_output_tokens=1000,
)
print(response.output)
```
### Multi-Turn Conversations
Use message arrays for multi-turn conversations with context:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/anthropic/claude-3-5-sonnet-20241022",
input=[
{"type": "message", "role": "system", "content": "You are a helpful assistant."},
{"type": "message", "role": "user", "content": "What are the latest AI developments?"},
],
custom_llm_provider="perplexity",
instructions="Provide detailed, well-researched answers.",
max_output_tokens=800,
)
print(response.output)
```
### Streaming Responses
Stream responses for real-time output:
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
response = responses(
model="perplexity/openai/gpt-4o",
input="Tell me a story about space exploration",
custom_llm_provider="perplexity",
stream=True,
max_output_tokens=500,
)
for chunk in response:
if hasattr(chunk, 'type'):
if chunk.type == "response.output_text.delta":
print(chunk.delta, end="", flush=True)
```
### Supported Third-Party Models
| Provider | Model Name | Function Call |
|----------|------------|---------------|
| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
| OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
### Available Presets
| Preset Name | Function Call |
|----------------|--------------------------------------------------------|
| fast-search | `responses(model="perplexity/preset/fast-search", ...)`|
| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
| deep-research | `responses(model="perplexity/preset/deep-research", ...)`|
### Complete Example
```python
from litellm import responses
import os
os.environ['PERPLEXITY_API_KEY'] = ""
# Comprehensive example with multiple features
response = responses(
model="perplexity/openai/gpt-4o",
input="Research the latest developments in quantum computing and provide sources",
custom_llm_provider="perplexity",
tools=[
{"type": "web_search"},
{"type": "fetch_url"}
],
instructions="Use web_search to find relevant information and fetch_url to retrieve detailed content from sources. Provide citations for all claims.",
max_output_tokens=1000,
temperature=0.7,
)
print(f"Response ID: {response.id}")
print(f"Model: {response.model}")
print(f"Status: {response.status}")
print(f"Output: {response.output}")
print(f"Usage: {response.usage}")
```
:::info
For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)

35
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -223,6 +223,7 @@ GENERIC_USER_FIRST_NAME_ATTRIBUTE = "first_name"
GENERIC_USER_LAST_NAME_ATTRIBUTE = "last_name"
GENERIC_USER_ROLE_ATTRIBUTE = "given_role"
GENERIC_USER_PROVIDER_ATTRIBUTE = "provider"
GENERIC_USER_EXTRA_ATTRIBUTES = "department,employee_id,manager" # comma-separated list of additional fields to extract from SSO response
GENERIC_CLIENT_STATE = "some-state" # if the provider needs a state parameter
GENERIC_INCLUDE_CLIENT_ID = "false" # some providers enforce that the client_id is not in the body
GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not enough to retrieve basic user info like first_name and last_name located in profile scope
@ -239,6 +240,40 @@ Use `GENERIC_USER_ROLE_ATTRIBUTE` to specify which attribute in the SSO token co
Nested attribute paths are supported (e.g., `claims.role` or `attributes.litellm_role`).
**Capturing Additional SSO Fields**
Use `GENERIC_USER_EXTRA_ATTRIBUTES` to extract additional fields from the SSO provider response beyond the standard user attributes (id, email, name, etc.). This is useful when you need to access custom organization-specific data (e.g., department, employee ID, groups) in your [custom SSO handler](./custom_sso.md).
```shell
# Comma-separated list of field names to extract
GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,manager,groups"
```
**Accessing Extra Fields in Custom SSO Handler:**
```python
from litellm.proxy.management_endpoints.types import CustomOpenID
async def custom_sso_handler(userIDPInfo: CustomOpenID):
# Access the extra fields
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
user_department = extra_fields.get("department")
employee_id = extra_fields.get("employee_id")
user_groups = extra_fields.get("groups", [])
# Use these fields for custom logic (e.g., team assignment, access control)
# ...
```
**Nested Field Paths:**
Dot notation is supported for nested fields:
```shell
GENERIC_USER_EXTRA_ATTRIBUTES="org_info.department,org_info.cost_center,metadata.employee_type"
```
- Set Redirect URI, if your provider requires it
- Set a redirect url = `<your proxy base url>/sso/callback`
```shell

5
docs/my-website/docs/proxy/config_settings.md
View file

@ -548,6 +548,10 @@ router_settings:
| DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL | Default embedding model for MCP semantic tool filtering. Default is "text-embedding-3-small"
| DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
| DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
| MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS | Seconds to subtract from token expiry when computing cache TTL. Default is 60
| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20
| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10
| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602
@ -640,6 +644,7 @@ router_settings:
| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
| GENERIC_USER_EXTRA_ATTRIBUTES | Comma-separated list of additional fields to extract from generic SSO provider response (e.g., "department,employee_id,groups"). Accessible via `CustomOpenID.extra_fields` in custom SSO handlers. Supports dot notation for nested fields
| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth

12
docs/my-website/docs/proxy/custom_sso.md
View file

@ -142,6 +142,18 @@ async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:
f"No ID found for user. userIDPInfo.id is None {userIDPInfo}"
)
#################################################
# Access extra fields from SSO provider (requires GENERIC_USER_EXTRA_ATTRIBUTES env var)
# Example: Set GENERIC_USER_EXTRA_ATTRIBUTES="department,employee_id,groups"
extra_fields = getattr(userIDPInfo, 'extra_fields', None) or {}
user_department = extra_fields.get("department")
employee_id = extra_fields.get("employee_id")
user_groups = extra_fields.get("groups", [])
print(f"User department: {user_department}") # noqa
print(f"Employee ID: {employee_id}") # noqa
print(f"User groups: {user_groups}") # noqa
#################################################
#################################################
# Run your custom code / logic here

46
docs/my-website/docs/proxy/forward_client_headers.md
View file

@ -6,6 +6,52 @@ Control which model groups can forward client headers to the underlying LLM prov
By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
## How it Works
LiteLLM does **not** forward all client headers to the LLM provider. Instead, it uses an **allowlist** approach — only headers matching specific rules are forwarded. This ensures sensitive headers (like your LiteLLM API key) are never accidentally sent to upstream providers.
```mermaid
sequenceDiagram
participant Client as Client (SDK / curl)
participant Proxy as LiteLLM Proxy
participant Filter as Header Filter (Allowlist)
participant LLM as LLM Provider (OpenAI, Anthropic, etc.)
Client->>Proxy: Request with all headers<br/>(Authorization, x-trace-id,<br/>x-custom-header, anthropic-beta, etc.)
Proxy->>Filter: Check forward_client_headers_to_llm_api<br/>setting for this model group
Note over Filter: Allowlist rules:<br/>1. Headers starting with "x-" ✅<br/>2. "anthropic-beta" ✅<br/>3. "x-stainless-*" ❌ (blocked)<br/>4. All other headers ❌ (blocked)
Filter-->>Proxy: Return only allowed headers
Proxy->>LLM: Request with filtered headers<br/>(x-trace-id, x-custom-header,<br/>anthropic-beta)
LLM-->>Proxy: Response
Proxy-->>Client: Response
```
### Header Allowlist Rules
The following rules determine which headers are forwarded (see [`_get_forwardable_headers`](https://github.com/litellm/litellm/blob/main/litellm/proxy/litellm_pre_call_utils.py) in `litellm/proxy/litellm_pre_call_utils.py`):
| Rule | Example | Forwarded? |
|---|---|---|
| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes |
| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes |
| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) |
| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No |
| Other provider headers | `Accept`, `User-Agent` | ❌ No |
### Additional Header Mechanisms
| Mechanism | Description | Reference |
|---|---|---|
| **`x-pass-` prefix** | Headers prefixed with `x-pass-` are always forwarded with the prefix stripped, regardless of settings. E.g., `x-pass-anthropic-beta: value` → `anthropic-beta: value`. Works for all pass-through endpoints. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/passthrough/utils.py) |
| **`openai-organization`** | Forwarded only when `forward_openai_org_id: true` is set in `general_settings`. | [Forward OpenAI Org ID](#enable-globally) |
| **User information headers** | When `add_user_information_to_llm_headers: true`, LiteLLM adds `x-litellm-user-id`, `x-litellm-org-id`, etc. | [User Information Headers](#user-information-headers-optional) |
| **Vertex AI pass-through** | Uses a separate, stricter allowlist: only `anthropic-beta` and `content-type`. | [Source code](https://github.com/litellm/litellm/blob/main/litellm/constants.py) |
## Configuration
## Enable Globally

28
docs/my-website/docs/proxy/guardrails/zscaler_ai_guard.md
View file

@ -100,7 +100,7 @@ In cases where encounter other errors when apply Zscaler AI Guard, return exampl
}
}
```
## 6. Sending User Information to Zscaler AI Guard for Analysis (Optional)
## 6. Sending User Information to Zscaler AI Guard (Optional)
If you need to send end-user information to Zscaler AI Guard for analysis, you can set the configuration in the environment variables to True and include the relevant information in custom_headers on Zscaler AI Guard.
- To send user_api_key_alias:
@ -133,4 +133,30 @@ curl -i http://localhost:8165/v1/chat/completions \
"zguard_policy_id": <the custom policy id>
}
}'
```
## 8. Set Custom Zscaler AI Guard Policy on Litellm Team OR Key Metadata (Optional)
In addition to setting `zguard_policy_id` in a request or the configuration file, you can also set it in the metadata for LiteLLM Team or Key. The `zguard_policy_id` is determined using the following order of precedence: request, Key, Team, config file. This logic is illustrated below:
```
user_api_key_metadata = metadata.get("user_api_key_metadata", {}) or {}
team_metadata = metadata.get("team_metadata", {}) or {}
policy_id = (
metadata.get("zguard_policy_id")
if "zguard_policy_id" in metadata
else (
user_api_key_metadata.get("zguard_policy_id")
if "zguard_policy_id" in user_api_key_metadata
else (
team_metadata.get("zguard_policy_id")
if "zguard_policy_id" in team_metadata
else self.policy_id
)
)
)
```
You can leverage this feature to apply multiple policies configured on the Zscaler AI Guard (ZGuard) to traffic from different applications. (Note: It is recommended to map policies using either Team or Key metadata, but not a mix of both.)
Example set in Team/Key Metadata, you can set From UI:
```
{"zguard_policy_id": 100}
```

333
docs/my-website/docs/proxy_auth.md Normal file
View file

@ -0,0 +1,333 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# SDK Proxy Authentication (OAuth2/JWT Auto-Refresh)
Automatically obtain and refresh OAuth2/JWT tokens when using the LiteLLM Python SDK with a LiteLLM Proxy that requires JWT authentication.
## Overview
When your LiteLLM Proxy is protected by an OAuth2/OIDC provider (Azure AD, Keycloak, Okta, Auth0, etc.), your SDK clients need valid JWT tokens for every request. Instead of manually managing token lifecycle, `litellm.proxy_auth` handles this automatically:
- Obtains tokens from your identity provider
- Caches tokens to avoid unnecessary requests
- Refreshes tokens before they expire (60-second buffer)
- Injects `Authorization: Bearer <token>` headers into every request
## Quick Start
### Azure AD
<Tabs>
<TabItem value="default" label="DefaultAzureCredential">
Uses the [DefaultAzureCredential](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential) chain (environment variables, managed identity, Azure CLI, etc.):
```python
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
# One-time setup
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(), # uses DefaultAzureCredential
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
# All requests now include Authorization headers automatically
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
</TabItem>
<TabItem value="client-secret" label="ClientSecretCredential">
Use a specific Azure AD app registration:
```python
import litellm
from azure.identity import ClientSecretCredential
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
azure_cred = ClientSecretCredential(
tenant_id="your-tenant-id",
client_id="your-client-id",
client_secret="your-client-secret"
)
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(credential=azure_cred),
scope="api://my-litellm-proxy/.default"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
</TabItem>
</Tabs>
**Required package:** `pip install azure-identity`
### Generic OAuth2 (Okta, Auth0, Keycloak, etc.)
Works with any OAuth2 provider that supports the `client_credentials` grant type:
```python
import litellm
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-client-id",
client_secret="your-client-secret",
token_url="https://your-idp.example.com/oauth2/token"
),
scope="litellm_proxy_api"
)
litellm.api_base = "https://my-proxy.example.com"
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Custom Credential Provider
Implement the `TokenCredential` protocol to use any authentication mechanism:
```python
import time
import litellm
from litellm.proxy_auth import AccessToken, ProxyAuthHandler
class MyCustomCredential:
"""Any class with a get_token(scope) -> AccessToken method works."""
def get_token(self, scope: str) -> AccessToken:
# Your custom logic to obtain a token
token = my_auth_system.get_jwt(scope=scope)
return AccessToken(
token=token,
expires_on=int(time.time()) + 3600
)
litellm.proxy_auth = ProxyAuthHandler(
credential=MyCustomCredential(),
scope="my-scope"
)
```
## Supported Endpoints
Auth headers are automatically injected for:
| Endpoint | Function |
|----------|----------|
| Chat Completions | `litellm.completion()` / `litellm.acompletion()` |
| Embeddings | `litellm.embedding()` / `litellm.aembedding()` |
## How It Works
```
┌──────────┐ ┌──────────────────┐ ┌──────────────┐ ┌──────────────┐
│ Your │ │ ProxyAuthHandler │ │ Identity │ │ LiteLLM │
│ Code │────▶│ (token cache) │────▶│ Provider │ │ Proxy │
│ │ │ │◀────│ (Azure AD, │ │ │
│ │ │ │ │ Okta, etc) │ │ │
│ │ └────────┬─────────┘ └──────────────┘ │ │
│ │ │ Authorization: Bearer <token> │ │
│ │──────────────┼───────────────────────────────────▶│ │
│ │◀─────────────┼────────────────────────────────────│ │
└──────────┘ │ └──────────────┘
```
1. You set `litellm.proxy_auth` once at startup
2. On each SDK call (`completion()`, `embedding()`), the handler checks its cached token
3. If the token is missing or expires within 60 seconds, it requests a new one from your identity provider
4. The `Authorization: Bearer <token>` header is injected into the request
5. If token retrieval fails, a warning is logged and the request proceeds without auth headers
## API Reference
### ProxyAuthHandler
The main handler that manages the token lifecycle.
```python
from litellm.proxy_auth import ProxyAuthHandler
handler = ProxyAuthHandler(
credential=<TokenCredential>, # required - credential provider
scope="<oauth2-scope>" # required - OAuth2 scope to request
)
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `credential` | `TokenCredential` | Yes | A credential provider (AzureADCredential, GenericOAuth2Credential, or custom) |
| `scope` | `str` | Yes | The OAuth2 scope to request tokens for |
**Methods:**
| Method | Returns | Description |
|--------|---------|-------------|
| `get_token()` | `AccessToken` | Get a valid token, refreshing if needed |
| `get_auth_headers()` | `dict` | Get `{"Authorization": "Bearer <token>"}` headers |
### AzureADCredential
Wraps any `azure-identity` credential with lazy initialization.
```python
from litellm.proxy_auth import AzureADCredential
# Uses DefaultAzureCredential (recommended)
cred = AzureADCredential()
# Or wrap a specific azure-identity credential
from azure.identity import ManagedIdentityCredential
cred = AzureADCredential(credential=ManagedIdentityCredential())
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `credential` | Azure `TokenCredential` | No | An azure-identity credential. If `None`, uses `DefaultAzureCredential` |
### GenericOAuth2Credential
Standard OAuth2 client credentials flow for any provider.
```python
from litellm.proxy_auth import GenericOAuth2Credential
cred = GenericOAuth2Credential(
client_id="your-client-id",
client_secret="your-client-secret",
token_url="https://your-idp.com/oauth2/token"
)
```
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `client_id` | `str` | Yes | OAuth2 client ID |
| `client_secret` | `str` | Yes | OAuth2 client secret |
| `token_url` | `str` | Yes | Token endpoint URL |
### AccessToken
Dataclass representing an OAuth2 access token.
```python
from litellm.proxy_auth import AccessToken
token = AccessToken(
token="eyJhbG...", # JWT string
expires_on=1234567890 # Unix timestamp
)
```
### TokenCredential Protocol
Any class implementing this protocol can be used as a credential provider:
```python
from litellm.proxy_auth import AccessToken
class MyCredential:
def get_token(self, scope: str) -> AccessToken:
...
```
## Provider-Specific Examples
### Keycloak
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="litellm-client",
client_secret="your-keycloak-client-secret",
token_url="https://keycloak.example.com/realms/your-realm/protocol/openid-connect/token"
),
scope="openid"
)
```
### Okta
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-okta-client-id",
client_secret="your-okta-client-secret",
token_url="https://your-org.okta.com/oauth2/default/v1/token"
),
scope="litellm_api"
)
```
### Auth0
```python
from litellm.proxy_auth import GenericOAuth2Credential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=GenericOAuth2Credential(
client_id="your-auth0-client-id",
client_secret="your-auth0-client-secret",
token_url="https://your-tenant.auth0.com/oauth/token"
),
scope="https://my-proxy.example.com/api"
)
```
### Azure AD with Managed Identity
```python
from azure.identity import ManagedIdentityCredential
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(
credential=ManagedIdentityCredential()
),
scope="api://my-litellm-proxy/.default"
)
```
## Combining with `use_litellm_proxy`
You can use `proxy_auth` together with [`use_litellm_proxy`](./providers/litellm_proxy#send-all-sdk-requests-to-litellm-proxy) to route all SDK requests through an authenticated proxy:
```python
import os
import litellm
from litellm.proxy_auth import AzureADCredential, ProxyAuthHandler
# Route all requests through the proxy
os.environ["LITELLM_PROXY_API_BASE"] = "https://my-proxy.example.com"
litellm.use_litellm_proxy = True
# Authenticate with OAuth2/JWT
litellm.proxy_auth = ProxyAuthHandler(
credential=AzureADCredential(),
scope="api://my-litellm-proxy/.default"
)
# This request goes through the proxy with automatic JWT auth
response = litellm.completion(
model="vertex_ai/gemini-2.0-flash-001",
messages=[{"role": "user", "content": "Hello!"}]
)
```

68
docs/my-website/docs/troubleshoot/max_callbacks.md Normal file
View file

@ -0,0 +1,68 @@
# MAX_CALLBACKS Limit
## Error Message
```
Cannot add callback - would exceed MAX_CALLBACKS limit of 30. Current callbacks: 30
```
## What This Means
LiteLLM limits the number of callbacks that can be registered to prevent performance degradation. Each callback runs on every LLM request, so having too many callbacks can cause exponential CPU usage and slow down your proxy.
The default limit is **30 callbacks**.
## When You Might Hit This Limit
- **Large enterprise deployments** with many teams, each having their own guardrails
- **Multiple logging integrations** combined with custom callbacks
- **Per-team callback configurations** that add up across your organization
## How to Override
Set the `LITELLM_MAX_CALLBACKS` environment variable to increase the limit:
```bash
# Docker
docker run -e LITELLM_MAX_CALLBACKS=100 ...
# Docker Compose
environment:
- LITELLM_MAX_CALLBACKS=100
# Kubernetes
env:
- name: LITELLM_MAX_CALLBACKS
value: "100"
# Direct
export LITELLM_MAX_CALLBACKS=100
litellm --config config.yaml
```
## Recommendations
1. **Start conservative** - Only increase as much as you need. If you have 60 teams with guardrails, try `LITELLM_MAX_CALLBACKS=75` to leave headroom.
2. **Monitor performance** - More callbacks means more processing per request. Watch your CPU usage and response latency after increasing the limit.
3. **Consolidate where possible** - If multiple teams use identical guardrails, consider using shared callback configurations rather than per-team duplicates.
## Example: Large Enterprise Setup
For an organization with 60+ teams, each with a guardrail callback:
```yaml
# config.yaml
litellm_settings:
callbacks: ["prometheus", "langfuse"] # 2 global callbacks
# Each team adds 1 guardrail callback = 60+ callbacks
# Total: 62+ callbacks needed
```
Set the environment variable:
```bash
export LITELLM_MAX_CALLBACKS=100
```

43
docs/my-website/docs/tutorials/claude_code_prompt_cache_routing.md Normal file
View file

@ -0,0 +1,43 @@
# Claude Code - Prompt Cache Routing
Claude's [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) feature helps to optimize API usage through attempting to cache prompts and re-use cached prompts during subsequent API calls. This feature is used by Claude Code.
When LiteLLM [load balancing](../proxy/load_balancing.md) is enabled, to ensure this prompt caching feature still works with Claude Code, LiteLLM needs to be configured to use the `PromptCachingDeploymentCheck` pre-call check. This pre-call check will ensure that API calls that used prompt caching are remembered and that subsequent API calls that try to use that prompt caching are routed to the same model deployment where a cache write occurred.
## Set Up
1. Configure the router so that it uses the `PromptCachingDeploymentCheck` (via setting the `optional_pre_call_checks` property), and configure the models so that they can access multiple deployments of Claude; below, we show an example for multiple AWS accounts (referred to as `account-1` and `account-2`, using the `aws_profile_name` property):
```yaml
router_settings:
optional_pre_call_checks: ["prompt_caching"]
model_list:
- litellm_params:
model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_profile_name: account-1
aws_region_name: us-west-2
model_info:
litellm_provider: bedrock
model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
- litellm_params:
model: us.anthropic.claude-sonnet-4-5-20250929-v1:0
aws_profile_name: account-2
aws_region_name: us-west-2
model_info:
litellm_provider: bedrock
model_name: us.anthropic.claude-sonnet-4-5-20250929-v1:0
```
2. Utilize Claude Code:
1. Launch Claude Code, which will do a warm-up API call that tries to cache its warm-up prompt and its system prompt.
2. Wait a few seconds, then quit Claude Code and re-open it.
3. You'll notice that the warm-up API call successfully gets a cache hit (if using Claude Code in an IDE like VS Code, ensure that you don't do anything between step 2.1 and 2.2 here, otherwise there may not be a cache hit):
1. Go to the [LiteLLM Request Logs page](../proxy/ui_logs.md) in the Admin UI
2. Click on the individual requests to see (a) the cache creation and cache read tokens; and (b) the Model ID. In particular, the API call from step 2.1 should show a cache write, and the API call from step 2.2 should show a cache read; in addition, the Model ID should be equal (meaning the API call is getting forwarded to the same AWS account).
## Related
- [Claude Code - Quickstart](./claude_responses_api.md)
- [Claude Code - Customer Tracking](./claude_code_customer_tracking.md)
- [Claude Code - Plugin Marketplace](./claude_code_plugin_marketplace.md)
- [Claude Code - WebSearch](./claude_code_websearch.md)
- [Proxy - Load Balancing](../proxy/load_balancing.md)

2
docs/my-website/package.json
View file

@ -61,6 +61,8 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"tar": ">=7.5.7",
"@isaacs/brace-expansion": ">=5.0.1",
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1",
"lodash-es": ">=4.17.23"

2
docs/my-website/release_notes/v1.81.3-stable/index.md
View file

@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.81.3.rc.2
docker.litellm.ai/berriai/litellm:v1.81.3-stable
```
</TabItem>

20
docs/my-website/sidebars.js
View file

@ -96,6 +96,11 @@ const sidebars = {
"proxy/prometheus"
]
},
{
type: "doc",
id: "integrations/websearch_interception",
label: "Web Search Integration"
},
{
type: "category",
label: "[Beta] Prompt Management",
@ -125,6 +130,7 @@ const sidebars = {
"tutorials/claude_responses_api",
"tutorials/claude_code_max_subscription",
"tutorials/claude_code_customer_tracking",
"tutorials/claude_code_prompt_cache_routing",
"tutorials/claude_code_websearch",
"tutorials/claude_mcp",
"tutorials/claude_non_anthropic_models",
@ -223,6 +229,7 @@ const sidebars = {
label: "Configuration",
items: [
"set_keys",
"proxy_auth",
"caching/all_caches",
],
},
@ -556,6 +563,7 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_oauth",
"mcp_public_internet",
"mcp_semantic_filter",
"mcp_control",
@ -1074,6 +1082,18 @@ const sidebars = {
"troubleshoot/cpu_issues",
"troubleshoot/memory_issues",
"troubleshoot/spend_queue_warnings",
"troubleshoot/max_callbacks",
],
},
{
type: "category",
label: "Blog",
items: [
{
type: "link",
label: "Incident: Broken Model Cost Map",
href: "/blog/model-cost-map-incident",
},
],
},
],

98
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -899,49 +899,49 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
batch_id=response.id, model_id=model_id
)
if (
response.output_file_id and model_id
): # return a file id with the model_id and output_file_id
original_output_file_id = response.output_file_id
response.output_file_id = self.get_unified_output_file_id(
output_file_id=response.output_file_id,
model_id=model_id,
model_name=model_name,
)
# Fetch the actual file object for the output file
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_output_file_id
# Handle both output_file_id and error_file_id
for file_attr in ["output_file_id", "error_file_id"]:
file_id_value = getattr(response, file_attr, None)
if file_id_value and model_id:
original_file_id = file_id_value
unified_file_id = self.get_unified_output_file_id(
output_file_id=original_file_id,
model_id=model_id,
model_name=model_name,
)
verbose_logger.debug(
f"Successfully retrieved file object for output_file_id={original_output_file_id}"
setattr(response, file_attr, unified_file_id)
# Fetch the actual file object from the provider
file_object = None
try:
# Use litellm to retrieve the file object from the provider
from litellm import afile_retrieve
file_object = await afile_retrieve(
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
file_id=original_file_id
)
verbose_logger.debug(
f"Successfully retrieved file object for {file_attr}={original_file_id}"
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: original_file_id},
user_api_key_dict=user_api_key_dict,
)
except Exception as e:
verbose_logger.warning(
f"Failed to retrieve file object for output_file_id={original_output_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=response.output_file_id,
file_object=file_object,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_mappings={model_id: original_output_file_id},
user_api_key_dict=user_api_key_dict,
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
await self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
@ -958,15 +958,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
response.id = self.get_unified_generic_response_id(
model_id=model_id, generic_response_id=response.id
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
await self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
elif isinstance(response, AsyncCursorPage):
"""

4
litellm-js/spend-logs/package.json
View file

@ -11,6 +11,8 @@
"tsx": "^4.7.1"
},
"overrides": {
"glob": ">=11.1.0"
"glob": ">=11.1.0",
"tar": ">=7.5.7",
"@isaacs/brace-expansion": ">=5.0.1"
}
}

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.34.tar.gz vendored Normal file
View file

Binary file not shown.

8
litellm-proxy-extras/litellm_proxy_extras/migrations/20260209085821_add_verificationtoken_indexes/migration.sql Normal file
View file

@ -0,0 +1,8 @@
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_user_id_team_id_idx" ON "LiteLLM_VerificationToken"("user_id", "team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_team_id_idx" ON "LiteLLM_VerificationToken"("team_id");
-- CreateIndex
CREATE INDEX "LiteLLM_VerificationToken_budget_reset_at_expires_idx" ON "LiteLLM_VerificationToken"("budget_reset_at", "expires");

10
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -310,6 +310,16 @@ model LiteLLM_VerificationToken {
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
// SELECT COUNT(*) FROM (SELECT "public"."LiteLLM_VerificationToken"."token" FROM "public"."LiteLLM_VerificationToken" WHERE ("public"."LiteLLM_VerificationToken"."user_id" = $1 AND ("public"."LiteLLM_VerificationToken"."team_id" IS NULL OR "public"."LiteLLM_VerificationToken"."team_id" <> $2)) OFFSET $3 ) AS "sub"
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."user_id" = $1 OFFSET $2
@@index([user_id, team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE "public"."LiteLLM_VerificationToken"."team_id" = $1 OFFSET $2
@@index([team_id])
// SELECT ... FROM "public"."LiteLLM_VerificationToken" WHERE (("public"."LiteLLM_VerificationToken"."expires" IS NULL OR "public"."LiteLLM_VerificationToken"."expires" > $1) AND "public"."LiteLLM_VerificationToken"."budget_reset_at" < $2) OFFSET $3
@@index([budget_reset_at, expires])
}
// Audit table for deleted keys - preserves spend and key information for historical tracking

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.33"
version = "0.4.34"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.33"
version = "0.4.34"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

1
litellm/__init__.py
View file

@ -1393,6 +1393,7 @@ if TYPE_CHECKING:
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig

6
litellm/_lazy_imports_registry.py
View file

@ -226,6 +226,7 @@ LLM_CONFIG_NAMES = (
"XAIResponsesAPIConfig",
"LiteLLMProxyResponsesAPIConfig",
"VolcEngineResponsesAPIConfig",
"PerplexityResponsesConfig",
"GoogleAIStudioInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
@ -274,6 +275,7 @@ LLM_CONFIG_NAMES = (
"LmStudioEmbeddingConfig",
"NscaleConfig",
"PerplexityChatConfig",
"PerplexityResponsesConfig",
"AzureOpenAIO1Config",
"IBMWatsonXAIConfig",
"IBMWatsonXChatConfig",
@ -901,6 +903,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.manus.responses.transformation",
"ManusResponsesAPIConfig",
),
"PerplexityResponsesConfig": (
".llms.perplexity.responses.transformation",
"PerplexityResponsesConfig",
),
"GoogleAIStudioInteractionsConfig": (
".llms.gemini.interactions.transformation",
"GoogleAIStudioInteractionsConfig",

96
litellm/_logging.py
View file

@ -1,9 +1,13 @@
import json
import ast
import logging
import os
import sys
from datetime import datetime
from logging import Formatter
from typing import Any, Dict, Optional
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
set_verbose = False
@ -19,6 +23,67 @@ handler = logging.StreamHandler()
handler.setLevel(numeric_level)
def _try_parse_json_message(message: str) -> Optional[Dict[str, Any]]:
"""
Try to parse a log message as JSON. Returns parsed dict if valid, else None.
Handles messages that are entirely valid JSON (e.g. json.dumps output).
Uses shared safe_json_loads for consistent error handling.
"""
if not message or not isinstance(message, str):
return None
msg_stripped = message.strip()
if not (msg_stripped.startswith("{") or msg_stripped.startswith("[")):
return None
parsed = safe_json_loads(message, default=None)
if parsed is None or not isinstance(parsed, dict):
return None
return parsed
def _try_parse_embedded_python_dict(message: str) -> Optional[Dict[str, Any]]:
"""
Try to find and parse a Python dict repr (e.g. str(d) or repr(d)) embedded in
the message. Handles patterns like:
"get_available_deployment for model: X, Selected deployment: {'model_name': '...', ...} for model: X"
Uses ast.literal_eval for safe parsing. Returns the parsed dict or None.
"""
if not message or not isinstance(message, str) or "{" not in message:
return None
i = 0
while i < len(message):
start = message.find("{", i)
if start == -1:
break
depth = 0
for j in range(start, len(message)):
c = message[j]
if c == "{":
depth += 1
elif c == "}":
depth -= 1
if depth == 0:
substr = message[start : j + 1]
try:
result = ast.literal_eval(substr)
if isinstance(result, dict) and len(result) > 0:
return result
except (ValueError, SyntaxError, TypeError):
pass
break
i = start + 1
return None
# Standard LogRecord attribute names - used to identify 'extra' fields.
# Derived at runtime so we automatically include version-specific attrs (e.g. taskName).
def _get_standard_record_attrs() -> frozenset:
"""Standard LogRecord attribute names - excludes extra keys from logger.debug(..., extra={...})."""
return frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__.keys())
_STANDARD_RECORD_ATTRS = _get_standard_record_attrs()
class JsonFormatter(Formatter):
def __init__(self):
super(JsonFormatter, self).__init__()
@ -29,16 +94,31 @@ class JsonFormatter(Formatter):
return dt.isoformat()
def format(self, record):
json_record = {
"message": record.getMessage(),
message_str = record.getMessage()
json_record: Dict[str, Any] = {
"message": message_str,
"level": record.levelname,
"timestamp": self.formatTime(record),
}
# Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties
parsed = _try_parse_json_message(message_str)
if parsed is None:
parsed = _try_parse_embedded_python_dict(message_str)
if parsed is not None:
for key, value in parsed.items():
if key not in json_record:
json_record[key] = value
# Include extra attributes passed via logger.debug("msg", extra={...})
for key, value in record.__dict__.items():
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
json_record[key] = value
if record.exc_info:
json_record["stacktrace"] = self.formatException(record.exc_info)
return json.dumps(json_record)
return safe_dumps(json_record)
# Function to set up exception handlers for JSON logging
@ -169,15 +249,15 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
def _get_uvicorn_json_log_config():
"""
Generate a uvicorn log_config dictionary that applies JSON formatting to all loggers.
This ensures that uvicorn's access logs, error logs, and all application logs
are formatted as JSON when json_logs is enabled.
"""
json_formatter_class = "litellm._logging.JsonFormatter"
# Use the module-level log_level variable for consistency
uvicorn_log_level = log_level.upper()
log_config = {
"version": 1,
"disable_existing_loggers": False,
@ -222,7 +302,7 @@ def _get_uvicorn_json_log_config():
},
},
}
return log_config

7
litellm/anthropic_beta_headers_config.json
View file

@ -13,7 +13,8 @@
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
"files-api-2025-04-14",
"fast-mode-2026-02-01"
],
"bedrock": [
"advanced-tool-use-2025-11-20",
@ -22,7 +23,9 @@
"web-fetch-2025-09-10",
"code-execution-2025-08-25",
"skills-2025-10-02",
"files-api-2025-04-14"
"files-api-2025-04-14",
"fast-mode-2026-02-01",
"mcp-servers-2025-12-04"
],
"vertex_ai": [
"prompt-caching-scope-2026-01-05"

29
litellm/constants.py
View file

@ -2,6 +2,8 @@ import os
import sys
from typing import List, Literal
from litellm.litellm_core_utils.env_utils import get_env_int
DEFAULT_HEALTH_CHECK_PROMPT = str(
os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")
)
@ -46,6 +48,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
)
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
# Model cost map validation constants
MODEL_COST_MAP_MIN_MODEL_COUNT = int(
os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50)
) # Minimum number of models a fetched cost map must contain to be considered valid
MODEL_COST_MAP_MAX_SHRINK_RATIO = float(
os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5)
) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup)
DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)
@ -81,6 +91,20 @@ MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH = int(
os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)
)
# MCP OAuth2 Client Credentials Defaults
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS = int(
os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60")
)
MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200")
)
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600")
)
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(
os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -99,6 +123,11 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
)
)
# Maximum number of callbacks that can be registered
# This prevents callbacks from exponentially growing and consuming CPU resources
# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 30)
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)

53
litellm/integrations/custom_guardrail.py
View file

@ -616,6 +616,7 @@ class CustomGuardrail(CustomLogger):
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
original_inputs: Optional[Dict] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -625,6 +626,17 @@ class CustomGuardrail(CustomLogger):
# Convert None to empty dict to satisfy type requirements
guardrail_response = {} if response is None else response
# For apply_guardrail functions in custom_code_guardrail scenario,
# simplify the logged response to "allow", "deny", or "mask"
if original_inputs is not None and isinstance(response, dict):
# Check if inputs were modified by comparing them
if self._inputs_were_modified(original_inputs, response):
guardrail_response = "mask"
else:
guardrail_response = "allow"
verbose_logger.debug(f"Guardrail response: {response}")
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=guardrail_response,
request_data=request_data,
@ -650,8 +662,14 @@ class CustomGuardrail(CustomLogger):
This gets logged on downsteam Langfuse, DataDog, etc.
"""
# For custom_code_guardrail scenario, log as "deny" instead of full exception
# Check if this is from custom_code_guardrail by checking the class name
guardrail_response: Union[Exception, str] = e
if "CustomCodeGuardrail" in self.__class__.__name__:
guardrail_response = "deny"
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=e,
guardrail_json_response=guardrail_response,
request_data=request_data,
guardrail_status="guardrail_failed_to_respond",
duration=duration,
@ -661,6 +679,25 @@ class CustomGuardrail(CustomLogger):
)
raise e
def _inputs_were_modified(self, original_inputs: Dict, response: Dict) -> bool:
"""
Compare original inputs with response to determine if content was modified.
Returns True if the inputs were modified (mask scenario), False otherwise (allow scenario).
"""
# Get all keys from both dictionaries
all_keys = set(original_inputs.keys()) | set(response.keys())
# Compare each key's value
for key in all_keys:
original_value = original_inputs.get(key)
response_value = response.get(key)
if original_value != response_value:
return True
# No modifications detected
return False
def mask_content_in_string(
self,
content_string: str,
@ -768,6 +805,12 @@ def log_guardrail_information(func):
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
# Store original inputs for comparison (for apply_guardrail functions)
original_inputs = None
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
try:
response = await func(*args, **kwargs)
return self._process_response(
@ -777,6 +820,7 @@ def log_guardrail_information(func):
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
original_inputs=original_inputs,
)
except Exception as e:
return self._process_error(
@ -794,6 +838,12 @@ def log_guardrail_information(func):
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
# Store original inputs for comparison (for apply_guardrail functions)
original_inputs = None
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
try:
response = func(*args, **kwargs)
return self._process_response(
@ -801,6 +851,7 @@ def log_guardrail_information(func):
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
original_inputs=original_inputs,
)
except Exception as e:
return self._process_error(

31
litellm/integrations/custom_logger.py
View file

@ -664,6 +664,37 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
return final_response
"""
pass
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
) -> Tuple[bool, Dict]:
"""
Hook to determine if chat completion agentic loop should be executed.
"""
return False, {}
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
optional_params: Dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: Dict,
) -> Any:
"""
Hook to execute chat completion agentic loop based on context from should_run hook.
"""
pass
# Useful helpers for custom logger classes

113
litellm/integrations/datadog/datadog.py
View file

@ -45,7 +45,14 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.datadog import *
from litellm.types.integrations.datadog import (
DD_ERRORS,
DD_MAX_BATCH_SIZE,
DataDogStatus,
DatadogInitParams,
DatadogPayload,
DatadogProxyFailureHookJsonMessage,
)
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
from litellm.types.utils import StandardLoggingPayload
@ -85,12 +92,14 @@ class DataDogLogger(
"""
try:
verbose_logger.debug("Datadog: in init datadog logger")
self.is_mock_mode = should_use_datadog_mock()
if self.is_mock_mode:
create_mock_datadog_client()
verbose_logger.debug("[DATADOG MOCK] Datadog logger initialized in mock mode")
verbose_logger.debug(
"[DATADOG MOCK] Datadog logger initialized in mock mode"
)
#########################################################
# Handle datadog_params set as litellm.datadog_params
@ -209,6 +218,96 @@ class DataDogLogger(
)
pass
async def async_post_call_failure_hook(
self,
request_data: dict,
original_exception: Exception,
user_api_key_dict: Any,
traceback_str: Optional[str] = None,
) -> Optional[Any]:
"""
Log proxy-level failures (e.g. 401 auth, DB connection errors) to Datadog.
Ensures failures that occur before or outside the LLM completion flow
(e.g. ConnectError during auth when DB is down) are visible in Datadog
alongside Prometheus.
"""
try:
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,
traceback_str=traceback_str,
)
_code = error_information.get("error_code") or ""
status_code: Optional[int] = None
if _code and str(_code).strip().isdigit():
status_code = int(_code)
# Use project-standard sanitized user context when running in proxy
user_context: Dict[str, Any] = {}
try:
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
)
_meta = (
LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
user_api_key_dict=user_api_key_dict
)
)
user_context = dict(_meta) if isinstance(_meta, dict) else _meta
except Exception:
# Fallback if proxy not available (e.g. SDK-only): minimal safe fields
if hasattr(user_api_key_dict, "request_route"):
user_context["request_route"] = getattr(
user_api_key_dict, "request_route", None
)
if hasattr(user_api_key_dict, "team_id"):
user_context["team_id"] = getattr(
user_api_key_dict, "team_id", None
)
if hasattr(user_api_key_dict, "user_id"):
user_context["user_id"] = getattr(
user_api_key_dict, "user_id", None
)
if hasattr(user_api_key_dict, "end_user_id"):
user_context["end_user_id"] = getattr(
user_api_key_dict, "end_user_id", None
)
message_payload: DatadogProxyFailureHookJsonMessage = {
"exception": error_information.get("error_message")
or str(original_exception),
"error_class": error_information.get("error_class")
or original_exception.__class__.__name__,
"status_code": status_code,
"traceback": error_information.get("traceback") or "",
"user_api_key_dict": user_context,
}
dd_payload = DatadogPayload(
ddsource=get_datadog_source(),
ddtags=get_datadog_tags(),
hostname=get_datadog_hostname(),
message=safe_dumps(message_payload),
service=get_datadog_service(),
status=DataDogStatus.ERROR,
)
self._add_trace_context_to_payload(dd_payload=dd_payload)
self.log_queue.append(dd_payload)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"Datadog: async_post_call_failure_hook - {str(e)}\n{traceback.format_exc()}"
)
return None
async def async_send_batch(self):
"""
Sends the in memory logs queue to datadog api
@ -230,9 +329,11 @@ class DataDogLogger(
len(self.log_queue),
self.intake_url,
)
if self.is_mock_mode:
verbose_logger.debug("[DATADOG MOCK] Mock mode enabled - API calls will be intercepted")
verbose_logger.debug(
"[DATADOG MOCK] Mock mode enabled - API calls will be intercepted"
)
response = await self.async_send_compressed_data(self.log_queue)
if response.status_code == 413:

242
litellm/integrations/opentelemetry.py
View file

@ -5,6 +5,10 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
import litellm
from litellm._logging import verbose_logger
from litellm.integrations._types.open_inference import (
OpenInferenceSpanKindValues,
SpanAttributes,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.secret_managers.main import get_secret_bool
@ -17,10 +21,6 @@ from litellm.types.utils import (
StandardCallbackDynamicParams,
StandardLoggingPayload,
)
from litellm.integrations._types.open_inference import (
OpenInferenceSpanKindValues,
SpanAttributes,
)
# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
@ -40,7 +40,9 @@ if TYPE_CHECKING:
Context = Union[_Context, Any]
SpanExporter = Union[_SpanExporter, Any]
UserAPIKeyAuth = Union[_UserAPIKeyAuth, Any]
ManagementEndpointLoggingPayload = Union[_ManagementEndpointLoggingPayload, Any]
ManagementEndpointLoggingPayload = Union[
_ManagementEndpointLoggingPayload, Any
]
else:
Span = Any
Tracer = Any
@ -70,6 +72,13 @@ class OpenTelemetryConfig:
model_id: Optional[str] = None
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
# automatically infer "otlp_http" to send traces to the endpoint.
# This fixes an issue where UI-configured OTEL settings would default
# to console output instead of sending traces to the configured endpoint.
if self.endpoint and isinstance(self.exporter, str) and self.exporter == "console":
self.exporter = "otlp_http"
if not self.service_name:
self.service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
if not self.deployment_environment:
@ -95,12 +104,16 @@ class OpenTelemetryConfig:
exporter = os.getenv(
"OTEL_EXPORTER_OTLP_PROTOCOL", os.getenv("OTEL_EXPORTER", "console")
)
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT"))
endpoint = os.getenv(
"OTEL_EXPORTER_OTLP_ENDPOINT", os.getenv("OTEL_ENDPOINT")
)
headers = os.getenv(
"OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS")
) # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
enable_metrics: bool = (
os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower()
os.getenv(
"LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false"
).lower()
== "true"
)
enable_events: bool = (
@ -108,7 +121,9 @@ class OpenTelemetryConfig:
== "true"
)
service_name = os.getenv("OTEL_SERVICE_NAME", "litellm")
deployment_environment = os.getenv("OTEL_ENVIRONMENT_NAME", "production")
deployment_environment = os.getenv(
"OTEL_ENVIRONMENT_NAME", "production"
)
model_id = os.getenv("OTEL_MODEL_ID", service_name)
if exporter == "in_memory":
@ -157,7 +172,9 @@ class OpenTelemetry(CustomLogger):
logging.getLogger(__name__)
# Enable OpenTelemetry logging
otel_exporter_logger = logging.getLogger("opentelemetry.sdk.trace.export")
otel_exporter_logger = logging.getLogger(
"opentelemetry.sdk.trace.export"
)
otel_exporter_logger.setLevel(logging.DEBUG)
# init CustomLogger params
@ -253,7 +270,9 @@ class OpenTelemetry(CustomLogger):
# Don't call set_provider to preserve existing context
else:
# Default proxy provider or unknown type, create our own
verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
verbose_logger.debug(
"OpenTelemetry: Creating new %s", provider_name
)
provider = create_new_provider_fn()
set_provider_fn(provider)
except Exception as e:
@ -274,7 +293,9 @@ class OpenTelemetry(CustomLogger):
from opentelemetry.trace import SpanKind
def create_tracer_provider():
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
provider = TracerProvider(
resource=self._get_litellm_resource(self.config)
)
provider.add_span_processor(self._get_span_processor())
return provider
@ -388,10 +409,14 @@ class OpenTelemetry(CustomLogger):
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
self._handle_failure(kwargs, response_obj, start_time, end_time)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_success_event(
self, kwargs, response_obj, start_time, end_time
):
self._handle_success(kwargs, response_obj, start_time, end_time)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
async def async_log_failure_event(
self, kwargs, response_obj, start_time, end_time
):
self._handle_failure(kwargs, response_obj, start_time, end_time)
async def async_service_success_hook(
@ -588,7 +613,9 @@ class OpenTelemetry(CustomLogger):
if dynamic_headers is not None:
# Create spans using a temporary tracer with dynamic headers
tracer_to_use = self._get_tracer_with_dynamic_headers(dynamic_headers)
tracer_to_use = self._get_tracer_with_dynamic_headers(
dynamic_headers
)
verbose_logger.debug(
"Using dynamic headers for this request: %s", dynamic_headers
)
@ -624,7 +651,9 @@ class OpenTelemetry(CustomLogger):
)
# Create a temporary tracer provider with dynamic headers
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
temp_provider = TracerProvider(
resource=self._get_litellm_resource(self.config)
)
temp_provider.add_span_processor(
self._get_span_processor(dynamic_headers=dynamic_headers)
)
@ -755,7 +784,9 @@ class OpenTelemetry(CustomLogger):
metadata = litellm_params.get("metadata") or {}
generation_name = metadata.get("generation_name")
raw_span_name = generation_name if generation_name else RAW_REQUEST_SPAN_NAME
raw_span_name = (
generation_name if generation_name else RAW_REQUEST_SPAN_NAME
)
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
raw_span = otel_tracer.start_span(
@ -780,7 +811,9 @@ class OpenTelemetry(CustomLogger):
}
std_log = kwargs.get("standard_logging_object")
md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
md = getattr(std_log, "metadata", None) or (std_log or {}).get(
"metadata", {}
)
for key in [
"user_api_key_hash",
"user_api_key_alias",
@ -802,9 +835,9 @@ class OpenTelemetry(CustomLogger):
common_attrs[f"metadata.{key}"] = str(md[key])
# get hidden params
hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get(
"hidden_params", {}
)
hidden_params = getattr(std_log, "hidden_params", None) or (
std_log or {}
).get("hidden_params", {})
if hidden_params:
common_attrs["hidden_params"] = safe_dumps(hidden_params)
@ -838,7 +871,9 @@ class OpenTelemetry(CustomLogger):
self._record_response_duration_metric(kwargs, end_time, common_attrs)
@staticmethod
def _to_timestamp(val: Optional[Union[datetime, float, str]]) -> Optional[float]:
def _to_timestamp(
val: Optional[Union[datetime, float, str]],
) -> Optional[float]:
"""Convert datetime/float/string to timestamp."""
if val is None:
return None
@ -855,7 +890,9 @@ class OpenTelemetry(CustomLogger):
except ValueError:
return None
def _record_time_to_first_token_metric(self, kwargs: dict, common_attrs: dict):
def _record_time_to_first_token_metric(
self, kwargs: dict, common_attrs: dict
):
"""Record Time to First Token (TTFT) metric for streaming requests."""
optional_params = kwargs.get("optional_params", {})
is_streaming = optional_params.get("stream", False)
@ -868,7 +905,10 @@ class OpenTelemetry(CustomLogger):
api_call_start_time = kwargs.get("api_call_start_time", None)
completion_start_time = kwargs.get("completion_start_time", None)
if api_call_start_time is not None and completion_start_time is not None:
if (
api_call_start_time is not None
and completion_start_time is not None
):
# Convert to timestamps if needed (handles datetime, float, and string)
api_call_start_ts = self._to_timestamp(api_call_start_time)
completion_start_ts = self._to_timestamp(completion_start_time)
@ -876,7 +916,9 @@ class OpenTelemetry(CustomLogger):
if api_call_start_ts is None or completion_start_ts is None:
return # Skip recording if conversion failed
time_to_first_token_seconds = completion_start_ts - api_call_start_ts
time_to_first_token_seconds = (
completion_start_ts - api_call_start_ts
)
self._time_to_first_token_histogram.record(
time_to_first_token_seconds, attributes=common_attrs
)
@ -946,7 +988,9 @@ class OpenTelemetry(CustomLogger):
generation_time_seconds = duration_s
if generation_time_seconds > 0:
time_per_output_token_seconds = generation_time_seconds / completion_tokens
time_per_output_token_seconds = (
generation_time_seconds / completion_tokens
)
self._time_per_output_token_histogram.record(
time_per_output_token_seconds, attributes=common_attrs
)
@ -1007,21 +1051,26 @@ class OpenTelemetry(CustomLogger):
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
from opentelemetry._logs import (
SeverityNumber,
get_logger,
)
try:
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
except ImportError:
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0
# MyPy evaluates both branches of try/except imports and can fail when
# newer OTEL stubs remove/relocate symbols. Gate the typing import so
# only the canonical location is type-checked.
if TYPE_CHECKING:
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord
else:
try:
from opentelemetry.sdk._logs import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined]
)
except ImportError:
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord
otel_logger = get_logger(LITELLM_LOGGER_NAME)
# Get the resource from the logger provider
logger_provider = get_logger_provider()
resource = getattr(
logger_provider, "_resource", None
) or self._get_litellm_resource(self.config)
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(
"custom_llm_provider", "Unknown"
@ -1030,7 +1079,10 @@ class OpenTelemetry(CustomLogger):
# per-message events
for msg in kwargs.get("messages", []):
role = msg.get("role", "user")
attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider}
attrs = {
"event_name": "gen_ai.content.prompt",
"gen_ai.system": provider,
}
if role == "tool" and msg.get("id"):
attrs["id"] = msg["id"]
if self.message_logging and msg.get("content"):
@ -1044,7 +1096,6 @@ class OpenTelemetry(CustomLogger):
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=msg.copy(),
resource=resource,
attributes=attrs,
)
otel_logger.emit(log_record)
@ -1076,7 +1127,6 @@ class OpenTelemetry(CustomLogger):
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=body,
resource=resource,
attributes=attrs,
)
otel_logger.emit(log_record)
@ -1146,7 +1196,9 @@ class OpenTelemetry(CustomLogger):
value=guardrail_information.get("guardrail_mode"),
)
masked_entity_count = guardrail_information.get("masked_entity_count")
masked_entity_count = guardrail_information.get(
"masked_entity_count"
)
if masked_entity_count is not None:
guardrail_span.set_attribute(
"masked_entity_count", safe_dumps(masked_entity_count)
@ -1173,8 +1225,9 @@ class OpenTelemetry(CustomLogger):
# Decide whether to create a primary span
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span = parent_otel_span is None or get_secret_bool(
"USE_OTEL_LITELLM_REQUEST_SPAN"
should_create_primary_span = (
parent_otel_span is None
or get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN")
)
if should_create_primary_span:
@ -1200,7 +1253,9 @@ class OpenTelemetry(CustomLogger):
if parent_otel_span.is_recording():
parent_otel_span.set_status(Status(StatusCode.ERROR))
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
self._record_exception_on_span(
span=parent_otel_span, kwargs=kwargs
)
# Create span for guardrail information
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
@ -1223,7 +1278,9 @@ class OpenTelemetry(CustomLogger):
2. Sets structured error attributes from StandardLoggingPayloadErrorInformation
"""
try:
from litellm.integrations._types.open_inference import ErrorAttributes
from litellm.integrations._types.open_inference import (
ErrorAttributes,
)
# Get the exception object if available
exception = kwargs.get("exception")
@ -1233,15 +1290,17 @@ class OpenTelemetry(CustomLogger):
span.record_exception(exception)
# Get StandardLoggingPayload for structured error information
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
standard_logging_payload: Optional[StandardLoggingPayload] = (
kwargs.get("standard_logging_object")
)
if standard_logging_payload is None:
return
# Extract error_information from StandardLoggingPayload
error_information = standard_logging_payload.get("error_information")
error_information = standard_logging_payload.get(
"error_information"
)
if error_information is None:
# Fallback to error_str if error_information is not available
@ -1331,7 +1390,9 @@ class OpenTelemetry(CustomLogger):
)
pass
def cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]:
def cast_as_primitive_value_type(
self, value
) -> Union[str, bool, int, float]:
"""
Casts the value to a primitive OTEL type if it is not already a primitive type.
@ -1401,8 +1462,8 @@ class OpenTelemetry(CustomLogger):
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {}) or {}
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"
standard_logging_payload: Optional[StandardLoggingPayload] = (
kwargs.get("standard_logging_object")
)
if standard_logging_payload is None:
raise ValueError("standard_logging_object not found in kwargs")
@ -1424,11 +1485,13 @@ class OpenTelemetry(CustomLogger):
) or (standard_logging_payload or {}).get("hidden_params", {})
if hidden_params:
self.safe_set_attribute(
span=span, key="hidden_params", value=safe_dumps(hidden_params)
span=span,
key="hidden_params",
value=safe_dumps(hidden_params),
)
# Cost breakdown tracking
cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get(
"cost_breakdown"
cost_breakdown: Optional[CostBreakdown] = (
standard_logging_payload.get("cost_breakdown")
)
if cost_breakdown:
for key, value in cost_breakdown.items():
@ -1504,7 +1567,9 @@ class OpenTelemetry(CustomLogger):
# The unique identifier for the completion.
if response_obj and response_obj.get("id"):
self.safe_set_attribute(
span=span, key="gen_ai.response.id", value=response_obj.get("id")
span=span,
key="gen_ai.response.id",
value=response_obj.get("id"),
)
# The model used to generate the response.
@ -1639,7 +1704,9 @@ class OpenTelemetry(CustomLogger):
"OpenTelemetry logging error in set_attributes %s", str(e)
)
def _cast_as_primitive_value_type(self, value) -> Union[str, bool, int, float]:
def _cast_as_primitive_value_type(
self, value
) -> Union[str, bool, int, float]:
"""
Casts the value to a primitive OTEL type if it is not already a primitive type.
@ -1673,7 +1740,10 @@ class OpenTelemetry(CustomLogger):
if isinstance(messages, str):
# Handle system_instructions passed as a string
return [
{"role": "system", "parts": [{"type": "text", "content": messages}]}
{
"role": "system",
"parts": [{"type": "text", "content": messages}],
}
]
transformed = []
@ -1714,9 +1784,11 @@ class OpenTelemetry(CustomLogger):
message = choice.get("message") or {}
finish_reason = choice.get("finish_reason")
transformed_msg = self._transform_messages_to_otel_semantic_conventions(
[message]
)[0]
transformed_msg = (
self._transform_messages_to_otel_semantic_conventions(
[message]
)[0]
)
if finish_reason:
transformed_msg["finish_reason"] = finish_reason
@ -1728,7 +1800,9 @@ class OpenTelemetry(CustomLogger):
self.set_attributes(span, kwargs, response_obj)
kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")
custom_llm_provider = litellm_params.get(
"custom_llm_provider", "Unknown"
)
_raw_response = kwargs.get("original_response")
_additional_args = kwargs.get("additional_args", {}) or {}
@ -1741,7 +1815,9 @@ class OpenTelemetry(CustomLogger):
if complete_input_dict and isinstance(complete_input_dict, dict):
for param, val in complete_input_dict.items():
self.safe_set_attribute(
span=span, key=f"llm.{custom_llm_provider}.{param}", value=val
span=span,
key=f"llm.{custom_llm_provider}.{param}",
value=val,
)
#############################################
@ -1773,7 +1849,8 @@ class OpenTelemetry(CustomLogger):
)
except Exception as e:
verbose_logger.exception(
"OpenTelemetry logging error in set_raw_request_attributes %s", str(e)
"OpenTelemetry logging error in set_raw_request_attributes %s",
str(e),
)
def _to_ns(self, dt):
@ -1813,7 +1890,9 @@ class OpenTelemetry(CustomLogger):
)
litellm_params = kwargs.get("litellm_params", {}) or {}
proxy_server_request = litellm_params.get("proxy_server_request", {}) or {}
proxy_server_request = (
litellm_params.get("proxy_server_request", {}) or {}
)
headers = proxy_server_request.get("headers", {}) or {}
traceparent = headers.get("traceparent", None)
_metadata = litellm_params.get("metadata", {}) or {}
@ -1832,7 +1911,10 @@ class OpenTelemetry(CustomLogger):
"OpenTelemetry: Using traceparent header for context propagation"
)
carrier = {"traceparent": traceparent}
return TraceContextTextMapPropagator().extract(carrier=carrier), None
return (
TraceContextTextMapPropagator().extract(carrier=carrier),
None,
)
# Priority 3: Active span from global context (auto-detection)
try:
@ -1960,10 +2042,14 @@ class OpenTelemetry(CustomLogger):
self.OTEL_HEADERS,
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(
self.OTEL_HEADERS
)
# Normalize endpoint for logs - ensure it points to /v1/logs instead of /v1/traces
normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "logs")
normalized_endpoint = self._normalize_otel_endpoint(
self.OTEL_ENDPOINT, "logs"
)
verbose_logger.debug(
"OpenTelemetry: Log endpoint normalized from %s to %s",
@ -2051,14 +2137,18 @@ class OpenTelemetry(CustomLogger):
self.OTEL_HEADERS,
)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
_split_otel_headers = OpenTelemetry._get_headers_dictionary(
self.OTEL_HEADERS
)
normalized_endpoint = self._normalize_otel_endpoint(
self.OTEL_ENDPOINT, "metrics"
)
if self.OTEL_EXPORTER == "console":
exporter = ConsoleMetricExporter()
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
return PeriodicExportingMetricReader(
exporter, export_interval_millis=5000
)
elif (
self.OTEL_EXPORTER == "otlp_http"
@ -2074,7 +2164,9 @@ class OpenTelemetry(CustomLogger):
headers=_split_otel_headers,
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
return PeriodicExportingMetricReader(
exporter, export_interval_millis=5000
)
elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
try:
@ -2092,7 +2184,9 @@ class OpenTelemetry(CustomLogger):
headers=_split_otel_headers,
preferred_temporality={Histogram: AggregationTemporality.DELTA},
)
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
return PeriodicExportingMetricReader(
exporter, export_interval_millis=5000
)
else:
verbose_logger.warning(
@ -2100,7 +2194,9 @@ class OpenTelemetry(CustomLogger):
self.OTEL_EXPORTER,
)
exporter = ConsoleMetricExporter()
return PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
return PeriodicExportingMetricReader(
exporter, export_interval_millis=5000
)
def _normalize_otel_endpoint(
self, endpoint: Optional[str], signal_type: str
@ -2171,7 +2267,9 @@ class OpenTelemetry(CustomLogger):
return endpoint
@staticmethod
def _get_headers_dictionary(headers: Optional[Union[str, dict]]) -> Dict[str, str]:
def _get_headers_dictionary(
headers: Optional[Union[str, dict]],
) -> Dict[str, str]:
"""
Convert a string or dictionary of headers into a dictionary of headers.
"""

46
litellm/integrations/prometheus.py
View file

@ -29,7 +29,10 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
from litellm.types.integrations.prometheus import (
_sanitize_prometheus_label_name,
_sanitize_prometheus_label_value,
)
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
@ -1276,11 +1279,17 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_api_key_requests_for_model.labels(
user_api_key, user_api_key_alias, model_group, model_id
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_requests)
self.litellm_remaining_api_key_tokens_for_model.labels(
user_api_key, user_api_key_alias, model_group, model_id
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_tokens)
def _set_latency_metrics(
@ -1401,14 +1410,14 @@ class PrometheusLogger(CustomLogger):
try:
self.litellm_llm_api_failed_requests_metric.labels(
end_user_id,
user_api_key,
user_api_key_alias,
model,
user_api_team,
user_api_team_alias,
user_id,
standard_logging_payload.get("model_id", ""),
_sanitize_prometheus_label_value(end_user_id),
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model),
_sanitize_prometheus_label_value(user_api_team),
_sanitize_prometheus_label_value(user_api_team_alias),
_sanitize_prometheus_label_value(user_id),
_sanitize_prometheus_label_value(standard_logging_payload.get("model_id", "")),
).inc()
self.set_llm_deployment_failure_metrics(kwargs)
except Exception as e:
@ -2354,7 +2363,11 @@ class PrometheusLogger(CustomLogger):
increment metric when litellm.Router / load balancing logic places a deployment in cool down
"""
self.litellm_deployment_cooled_down.labels(
litellm_model_name, model_id, api_base, api_provider, exception_status
_sanitize_prometheus_label_value(litellm_model_name),
_sanitize_prometheus_label_value(model_id),
_sanitize_prometheus_label_value(api_base),
_sanitize_prometheus_label_value(api_provider),
_sanitize_prometheus_label_value(exception_status),
).inc()
def increment_callback_logging_failure(
@ -3074,9 +3087,10 @@ def prometheus_label_factory(
# Extract dictionary from Pydantic object
enum_dict = enum_values.model_dump()
# Filter supported labels
# Filter supported labels and sanitize values to prevent breaking
# the Prometheus text format (e.g. U+2028 Line Separator in label values)
filtered_labels = {
label: value
label: _sanitize_prometheus_label_value(value)
for label, value in enum_dict.items()
if label in supported_enum_labels
}
@ -3094,14 +3108,14 @@ def prometheus_label_factory(
# check sanitized key
sanitized_key = _sanitize_prometheus_label_name(key)
if sanitized_key in supported_enum_labels:
filtered_labels[sanitized_key] = value
filtered_labels[sanitized_key] = _sanitize_prometheus_label_value(value)
# Add custom tags if configured
if enum_values.tags is not None:
custom_tag_labels = get_custom_labels_from_tags(enum_values.tags)
for key, value in custom_tag_labels.items():
if key in supported_enum_labels:
filtered_labels[key] = value
filtered_labels[key] = _sanitize_prometheus_label_value(value)
for label in supported_enum_labels:
if label not in filtered_labels:

269
litellm/integrations/websearch_interception/handler.py
View file

@ -17,6 +17,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.tools import (
get_litellm_web_search_tool,
is_web_search_tool,
is_web_search_tool_chat_completion,
)
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
@ -48,7 +49,8 @@ class WebSearchInterceptionLogger(CustomLogger):
Args:
enabled_providers: List of LLM providers to enable interception for.
Use LlmProviders enum values (e.g., [LlmProviders.BEDROCK])
Default: [LlmProviders.BEDROCK]
If None or empty list, enables for ALL providers.
Default: None (all providers enabled)
search_tool_name: Name of search tool configured in router's search_tools.
If None, will attempt to use first available search tool.
"""
@ -183,10 +185,10 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
f"WebSearchInterception: Pre-request hook called"
f" - custom_llm_provider={custom_llm_provider}"
f" - enabled_providers={self.enabled_providers}"
f" - enabled_providers={self.enabled_providers or 'ALL'}"
)
if custom_llm_provider not in self.enabled_providers:
if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers:
verbose_logger.debug(
f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}"
)
@ -245,7 +247,12 @@ class WebSearchInterceptionLogger(CustomLogger):
custom_llm_provider: str,
kwargs: Dict,
) -> Tuple[bool, Dict]:
"""Check if WebSearch tool interception is needed"""
"""
Check if WebSearch tool interception is needed for Anthropic Messages API.
This is the legacy method for Anthropic-style responses.
For chat completions, use async_should_run_chat_completion_agentic_loop instead.
"""
verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}")
verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
@ -253,7 +260,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# Check if provider should be intercepted
# Note: custom_llm_provider is already normalized by get_llm_provider()
# (e.g., "bedrock/invoke/..." -> "bedrock")
if custom_llm_provider not in self.enabled_providers:
if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers:
verbose_logger.debug(
f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
)
@ -267,10 +274,11 @@ class WebSearchInterceptionLogger(CustomLogger):
)
return False, {}
# Detect WebSearch tool_use in response
# Detect WebSearch tool_use in response (Anthropic format)
should_intercept, tool_calls = WebSearchTransformation.transform_request(
response=response,
stream=stream,
response_format="anthropic",
)
if not should_intercept:
@ -288,6 +296,67 @@ class WebSearchInterceptionLogger(CustomLogger):
"tool_calls": tool_calls,
"tool_type": "websearch",
"provider": custom_llm_provider,
"response_format": "anthropic",
}
return True, tools_dict
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
) -> Tuple[bool, Dict]:
"""
Check if WebSearch tool interception is needed for Chat Completions API.
Similar to async_should_run_agentic_loop but for OpenAI-style chat completions.
"""
verbose_logger.debug(f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}")
verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
# Check if provider should be intercepted
if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers:
verbose_logger.debug(
f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
)
return False, {}
# Check if tools include any web search tool (strict check for chat completions)
has_websearch_tool = any(is_web_search_tool_chat_completion(t) for t in (tools or []))
if not has_websearch_tool:
verbose_logger.debug(
"WebSearchInterception: No litellm_web_search tool in request"
)
return False, {}
# Detect WebSearch tool_calls in response (OpenAI format)
should_intercept, tool_calls = WebSearchTransformation.transform_request(
response=response,
stream=stream,
response_format="openai",
)
if not should_intercept:
verbose_logger.debug(
"WebSearchInterception: No WebSearch tool_calls detected in response"
)
return False, {}
verbose_logger.debug(
f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
)
# Return tools dict with tool calls
tools_dict = {
"tool_calls": tool_calls,
"tool_type": "websearch",
"provider": custom_llm_provider,
"response_format": "openai",
}
return True, tools_dict
@ -303,7 +372,11 @@ class WebSearchInterceptionLogger(CustomLogger):
stream: bool,
kwargs: Dict,
) -> Any:
"""Execute agentic loop with WebSearch execution"""
"""
Execute agentic loop with WebSearch execution for Anthropic Messages API.
This is the legacy method for Anthropic-style responses.
"""
tool_calls = tools["tool_calls"]
@ -321,6 +394,41 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs=kwargs,
)
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
optional_params: Dict,
logging_obj: Any,
stream: bool,
kwargs: Dict,
) -> Any:
"""
Execute agentic loop with WebSearch execution for Chat Completions API.
Similar to async_run_agentic_loop but for OpenAI-style chat completions.
"""
tool_calls = tools["tool_calls"]
response_format = tools.get("response_format", "openai")
verbose_logger.debug(
f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)"
)
return await self._execute_chat_completion_agentic_loop(
model=model,
messages=messages,
tool_calls=tool_calls,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs,
response_format=response_format,
)
async def _execute_agentic_loop(
self,
model: str,
@ -382,7 +490,8 @@ class WebSearchInterceptionLogger(CustomLogger):
)
# Make follow-up request with search results
follow_up_messages = messages + [assistant_message, user_message]
# Type cast: user_message is a Dict for Anthropic format (default response_format)
follow_up_messages = messages + [assistant_message, cast(Dict, user_message)]
verbose_logger.debug(
"WebSearchInterception: Making follow-up request with search results"
@ -521,6 +630,150 @@ class WebSearchInterceptionLogger(CustomLogger):
)
raise
async def _execute_chat_completion_agentic_loop( # noqa: PLR0915
self,
model: str,
messages: List[Dict],
tool_calls: List[Dict],
optional_params: Dict,
logging_obj: Any,
stream: bool,
kwargs: Dict,
response_format: str = "openai",
) -> Any:
"""Execute litellm.search() and make follow-up chat completion request"""
# Extract search queries from tool_calls
search_tasks = []
for tool_call in tool_calls:
# Handle both Anthropic-style input and OpenAI-style function.arguments
query = None
if "input" in tool_call and isinstance(tool_call["input"], dict):
query = tool_call["input"].get("query")
elif "function" in tool_call:
func = tool_call["function"]
if isinstance(func, dict):
args = func.get("arguments", {})
if isinstance(args, dict):
query = args.get("query")
if query:
verbose_logger.debug(
f"WebSearchInterception: Queuing search for query='{query}'"
)
search_tasks.append(self._execute_search(query))
else:
verbose_logger.warning(
f"WebSearchInterception: Tool call {tool_call.get('id')} has no query"
)
# Add empty result for tools without query
search_tasks.append(self._create_empty_search_result())
# Execute searches in parallel
verbose_logger.debug(
f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel"
)
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
# Handle any exceptions in search results
final_search_results: List[str] = []
for i, result in enumerate(search_results):
if isinstance(result, Exception):
verbose_logger.error(
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
)
final_search_results.append(
f"Search failed: {str(result)}"
)
elif isinstance(result, str):
final_search_results.append(cast(str, result))
else:
verbose_logger.warning(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
)
final_search_results.append(str(result))
# Build assistant and tool messages using transformation
assistant_message, tool_messages_or_user = WebSearchTransformation.transform_response(
tool_calls=tool_calls,
search_results=final_search_results,
response_format=response_format,
)
# Make follow-up request with search results
# For OpenAI format, tool_messages_or_user is a list of tool messages
if response_format == "openai":
follow_up_messages = messages + [assistant_message] + cast(List[Dict], tool_messages_or_user)
else:
# For Anthropic format (shouldn't happen in this method, but handle it)
follow_up_messages = messages + [assistant_message, cast(Dict, tool_messages_or_user)]
verbose_logger.debug(
"WebSearchInterception: Making follow-up chat completion request with search results"
)
verbose_logger.debug(
f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
)
# Use litellm.acompletion for follow-up request
try:
# Remove internal parameters that shouldn't be passed to follow-up request
internal_params = {
'_websearch_interception',
'acompletion',
'litellm_logging_obj',
'custom_llm_provider',
'model_alias_map',
'stream_response',
'custom_prompt_dict',
}
kwargs_for_followup = {
k: v for k, v in kwargs.items()
if not k.startswith('_websearch_interception') and k not in internal_params
}
# Get full model name from kwargs
full_model_name = model
if "custom_llm_provider" in kwargs:
custom_llm_provider = kwargs["custom_llm_provider"]
# Reconstruct full model name with provider prefix if needed
if not model.startswith(custom_llm_provider):
# Check if model already has a provider prefix
if "/" not in model:
full_model_name = f"{custom_llm_provider}/{model}"
verbose_logger.debug(
f"WebSearchInterception: Using model name: {full_model_name}"
)
# Prepare tools for follow-up request (same as original)
tools_param = optional_params.get("tools")
# Remove tools and extra_body from optional_params to avoid issues
# extra_body often contains internal LiteLLM params that shouldn't be forwarded
optional_params_clean = {
k: v for k, v in optional_params.items()
if k not in {"tools", "extra_body", "model_alias_map","stream_response", "custom_prompt_dict" }
}
final_response = await litellm.acompletion(
model=full_model_name,
messages=follow_up_messages,
tools=tools_param,
**optional_params_clean,
**kwargs_for_followup,
)
verbose_logger.debug(
f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
)
return final_response
except Exception as e:
verbose_logger.exception(
f"WebSearchInterception: Follow-up request failed: {str(e)}"
)
raise
async def _create_empty_search_result(self) -> str:
"""Create an empty search result for tool calls without queries"""
return "No search query provided"

56
litellm/integrations/websearch_interception/tools.py
View file

@ -49,12 +49,57 @@ def get_litellm_web_search_tool() -> Dict[str, Any]:
}
def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is a web search tool for Chat Completions API (strict check).
This is a stricter version that ONLY checks for the exact LiteLLM web search tool name.
Use this for Chat Completions API to avoid false positives with user-defined tools.
Detects ONLY:
- LiteLLM standard: name == "litellm_web_search" (Anthropic format)
- OpenAI format: type == "function" with function.name == "litellm_web_search"
Args:
tool: Tool dictionary to check
Returns:
True if tool is exactly the LiteLLM web search tool
Example:
>>> is_web_search_tool_chat_completion({"name": "litellm_web_search"})
True
>>> is_web_search_tool_chat_completion({"type": "function", "function": {"name": "litellm_web_search"}})
True
>>> is_web_search_tool_chat_completion({"name": "web_search"})
False
>>> is_web_search_tool_chat_completion({"name": "WebSearch"})
False
"""
tool_name = tool.get("name", "")
tool_type = tool.get("type", "")
# Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
if tool_type == "function" and "function" in tool:
function_def = tool.get("function", {})
function_name = function_def.get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
# Check for LiteLLM standard tool (Anthropic format)
if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
return False
def is_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is a web search tool (native or LiteLLM standard).
Detects:
- LiteLLM standard: name == "litellm_web_search"
- OpenAI format: type == "function" with function.name == "litellm_web_search"
- Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
- Claude Code: name == "web_search" with a type field
- Custom: name == "WebSearch" (legacy format)
@ -68,6 +113,8 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
Example:
>>> is_web_search_tool({"name": "litellm_web_search"})
True
>>> is_web_search_tool({"type": "function", "function": {"name": "litellm_web_search"}})
True
>>> is_web_search_tool({"type": "web_search_20250305", "name": "web_search"})
True
>>> is_web_search_tool({"name": "calculator"})
@ -75,8 +122,15 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
tool_name = tool.get("name", "")
tool_type = tool.get("type", "")
# Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
if tool_type == "function" and "function" in tool:
function_def = tool.get("function", {})
function_name = function_def.get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
# Check for LiteLLM standard tool
# Check for LiteLLM standard tool (Anthropic format)
if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True

190
litellm/integrations/websearch_interception/transformation.py
View file

@ -1,10 +1,10 @@
"""
WebSearch Tool Transformation
Transforms between Anthropic tool_use format and LiteLLM search format.
Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
"""
from typing import Any, Dict, List, Tuple
import json
from typing import Any, Dict, List, Tuple, Union
from litellm._logging import verbose_logger
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
@ -17,28 +17,31 @@ class WebSearchTransformation:
Handles transformation between:
- Anthropic tool_use format → LiteLLM search requests
- LiteLLM SearchResponse → Anthropic tool_result format
- OpenAI tool_calls format → LiteLLM search requests
- LiteLLM SearchResponse → Anthropic/OpenAI tool_result format
"""
@staticmethod
def transform_request(
response: Any,
stream: bool,
response_format: str = "anthropic",
) -> Tuple[bool, List[Dict]]:
"""
Transform Anthropic response to extract WebSearch tool calls.
Transform model response to extract WebSearch tool calls.
Detects if response contains WebSearch tool_use blocks and extracts
Detects if response contains WebSearch tool_use/tool_calls blocks and extracts
the search queries for execution.
Args:
response: Model response (dict or AnthropicMessagesResponse)
response: Model response (dict, AnthropicMessagesResponse, or ModelResponse)
stream: Whether response is streaming
response_format: Response format - "anthropic" or "openai" (default: "anthropic")
Returns:
(has_websearch, tool_calls):
has_websearch: True if WebSearch tool_use found
tool_calls: List of tool_use dicts with id, name, input
tool_calls: List of tool_use/tool_calls dicts with id, name, input/function
Note:
Streaming requests are handled by converting stream=True to stream=False
@ -54,8 +57,11 @@ class WebSearchTransformation:
)
return False, []
# Parse non-streaming response
return WebSearchTransformation._detect_from_non_streaming_response(response)
# Parse non-streaming response based on format
if response_format == "openai":
return WebSearchTransformation._detect_from_openai_response(response)
else:
return WebSearchTransformation._detect_from_non_streaming_response(response)
@staticmethod
def _detect_from_non_streaming_response(
@ -114,26 +120,142 @@ class WebSearchTransformation:
return len(tool_calls) > 0, tool_calls
@staticmethod
def _detect_from_openai_response(
response: Any,
) -> Tuple[bool, List[Dict]]:
"""Parse OpenAI-style response for WebSearch tool_calls"""
# Handle both dict and ModelResponse objects
if isinstance(response, dict):
choices = response.get("choices", [])
else:
if not hasattr(response, "choices"):
verbose_logger.debug(
"WebSearchInterception: Response has no choices attribute"
)
return False, []
choices = response.choices or []
if not choices:
verbose_logger.debug(
"WebSearchInterception: Response has empty choices"
)
return False, []
# Get first choice's message
first_choice = choices[0]
if isinstance(first_choice, dict):
message = first_choice.get("message", {})
else:
message = getattr(first_choice, "message", None)
if not message:
verbose_logger.debug(
"WebSearchInterception: First choice has no message"
)
return False, []
# Get tool_calls from message
if isinstance(message, dict):
openai_tool_calls = message.get("tool_calls", [])
else:
openai_tool_calls = getattr(message, "tool_calls", None) or []
if not openai_tool_calls:
verbose_logger.debug(
"WebSearchInterception: Message has no tool_calls"
)
return False, []
# Find all WebSearch tool calls
tool_calls = []
for tool_call in openai_tool_calls:
# Handle both dict and object tool calls
if isinstance(tool_call, dict):
tool_id = tool_call.get("id")
tool_type = tool_call.get("type")
function = tool_call.get("function", {})
function_name = function.get("name") if isinstance(function, dict) else getattr(function, "name", None)
function_arguments = function.get("arguments") if isinstance(function, dict) else getattr(function, "arguments", None)
else:
tool_id = getattr(tool_call, "id", None)
tool_type = getattr(tool_call, "type", None)
function = getattr(tool_call, "function", None)
function_name = getattr(function, "name", None) if function else None
function_arguments = getattr(function, "arguments", None) if function else None
# Check for LiteLLM standard or legacy web search tools
if tool_type == "function" and function_name in (
LITELLM_WEB_SEARCH_TOOL_NAME, "WebSearch", "web_search"
):
# Parse arguments (might be JSON string)
if isinstance(function_arguments, str):
try:
arguments = json.loads(function_arguments)
except json.JSONDecodeError:
verbose_logger.warning(
f"WebSearchInterception: Failed to parse function arguments: {function_arguments}"
)
arguments = {}
else:
arguments = function_arguments or {}
# Convert to internal format (similar to Anthropic)
tool_call_dict = {
"id": tool_id,
"type": "function",
"name": function_name,
"function": {
"name": function_name,
"arguments": arguments,
},
"input": arguments, # For compatibility with Anthropic format
}
tool_calls.append(tool_call_dict)
verbose_logger.debug(
f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}"
)
return len(tool_calls) > 0, tool_calls
@staticmethod
def transform_response(
tool_calls: List[Dict],
search_results: List[str],
) -> Tuple[Dict, Dict]:
response_format: str = "anthropic",
) -> Tuple[Dict, Union[Dict, List[Dict]]]:
"""
Transform LiteLLM search results to Anthropic tool_result format.
Transform LiteLLM search results to Anthropic/OpenAI tool_result format.
Builds the assistant and user messages needed for the agentic loop
Builds the assistant and user/tool messages needed for the agentic loop
follow-up request.
Args:
tool_calls: List of tool_use dicts from transform_request
tool_calls: List of tool_use/tool_calls dicts from transform_request
search_results: List of search result strings (one per tool_call)
response_format: Response format - "anthropic" or "openai" (default: "anthropic")
Returns:
(assistant_message, user_message):
assistant_message: Message with tool_use blocks
user_message: Message with tool_result blocks
(assistant_message, user_or_tool_messages):
For Anthropic: assistant_message with tool_use blocks, user_message with tool_result blocks
For OpenAI: assistant_message with tool_calls, tool_messages list with tool results
"""
if response_format == "openai":
return WebSearchTransformation._transform_response_openai(
tool_calls, search_results
)
else:
return WebSearchTransformation._transform_response_anthropic(
tool_calls, search_results
)
@staticmethod
def _transform_response_anthropic(
tool_calls: List[Dict],
search_results: List[str],
) -> Tuple[Dict, Dict]:
"""Transform to Anthropic format (single user message with tool_result blocks)"""
# Build assistant message with tool_use blocks
assistant_message = {
"role": "assistant",
@ -163,6 +285,40 @@ class WebSearchTransformation:
return assistant_message, user_message
@staticmethod
def _transform_response_openai(
tool_calls: List[Dict],
search_results: List[str],
) -> Tuple[Dict, List[Dict]]:
"""Transform to OpenAI format (assistant with tool_calls, separate tool messages)"""
# Build assistant message with tool_calls
assistant_message = {
"role": "assistant",
"tool_calls": [
{
"id": tc["id"],
"type": "function",
"function": {
"name": tc["name"],
"arguments": json.dumps(tc["input"]) if isinstance(tc["input"], dict) else str(tc["input"]),
},
}
for tc in tool_calls
],
}
# Build separate tool messages (one per tool call)
tool_messages = [
{
"role": "tool",
"tool_call_id": tool_calls[i]["id"],
"content": search_results[i],
}
for i in range(len(tool_calls))
]
return assistant_message, tool_messages
@staticmethod
def format_search_response(result: SearchResponse) -> str:
"""

21
litellm/litellm_core_utils/env_utils.py Normal file
View file

@ -0,0 +1,21 @@
"""
Utility helpers for reading and parsing environment variables.
"""
import os
def get_env_int(env_var: str, default: int) -> int:
"""Parse an environment variable as an integer, falling back to default on invalid values.
Handles empty strings, whitespace, and non-numeric values gracefully
so that misconfiguration doesn't crash the process at import time.
"""
raw = os.getenv(env_var)
if raw is None:
return default
raw = raw.strip()
try:
return int(raw)
except (ValueError, TypeError):
return default

201
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -8,40 +8,187 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
```
"""
import json
import os
from importlib.resources import files
import httpx
from litellm import verbose_logger
from litellm.constants import (
MODEL_COST_MAP_MAX_SHRINK_RATIO,
MODEL_COST_MAP_MIN_MODEL_COUNT,
)
class GetModelCostMap:
"""
Handles fetching, validating, and loading the model cost map.
Only the backup model *count* is cached (a single int). The full
backup dict is never held in memory — it is only parsed when it
needs to be *returned* as a fallback.
"""
_backup_model_count: int = -1 # -1 = not yet loaded
@staticmethod
def load_local_model_cost_map() -> dict:
"""Load the local backup model cost map bundled with the package."""
content = json.loads(
files("litellm")
.joinpath("model_prices_and_context_window_backup.json")
.read_text(encoding="utf-8")
)
return content
@classmethod
def _get_backup_model_count(cls) -> int:
"""Return the number of models in the local backup (cached int)."""
if cls._backup_model_count < 0:
backup = cls.load_local_model_cost_map()
cls._backup_model_count = len(backup)
return cls._backup_model_count
@staticmethod
def _check_is_valid_dict(fetched_map: dict) -> bool:
"""Check 1: fetched map is a non-empty dict."""
if not isinstance(fetched_map, dict):
verbose_logger.warning(
"LiteLLM: Fetched model cost map is not a dict (type=%s). "
"Falling back to local backup.",
type(fetched_map).__name__,
)
return False
if len(fetched_map) == 0:
verbose_logger.warning(
"LiteLLM: Fetched model cost map is empty. "
"Falling back to local backup.",
)
return False
return True
@classmethod
def _check_model_count_not_reduced(
cls,
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""Check 2: model count has not reduced significantly vs backup."""
fetched_count = len(fetched_map)
if fetched_count < min_model_count:
verbose_logger.warning(
"LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
"This may indicate a corrupted upstream file. "
"Falling back to local backup.",
fetched_count,
min_model_count,
)
return False
if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio:
verbose_logger.warning(
"LiteLLM: Fetched model cost map shrank significantly "
"(fetched=%d, backup=%d, threshold=%.0f%%). "
"This may indicate a corrupted upstream file. "
"Falling back to local backup.",
fetched_count,
backup_model_count,
max_shrink_ratio * 100,
)
return False
return True
@classmethod
def validate_model_cost_map(
cls,
fetched_map: dict,
backup_model_count: int,
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
) -> bool:
"""
Validate the integrity of a fetched model cost map.
Runs each check in order and returns False on the first failure.
Checks:
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
2. ``_check_model_count_not_reduced`` -- model count meets minimum
and has not shrunk >``max_shrink_ratio`` vs backup.
Returns True if all checks pass, False otherwise.
"""
if not cls._check_is_valid_dict(fetched_map):
return False
if not cls._check_model_count_not_reduced(
fetched_map=fetched_map,
backup_model_count=backup_model_count,
min_model_count=min_model_count,
max_shrink_ratio=max_shrink_ratio,
):
return False
return True
@staticmethod
def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict:
"""
Fetch the model cost map from a remote URL.
Returns the parsed JSON dict. Raises on network/parse errors
(caller is expected to handle).
"""
response = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
def get_model_cost_map(url: str) -> dict:
if (
os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False)
or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True"
):
from importlib.resources import files
import json
"""
Public entry point — returns the model cost map dict.
content = json.loads(
files("litellm")
.joinpath("model_prices_and_context_window_backup.json")
.read_text(encoding="utf-8")
)
return content
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only.
2. Otherwise fetches from ``url``, validates integrity, and falls back
to the local backup on any failure.
Only the backup model count is cached (a single int) for validation.
The full backup dict is only parsed when it must be *returned* as a
fallback — it is never held in memory long-term.
"""
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
return GetModelCostMap.load_local_model_cost_map()
try:
response = httpx.get(
url, timeout=5
) # set a 5 second timeout for the get request
response.raise_for_status() # Raise an exception if the request is unsuccessful
content = response.json()
return content
except Exception:
from importlib.resources import files
import json
content = json.loads(
files("litellm")
.joinpath("model_prices_and_context_window_backup.json")
.read_text(encoding="utf-8")
content = GetModelCostMap.fetch_remote_model_cost_map(url)
except Exception as e:
verbose_logger.warning(
"LiteLLM: Failed to fetch remote model cost map from %s: %s. "
"Falling back to local backup.",
url,
str(e),
)
return content
return GetModelCostMap.load_local_model_cost_map()
# Validate using cached count (cheap int comparison, no file I/O)
if not GetModelCostMap.validate_model_cost_map(
fetched_map=content,
backup_model_count=GetModelCostMap._get_backup_model_count(),
):
verbose_logger.warning(
"LiteLLM: Fetched model cost map failed integrity check. "
"Using local backup instead. url=%s",
url,
)
return GetModelCostMap.load_local_model_cost_map()
return content

8
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Set, Type, Uni
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_CALLBACKS
from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger
@ -24,9 +25,6 @@ class LoggingCallbackManager:
- Keep a reasonable MAX_CALLBACKS limit (this ensures callbacks don't exponentially grow and consume CPU Resources)
"""
# healthy maximum number of callbacks - unlikely someone needs more than 20
MAX_CALLBACKS = 30
def add_litellm_input_callback(self, callback: Union[CustomLogger, str]):
"""
Add a input callback to litellm.input_callback
@ -155,9 +153,9 @@ class LoggingCallbackManager:
Check if adding another callback would exceed MAX_CALLBACKS
Returns True if safe to add, False if would exceed limit
"""
if len(parent_list) >= self.MAX_CALLBACKS:
if len(parent_list) >= MAX_CALLBACKS:
verbose_logger.warning(
f"Cannot add callback - would exceed MAX_CALLBACKS limit of {self.MAX_CALLBACKS}. Current callbacks: {len(parent_list)}"
f"Cannot add callback - would exceed MAX_CALLBACKS limit of {MAX_CALLBACKS}. Current callbacks: {len(parent_list)}"
)
return False
return True

56
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1272,3 +1272,59 @@ def parse_tool_call_arguments(
)
raise ValueError(error_message) from e
def split_concatenated_json_objects(raw: str) -> List[Dict[str, Any]]:
"""
Split a string that contains one or more concatenated JSON objects into
a list of parsed dicts.
LLM providers (notably Bedrock Claude Sonnet 4.5) sometimes return
multiple tool-call argument objects concatenated in a single
``arguments`` string, e.g.::
'{"command":["curl",...]}{"command":["curl",...]}{"command":["curl",...]}'
``json.loads()`` fails on this with ``JSONDecodeError: Extra data``.
This helper uses ``json.JSONDecoder.raw_decode()`` to walk the string
and extract each JSON object individually.
Returns
-------
list[dict]
A list of parsed dicts – one per JSON object found. If *raw* is
empty or whitespace-only, an empty list is returned.
Raises
------
json.JSONDecodeError
If the string contains text that cannot be parsed as JSON at all.
"""
import json
raw = raw.strip()
if not raw:
return []
decoder = json.JSONDecoder()
results: List[Dict[str, Any]] = []
idx = 0
length = len(raw)
while idx < length:
# Skip whitespace between objects
while idx < length and raw[idx] in " \t\n\r":
idx += 1
if idx >= length:
break
obj, end_idx = decoder.raw_decode(raw, idx)
if isinstance(obj, dict):
results.append(obj)
else:
# Non-dict JSON value – wrap in empty dict (Bedrock requires
# toolUse.input to be an object).
results.append({})
idx = end_idx
return results

59
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3287,25 +3287,68 @@ def _convert_to_bedrock_tool_call_invoke(
- extract name
- extract id
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
split_concatenated_json_objects,
)
try:
_parts_list: List[BedrockContentBlock] = []
for tool in tool_calls:
if "function" in tool:
id = tool["id"]
tool_id = tool["id"]
name = tool["function"].get("name", "")
arguments = tool["function"].get("arguments", "")
arguments_dict = json.loads(arguments) if arguments else {}
# Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object)
# When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns ""
if not isinstance(arguments_dict, dict):
arguments_dict = {}
if not arguments or not arguments.strip():
arguments_dict = {}
else:
arguments_dict = json.loads(arguments)
try:
arguments_dict = json.loads(arguments)
# Ensure arguments_dict is always a dict
# (Bedrock requires toolUse.input to be an object).
# Some providers return arguments: '""' which
# json.loads decodes to a bare string.
if not isinstance(arguments_dict, dict):
arguments_dict = {}
except json.JSONDecodeError:
# The model may return multiple JSON objects
# concatenated in a single arguments string, e.g.
# '{"cmd":"a"}{"cmd":"b"}{"cmd":"c"}'
# Split them and emit one toolUse block per object.
# Fixes: https://github.com/BerriAI/litellm/issues/20543
parsed_objects = split_concatenated_json_objects(
arguments
)
if parsed_objects:
# First object keeps the original tool id.
for obj_idx, obj in enumerate(parsed_objects):
block_id = (
tool_id
if obj_idx == 0
else f"{tool_id}_{obj_idx}"
)
bedrock_tool = BedrockToolUseBlock(
input=obj, name=name, toolUseId=block_id
)
_parts_list.append(
BedrockContentBlock(toolUse=bedrock_tool)
)
# cache_control applies to the whole original
# tool call; attach after the last split block.
if tool.get("cache_control", None) is not None:
_parts_list.append(
BedrockContentBlock(
cachePoint=CachePointBlock(
type="default"
)
)
)
continue
# Fallback: no objects extracted — use empty dict.
arguments_dict = {}
bedrock_tool = BedrockToolUseBlock(
input=arguments_dict, name=name, toolUseId=id
input=arguments_dict, name=name, toolUseId=tool_id
)
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
_parts_list.append(bedrock_content_block)

58
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -1,6 +1,6 @@
import base64
import time
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from litellm.types.llms.openai import (
ChatCompletionAssistantContentValue,
@ -326,10 +326,22 @@ class ChunkProcessor:
thinking_blocks: List[
Union["ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
] = []
combined_thinking_text: Optional[str] = None
data: Optional[str] = None
signature: Optional[str] = None
type: Literal["thinking", "redacted_thinking"] = "thinking"
current_thinking_text_parts: List[str] = []
current_signature: Optional[str] = None
def _flush_thinking_block() -> None:
nonlocal current_thinking_text_parts, current_signature
if len(current_thinking_text_parts) > 0 and current_signature:
thinking_blocks.append(
ChatCompletionThinkingBlock(
type="thinking",
thinking="".join(current_thinking_text_parts),
signature=current_signature,
)
)
current_thinking_text_parts = []
current_signature = None
for chunk in chunks:
choices = chunk["choices"]
for choice in choices:
@ -339,33 +351,25 @@ class ChunkProcessor:
for thinking_block in thinking:
thinking_type = thinking_block.get("type", None)
if thinking_type and thinking_type == "redacted_thinking":
type = "redacted_thinking"
data = thinking_block.get("data", None)
_flush_thinking_block()
redacted_data = thinking_block.get("data", None)
if redacted_data:
thinking_blocks.append(
ChatCompletionRedactedThinkingBlock(
type="redacted_thinking",
data=redacted_data,
)
)
else:
type = "thinking"
thinking_text = thinking_block.get("thinking", None)
if thinking_text:
if combined_thinking_text is None:
combined_thinking_text = ""
combined_thinking_text += thinking_text
current_thinking_text_parts.append(thinking_text)
signature = thinking_block.get("signature", None)
if signature:
current_signature = signature
_flush_thinking_block()
if combined_thinking_text and type == "thinking" and signature:
thinking_blocks.append(
ChatCompletionThinkingBlock(
type=type,
thinking=combined_thinking_text,
signature=signature,
)
)
elif data and type == "redacted_thinking":
thinking_blocks.append(
ChatCompletionRedactedThinkingBlock(
type=type,
data=data,
)
)
_flush_thinking_block()
if len(thinking_blocks) > 0:
return thinking_blocks

12
litellm/llms/anthropic/chat/handler.py
View file

@ -75,6 +75,7 @@ async def make_call(
logging_obj,
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@ -103,6 +104,7 @@ async def make_call(
streaming_response=response.aiter_lines(),
sync_stream=False,
json_mode=json_mode,
speed=speed,
)
# LOGGING
@ -126,6 +128,7 @@ def make_sync_call(
logging_obj,
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@ -159,7 +162,7 @@ def make_sync_call(
)
completion_stream = ModelResponseIterator(
streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode
streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode, speed=speed
)
# LOGGING
@ -213,6 +216,7 @@ class AnthropicChatCompletion(BaseLLM):
logging_obj=logging_obj,
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
)
streamwrapper = CustomStreamWrapper(
completion_stream=completion_stream,
@ -427,6 +431,7 @@ class AnthropicChatCompletion(BaseLLM):
logging_obj=logging_obj,
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
)
return CustomStreamWrapper(
completion_stream=completion_stream,
@ -485,13 +490,14 @@ class AnthropicChatCompletion(BaseLLM):
class ModelResponseIterator:
def __init__(
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False, speed: Optional[str] = None
):
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
self.content_blocks: List[ContentBlockDelta] = []
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
@ -541,7 +547,7 @@ class ModelResponseIterator:
def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None
usage_object=cast(dict, anthropic_usage_chunk), reasoning_content=None, speed=self.speed
)
def _content_block_delta_helper(self, chunk: dict) -> Tuple[

14
litellm/llms/anthropic/chat/transformation.py
View file

@ -190,6 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"response_format",
"user",
"web_search_options",
"speed",
]
if "claude-3-7-sonnet" in model or supports_reasoning(
@ -882,6 +883,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif param == "context_management" and isinstance(value, dict):
# Pass through Anthropic-specific context_management parameter
optional_params["context_management"] = value
elif param == "speed" and isinstance(value, str):
# Pass through Anthropic-specific speed parameter for fast mode
optional_params["speed"] = value
## handle thinking tokens
self.update_optional_params_with_thinking_tokens(
@ -1096,6 +1100,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value
)
if optional_params.get("speed") == "fast":
self._ensure_beta_header(
headers, ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value
)
return headers
def transform_request(
@ -1349,6 +1357,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
usage_object: dict,
reasoning_content: Optional[str],
completion_response: Optional[dict] = None,
speed: Optional[str] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
@ -1447,6 +1456,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else None
),
inference_geo=inference_geo,
speed=speed,
)
return usage
@ -1457,6 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response: ModelResponse,
json_mode: Optional[bool] = None,
prefix_prompt: Optional[str] = None,
speed: Optional[str] = None,
):
_hidden_params: Dict = {}
_hidden_params["additional_headers"] = process_anthropic_headers(
@ -1553,6 +1564,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
usage_object=completion_response["usage"],
reasoning_content=reasoning_content,
completion_response=completion_response,
speed=speed,
)
setattr(model_response, "usage", usage) # type: ignore
@ -1621,6 +1633,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
prefix_prompt = self.get_prefix_prompt(messages=messages)
speed = optional_params.get("speed")
model_response = self.transform_parsed_response(
completion_response=completion_response,
@ -1628,6 +1641,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response=model_response,
json_mode=json_mode,
prefix_prompt=prefix_prompt,
speed=speed,
)
return model_response

15
litellm/llms/anthropic/cost_calculation.py
View file

@ -22,13 +22,18 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
# If usage has inference_geo, prepend it as prefix to model name
model_with_prefix = model
# First, prepend inference_geo if present
if hasattr(usage, "inference_geo") and usage.inference_geo and usage.inference_geo.lower() not in ["global", "not_available"]:
model_with_geo_prefix = f"{usage.inference_geo}/{model}"
else:
model_with_geo_prefix = model
model_with_prefix = f"{usage.inference_geo}/{model_with_prefix}"
# Then, prepend speed if it's "fast"
if hasattr(usage, "speed") and usage.speed == "fast":
model_with_prefix = f"fast/{model_with_prefix}"
prompt_cost, completion_cost = generic_cost_per_token(
model=model_with_geo_prefix, usage=usage, custom_llm_provider="anthropic"
model=model_with_prefix, usage=usage, custom_llm_provider="anthropic"
)
return prompt_cost, completion_cost

57
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -30,6 +30,58 @@ ANTHROPIC_ADAPTER = AnthropicAdapter()
class LiteLLMMessagesToCompletionTransformationHandler:
@staticmethod
def _route_openai_thinking_to_responses_api_if_needed(
completion_kwargs: Dict[str, Any],
*,
thinking: Optional[Dict[str, Any]],
) -> None:
"""
When users call `litellm.anthropic.messages.*` with a non-Anthropic model and
`thinking={"type": "enabled", ...}`, LiteLLM converts this into OpenAI
`reasoning_effort`.
For OpenAI models, Chat Completions typically does not return reasoning text
(only token accounting). To return a thinking-like content block in the
Anthropic response format, we route the request through OpenAI's Responses API
and request a reasoning summary.
"""
custom_llm_provider = completion_kwargs.get("custom_llm_provider")
if custom_llm_provider is None:
try:
_, inferred_provider, _, _ = litellm.utils.get_llm_provider(
model=cast(str, completion_kwargs.get("model"))
)
custom_llm_provider = inferred_provider
except Exception:
custom_llm_provider = None
if custom_llm_provider != "openai":
return
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
return
model = completion_kwargs.get("model")
if isinstance(model, str) and model and not model.startswith("responses/"):
# Prefix model with "responses/" to route to OpenAI Responses API
completion_kwargs["model"] = f"responses/{model}"
reasoning_effort = completion_kwargs.get("reasoning_effort")
if isinstance(reasoning_effort, str) and reasoning_effort:
completion_kwargs["reasoning_effort"] = {
"effort": reasoning_effort,
"summary": "detailed",
}
elif isinstance(reasoning_effort, dict):
if (
"summary" not in reasoning_effort
and "generate_summary" not in reasoning_effort
):
updated_reasoning_effort = dict(reasoning_effort)
updated_reasoning_effort["summary"] = "detailed"
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
@staticmethod
def _prepare_completion_kwargs(
*,
@ -123,6 +175,11 @@ class LiteLLMMessagesToCompletionTransformationHandler:
):
completion_kwargs[key] = value
LiteLLMMessagesToCompletionTransformationHandler._route_openai_thinking_to_responses_api_if_needed(
completion_kwargs,
thinking=thinking,
)
return completion_kwargs, tool_name_mapping
@staticmethod

10
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -46,6 +46,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"thinking",
"context_management",
"output_format",
"inference_geo",
"speed",
"output_config",
# TODO: Add Anthropic `metadata` support
# "metadata",
]
@ -183,10 +186,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
- context_management: adds 'context-management-2025-06-27'
- tool_search: adds provider-specific tool search header
- output_format: adds 'structured-outputs-2025-11-13'
- speed: adds 'fast-mode-2026-02-01'
Args:
headers: Request headers dict
optional_params: Optional parameters including tools, context_management, output_format
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
"""
beta_values: set = set()
@ -223,6 +227,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("output_format") is not None:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.STRUCTURED_OUTPUT_2025_09_25.value)
# Check for fast mode
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
# Check for tool search tools
tools = optional_params.get("tools")
if tools:

20
litellm/llms/azure/azure.py
View file

@ -1060,6 +1060,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers: dict,
client=None,
timeout=None,
model: Optional[str] = None,
) -> ImageResponse:
response: Optional[dict] = None
@ -1071,8 +1072,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
if api_base.endswith("/"):
api_base = api_base.rstrip("/")
api_version: str = azure_client_params.get("api_version", "")
# Use the deployment name (model) for URL construction, not the base_model from data
img_gen_api_base = self.create_azure_base_url(
azure_client_params=azure_client_params, model=data.get("model", "")
azure_client_params=azure_client_params, model=model or data.get("model", "")
)
## LOGGING
@ -1159,21 +1161,20 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
model = model
else:
model = None
## BASE MODEL CHECK
if (
model_response is not None
and optional_params.get("base_model", None) is not None
and litellm_params is not None
and litellm_params.get("base_model", None) is not None
):
model_response._hidden_params["model"] = optional_params.pop(
"base_model"
)
model_response._hidden_params["model"] = litellm_params.get("base_model", None)
# Azure image generation API doesn't support extra_body parameter
extra_body = optional_params.pop("extra_body", {})
flattened_params = {**optional_params, **extra_body}
data = {"model": model, "prompt": prompt, **flattened_params}
base_model = litellm_params.get("base_model", None) if litellm_params else None
data = {"model": base_model or model, "prompt": prompt, **flattened_params}
max_retries = data.pop("max_retries", 2)
if not isinstance(max_retries, int):
raise AzureOpenAIError(
@ -1196,10 +1197,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
is_async=False,
)
if aimg_generation is True:
return self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout, headers=headers) # type: ignore
return self.aimage_generation(data=data, input=input, logging_obj=logging_obj, model_response=model_response, api_key=api_key, client=client, azure_client_params=azure_client_params, timeout=timeout, headers=headers, model=model) # type: ignore
# Use the deployment name (model) for URL construction, not the base_model from data
img_gen_api_base = self.create_azure_base_url(
azure_client_params=azure_client_params, model=data.get("model", "")
azure_client_params=azure_client_params, model=model
)
## LOGGING

9
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -3,6 +3,9 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
"""
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from litellm.anthropic_beta_headers_manager import (
update_headers_with_filtered_beta,
)
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
@ -68,6 +71,12 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
optional_params=optional_params,
)
# Filter out unsupported beta headers for Azure AI
headers = update_headers_with_filtered_beta(
headers=headers,
provider="azure_ai",
)
return headers, api_base
def get_complete_url(

29
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, List, Optional
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
@ -133,27 +134,15 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
beta_set.add("tool-search-tool-2025-10-19")
# Filter out beta headers that Bedrock Invoke doesn't support
# AWS Bedrock only supports a specific whitelist of beta flags
# Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
BEDROCK_SUPPORTED_BETAS = {
"computer-use-2024-10-22", # Legacy computer use
"computer-use-2025-01-24", # Current computer use (Claude 3.7 Sonnet)
"token-efficient-tools-2025-02-19", # Tool use (Claude 3.7+ and Claude 4+)
"interleaved-thinking-2025-05-14", # Interleaved thinking (Claude 4+)
"output-128k-2025-02-19", # 128K output tokens (Claude 3.7 Sonnet)
"dev-full-thinking-2025-05-14", # Developer mode for raw thinking (Claude 4+)
"context-1m-2025-08-07", # 1 million tokens (Claude Sonnet 4)
"context-management-2025-06-27", # Context management (Claude Sonnet/Haiku 4.5)
"effort-2025-11-24", # Effort parameter (Claude Opus 4.5)
"tool-search-tool-2025-10-19", # Tool search (Claude Opus 4.5)
"tool-examples-2025-10-29", # Tool use examples (Claude Opus 4.5)
}
# Only keep beta headers that Bedrock supports
beta_set = {beta for beta in beta_set if beta in BEDROCK_SUPPORTED_BETAS}
# Uses centralized configuration from anthropic_beta_headers_config.json
beta_list = list(beta_set)
filtered_beta_list = filter_and_transform_beta_headers(
beta_headers=beta_list,
provider="bedrock",
)
if beta_set:
_anthropic_request["anthropic_beta"] = list(beta_set)
if filtered_beta_list:
_anthropic_request["anthropic_beta"] = filtered_beta_list
return _anthropic_request

23
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -1,6 +1,7 @@
import asyncio
import contextlib
import os
import ssl
import typing
import urllib.request
from typing import Callable, Dict, Optional, Union
@ -139,8 +140,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
Credit to: https://github.com/karpetrosyan/httpx-aiohttp for this implementation
"""
def __init__(self, client: Union[ClientSession, Callable[[], ClientSession]]):
def __init__(
self,
client: Union[ClientSession, Callable[[], ClientSession]],
ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None,
):
self.client = client
self._ssl_verify = ssl_verify # Store for per-request SSL override
super().__init__(client=client)
# Store the client factory for recreating sessions when needed
if callable(client):
@ -214,6 +220,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
timeout: dict,
proxy: Optional[str],
sni_hostname: Optional[str],
ssl_verify: Optional[Union[bool, ssl.SSLContext]] = None,
) -> ClientResponse:
"""
Helper function to make an aiohttp request with the given parameters.
@ -224,6 +231,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
timeout: Timeout settings dict with 'connect', 'read', 'pool' keys
proxy: Optional proxy URL
sni_hostname: Optional SNI hostname for SSL
ssl_verify: Optional SSL verification setting (False to disable, SSLContext for custom)
Returns:
ClientResponse from aiohttp
@ -237,6 +245,13 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
data = request.stream # type: ignore
request.headers.pop("transfer-encoding", None) # handled by aiohttp
# Only pass ssl kwarg when explicitly configured, to avoid
# overriding the session/connector defaults with None (which is
# not a valid value for aiohttp's ssl parameter).
ssl_kwargs: Dict[str, Union[bool, ssl.SSLContext]] = {}
if ssl_verify is not None:
ssl_kwargs["ssl"] = ssl_verify
response = await client_session.request(
method=request.method,
url=YarlURL(str(request.url), encoded=True),
@ -251,6 +266,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
),
proxy=proxy,
server_hostname=sni_hostname,
**ssl_kwargs,
).__aenter__()
return response
@ -268,6 +284,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
# Resolve proxy settings from environment variables
proxy = await self._get_proxy_settings(request)
# Use stored SSL configuration for per-request override
ssl_config = self._ssl_verify
try:
with map_aiohttp_exceptions():
response = await self._make_aiohttp_request(
@ -276,6 +295,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
timeout=timeout,
proxy=proxy,
sni_hostname=sni_hostname,
ssl_verify=ssl_config,
)
except RuntimeError as e:
# Handle the case where session was closed between our check and actual use
@ -296,6 +316,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
timeout=timeout,
proxy=proxy,
sni_hostname=sni_hostname,
ssl_verify=ssl_config,
)
else:
# Re-raise if it's a different RuntimeError

16
litellm/llms/custom_httpx/http_handler.py
View file

@ -846,6 +846,16 @@ class AsyncHTTPHandler:
if str_to_bool(os.getenv("AIOHTTP_TRUST_ENV", "False")) is True:
trust_env = True
#########################################################
# Determine SSL config to pass to transport for per-request override
# This ensures ssl_verify works even with shared sessions
#########################################################
ssl_for_transport: Optional[Union[bool, ssl.SSLContext]] = None
if ssl_context is not None:
ssl_for_transport = ssl_context
elif ssl_verify is False:
ssl_for_transport = False
verbose_logger.debug("Creating AiohttpTransport...")
# Use shared session if provided and valid
@ -853,7 +863,10 @@ class AsyncHTTPHandler:
verbose_logger.debug(
f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})"
)
return LiteLLMAiohttpTransport(client=shared_session)
return LiteLLMAiohttpTransport(
client=shared_session,
ssl_verify=ssl_for_transport,
)
# Create new session only if none provided or existing one is invalid
verbose_logger.debug(
@ -877,6 +890,7 @@ class AsyncHTTPHandler:
connector=TCPConnector(**transport_connector_kwargs),
trust_env=trust_env,
),
ssl_verify=ssl_for_transport,
)
@staticmethod

126
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -302,7 +302,7 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
signed_json_body=signed_json_body,
)
return provider_config.transform_response(
initial_response = provider_config.transform_response(
model=model,
raw_response=response,
model_response=model_response,
@ -316,6 +316,20 @@ class BaseLLMHTTPHandler:
json_mode=json_mode,
)
# Call agentic chat completion hooks
final_response = await self._call_agentic_chat_completion_hooks(
response=initial_response,
model=model,
messages=messages,
optional_params=optional_params,
logging_obj=logging_obj,
stream=False,
custom_llm_provider=custom_llm_provider,
kwargs=litellm_params,
)
return final_response if final_response is not None else initial_response
def completion(
self,
model: str,
@ -412,6 +426,11 @@ class BaseLLMHTTPHandler:
},
)
# Check if stream was converted for WebSearch interception
# This is set by the async_pre_request_hook in WebSearchInterceptionLogger
if litellm_params.get("_websearch_interception_converted_stream", False):
logging_obj.model_call_details["websearch_interception_converted_stream"] = True
if acompletion is True:
if stream is True:
data = self._add_stream_param_to_request_body(
@ -4361,10 +4380,10 @@ class BaseLLMHTTPHandler:
kwargs: Dict,
) -> Optional[Any]:
"""
Call agentic completion hooks for all custom loggers.
Call agentic completion hooks for all custom loggers (Anthropic Messages API).
1. Call async_should_run_agentic_completion to check if agentic loop is needed
2. If yes, call async_run_agentic_completion to execute the loop
1. Call async_should_run_agentic_loop to check if agentic loop is needed
2. If yes, call async_run_agentic_loop to execute the loop
Returns the response from agentic loop, or None if no hook runs.
"""
@ -4453,6 +4472,105 @@ class BaseLLMHTTPHandler:
return None
async def _call_agentic_chat_completion_hooks(
self,
response: Any,
model: str,
messages: List[Dict],
optional_params: Dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
) -> Optional[Any]:
"""
Call agentic chat completion hooks for all custom loggers (Chat Completions API).
1. Call async_should_run_chat_completion_agentic_loop to check if agentic loop is needed
2. If yes, call async_run_chat_completion_agentic_loop to execute the loop
Returns the response from agentic loop, or None if no hook runs.
"""
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
callbacks = litellm.callbacks + (
logging_obj.dynamic_success_callbacks or []
)
tools = optional_params.get("tools", [])
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
# Check if callback has the chat completion agentic loop method
if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
continue
# First: Check if agentic loop should run
should_run, tool_calls = (
await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
)
)
if should_run:
# Second: Execute agentic loop
# Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
kwargs_with_provider = kwargs.copy() if kwargs else {}
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
agentic_response = await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
# First hook that runs agentic loop wins
return agentic_response
except Exception as e:
verbose_logger.exception(
f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {str(e)}"
)
# Check if we need to convert response to fake stream for chat completions
# This happens when:
# 1. Stream was originally True but converted to False for WebSearch interception
# 2. No agentic loop ran (LLM didn't use the tool)
# 3. We have a non-streaming response that needs to be converted to streaming
websearch_converted_stream = (
logging_obj.model_call_details.get("websearch_interception_converted_stream", False)
if logging_obj is not None
else False
)
if websearch_converted_stream:
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_model_iterator import (
convert_model_response_to_streaming,
)
verbose_logger.debug(
"WebSearchInterception: No tool call made, converting non-streaming chat completion to fake stream"
)
# Convert the non-streaming ModelResponse to a fake stream
if hasattr(response, "choices"):
# Use the existing converter for ModelResponse
fake_stream = convert_model_response_to_streaming(response)
return fake_stream
return None
def _handle_error(
self,
e: Exception,

33
litellm/llms/databricks/chat/transformation.py
View file

@ -60,6 +60,38 @@ from ...anthropic.chat.transformation import AnthropicConfig
from ...openai_like.chat.transformation import OpenAILikeChatConfig
from ..common_utils import DatabricksBase, DatabricksException
def _sanitize_empty_content(message_dict: dict[str, Any]) -> None:
"""
Remove or filter content so empty text blocks are not sent.
Databricks Model Serving uses Anthropic Messages API spec and rejects empty text blocks.
"""
content = message_dict.get("content")
if content is None:
message_dict.pop("content", None)
return
if isinstance(content, str):
if not content.strip():
message_dict.pop("content")
return
if isinstance(content, list):
if not content:
message_dict.pop("content")
return
filtered = [
block
for block in content
if not (
isinstance(block, dict)
and block.get("type") == "text"
and not (block.get("text") or "").strip()
)
]
if not filtered:
message_dict.pop("content")
else:
message_dict["content"] = filtered
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -350,6 +382,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
# Move message-level cache_control into a content block when content is a string.
if "cache_control" in _message and isinstance(_message.get("content"), str):
_message = self._move_cache_control_into_string_content_block(_message)
_sanitize_empty_content(cast(dict[str, Any], _message))
new_messages.append(_message)
if is_async:

37
litellm/llms/oci/chat/transformation.py
View file

@ -218,6 +218,7 @@ class OCIChatConfig(BaseConfig):
"parallel_tool_calls": False,
"audio": False,
"web_search_options": False,
"response_format": "responseFormat",
}
# Cohere and Gemini use the same parameter mapping as GENERIC
@ -269,6 +270,9 @@ class OCIChatConfig(BaseConfig):
adapted_params[alias] = value
if alias == "responseFormat":
adapted_params["response_format"] = value
return adapted_params
def _sign_with_oci_signer(
@ -673,6 +677,36 @@ class OCIChatConfig(BaseConfig):
selected_params["tools"] = adapt_tool_definition_to_oci_standard( # type: ignore[assignment]
selected_params["tools"], vendor # type: ignore[arg-type]
)
# Transform response_format type to OCI uppercase format
if "responseFormat" in selected_params:
rf = selected_params["responseFormat"]
if isinstance(rf, dict) and "type" in rf:
rf_payload = dict(rf)
selected_params["responseFormat"] = rf_payload
response_type = rf_payload["type"]
schema_payload: Optional[Any] = None
if "json_schema" in rf_payload:
raw_schema_payload = rf_payload.pop("json_schema")
if isinstance(raw_schema_payload, dict):
schema_payload = dict(raw_schema_payload)
else:
schema_payload = raw_schema_payload
if schema_payload is not None:
rf_payload["jsonSchema"] = schema_payload
if vendor == OCIVendors.COHERE:
# Cohere expects lower-case type values
rf_payload["type"] = response_type
else:
format_type = response_type.upper()
if format_type == "JSON":
format_type = "JSON_OBJECT"
rf_payload["type"] = format_type
return selected_params
def adapt_messages_to_cohere_standard(self, messages: List[AllMessageValues]) -> List[CohereMessage]:
@ -806,11 +840,12 @@ class OCIChatConfig(BaseConfig):
# Create Cohere-specific chat request
optional_cohere_params = self._get_optional_params(OCIVendors.COHERE, optional_params)
chat_request = CohereChatRequest(
apiFormat="COHERE",
message=self._extract_text_content(user_messages[-1]["content"]),
chatHistory=self.adapt_messages_to_cohere_standard(messages),
**self._get_optional_params(OCIVendors.COHERE, optional_params)
**optional_cohere_params
)
data = OCICompletionPayload(

11
litellm/llms/ollama/chat/transformation.py
View file

@ -502,13 +502,12 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
reasoning_content: Optional[str] = None
content: Optional[str] = None
if chunk["message"].get("thinking") is not None:
if self.started_reasoning_content is False:
reasoning_content = chunk["message"].get("thinking")
self.started_reasoning_content = True
elif self.finished_reasoning_content is False:
reasoning_content = chunk["message"].get("thinking")
self.finished_reasoning_content = True
reasoning_content = chunk["message"].get("thinking")
self.started_reasoning_content = True
elif chunk["message"].get("content") is not None:
if self.started_reasoning_content and not self.finished_reasoning_content:
self.finished_reasoning_content = True
message_content = chunk["message"].get("content")
if "<think>" in message_content:
message_content = message_content.replace("<think>", "")

97
litellm/llms/openai/openai.py
View file

@ -501,6 +501,88 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
else:
raise e
async def _call_agentic_completion_hooks_openai(
self,
response: Any,
model: str,
messages: List[Dict],
optional_params: Dict,
logging_obj: LiteLLMLoggingObj,
stream: bool,
litellm_params: Dict,
) -> Optional[Any]:
"""
Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API).
1. Call async_should_run_chat_completion_agentic_loop to check if agentic loop is needed
2. If yes, call async_run_chat_completion_agentic_loop to execute the loop
Returns the response from agentic loop, or None if no hook runs.
"""
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
callbacks = litellm.callbacks + (
logging_obj.dynamic_success_callbacks or []
)
# Avoid logging full callback objects to prevent leaking sensitive data
verbose_logger.debug(
"LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks)
)
tools = optional_params.get("tools", [])
# Avoid logging full tools payloads; they may contain sensitive parameters
verbose_logger.debug(
"LiteLLM.AgenticHooks: tools_count=%s", len(tools) if isinstance(tools, list) else 1 if tools else 0
)
# Get custom_llm_provider from litellm_params
custom_llm_provider = litellm_params.get("custom_llm_provider", "openai")
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
# Check if the callback has the chat completion agentic loop methods
if not hasattr(callback, 'async_should_run_chat_completion_agentic_loop'):
continue
# First: Check if agentic loop should run (using chat completion method)
should_run, tool_calls = (
await callback.async_should_run_chat_completion_agentic_loop(
response=response,
model=model,
messages=messages,
tools=tools,
stream=stream,
custom_llm_provider=custom_llm_provider,
kwargs=litellm_params,
)
)
if should_run:
# Second: Execute agentic loop
kwargs_with_provider = litellm_params.copy() if litellm_params else {}
kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
# For OpenAI Chat Completions, use the chat completion agentic loop method
agentic_response = await callback.async_run_chat_completion_agentic_loop(
tools=tool_calls,
model=model,
messages=messages,
response=response,
optional_params=optional_params,
logging_obj=logging_obj,
stream=stream,
kwargs=kwargs_with_provider,
)
# First hook that runs agentic loop wins
return agentic_response
except Exception as e:
verbose_logger.exception(
f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {str(e)}"
)
return None
def mock_streaming(
self,
response: ModelResponse,
@ -844,7 +926,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
logging_obj=logging_obj,
)
stringified_response = response.model_dump()
logging_obj.post_call(
input=data["messages"],
api_key=api_key,
@ -859,6 +940,20 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
_response_headers=headers,
)
# Call agentic completion hooks (e.g., for websearch_interception)
agentic_response = await self._call_agentic_completion_hooks_openai(
response=final_response_obj,
model=model,
messages=messages,
optional_params=optional_params,
logging_obj=logging_obj,
stream=False,
litellm_params=litellm_params,
)
if agentic_response is not None:
final_response_obj = agentic_response
if fake_stream is True:
return self.mock_streaming(
response=cast(ModelResponse, final_response_obj),

45
litellm/llms/openai/videos/transformation.py
View file

@ -269,26 +269,27 @@ class OpenAIVideoConfig(BaseVideoConfig):
) -> Tuple[str, Dict]:
"""
Transform the video list request for OpenAI API.
OpenAI API expects the following request:
- GET /v1/videos
"""
# Use the api_base directly for video list
url = api_base
# Prepare query parameters
params = {}
if after is not None:
params["after"] = after
# Decode the wrapped video ID back to the original provider ID
params["after"] = extract_original_video_id(after)
if limit is not None:
params["limit"] = str(limit)
if order is not None:
params["order"] = order
# Add any extra query parameters
if extra_query:
params.update(extra_query)
return url, params
def transform_video_list_response(
@ -296,18 +297,40 @@ class OpenAIVideoConfig(BaseVideoConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
custom_llm_provider: Optional[str] = None,
) -> Dict[str,str]:
) -> Dict[str, str]:
response_data = raw_response.json()
if custom_llm_provider and "data" in response_data:
for video_obj in response_data.get("data", []):
if isinstance(video_obj, dict) and "id" in video_obj:
video_obj["id"] = encode_video_id_with_provider(
video_obj["id"],
custom_llm_provider,
video_obj.get("model")
video_obj["id"],
custom_llm_provider,
video_obj.get("model"),
)
# Encode pagination cursor IDs so they remain consistent
# with the wrapped data[].id format
data_list = response_data.get("data", [])
if response_data.get("first_id"):
first_model = None
if data_list and isinstance(data_list[0], dict):
first_model = data_list[0].get("model")
response_data["first_id"] = encode_video_id_with_provider(
response_data["first_id"],
custom_llm_provider,
first_model,
)
if response_data.get("last_id"):
last_model = None
if data_list and isinstance(data_list[-1], dict):
last_model = data_list[-1].get("model")
response_data["last_id"] = encode_video_id_with_provider(
response_data["last_id"],
custom_llm_provider,
last_model,
)
return response_data
def transform_video_delete_request(

7
litellm/llms/perplexity/responses/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
Perplexity Agentic Research API (Responses API) module
"""
from .transformation import PerplexityResponsesConfig
__all__ = ["PerplexityResponsesConfig"]

409
litellm/llms/perplexity/responses/transformation.py Normal file
View file

@ -0,0 +1,409 @@
"""
Transformation logic for Perplexity Agentic Research API (Responses API)
This module handles the translation between OpenAI's Responses API format
and Perplexity's Responses API format, which supports:
- Third-party model access (OpenAI, Anthropic, Google, xAI, etc.)
- Presets for optimized configurations
- Web search and URL fetching tools
- Reasoning effort control
- Instructions parameter for system-level guidance
"""
from typing import Any, Dict, List, Optional, Union
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"""
Configuration for Perplexity Agentic Research API (Responses API)
Reference: https://docs.perplexity.ai/agentic-research/quickstart
"""
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.PERPLEXITY
def get_supported_openai_params(self, model: str) -> list:
"""
Perplexity Responses API supports a different set of parameters
Ref: https://docs.perplexity.ai/api-reference/responses-post
"""
return [
"max_output_tokens",
"stream",
"temperature",
"top_p",
"tools",
"reasoning",
"preset",
"instructions",
"models", # Model fallback support
]
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
"""Validate environment and set up headers"""
# Get API key from environment
api_key = (
get_secret_str("PERPLEXITYAI_API_KEY")
or get_secret_str("PERPLEXITY_API_KEY")
)
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""Get the complete URL for the Perplexity Responses API"""
if api_base is None:
api_base = get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai"
# Ensure api_base doesn't end with a slash
api_base = api_base.rstrip("/")
# Add the responses endpoint
return f"{api_base}/v1/responses"
def map_openai_params(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""
Map OpenAI Responses API parameters to Perplexity format
Key differences:
- Supports 'preset' parameter for predefined configurations
- Supports 'instructions' parameter for system-level guidance
- Tools are specified differently (web_search, fetch_url)
"""
mapped_params: Dict[str, Any] = {}
# Map standard parameters
if response_api_optional_params.get("max_output_tokens"):
mapped_params["max_output_tokens"] = response_api_optional_params["max_output_tokens"]
if response_api_optional_params.get("temperature"):
mapped_params["temperature"] = response_api_optional_params["temperature"]
if response_api_optional_params.get("top_p"):
mapped_params["top_p"] = response_api_optional_params["top_p"]
if response_api_optional_params.get("stream"):
mapped_params["stream"] = response_api_optional_params["stream"]
if response_api_optional_params.get("stream_options"):
mapped_params["stream_options"] = response_api_optional_params["stream_options"]
# Map Perplexity-specific parameters (using .get() with Any dict access)
preset = response_api_optional_params.get("preset") # type: ignore
if preset:
mapped_params["preset"] = preset
instructions = response_api_optional_params.get("instructions") # type: ignore
if instructions:
mapped_params["instructions"] = instructions
if response_api_optional_params.get("reasoning"):
mapped_params["reasoning"] = response_api_optional_params["reasoning"]
tools = response_api_optional_params.get("tools")
if tools:
# Convert tools to list of dicts for transformation
tools_list = [dict(tool) if hasattr(tool, '__dict__') else tool for tool in tools] # type: ignore
mapped_params["tools"] = self._transform_tools(tools_list) # type: ignore
return mapped_params
def _transform_tools(self, tools: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Transform tools to Perplexity format
Perplexity supports:
- web_search: Performs web searches
- fetch_url: Fetches content from URLs
"""
perplexity_tools = []
for tool in tools:
if isinstance(tool, dict):
tool_type = tool.get("type")
# Direct Perplexity tool format
if tool_type in ["web_search", "fetch_url"]:
perplexity_tools.append(tool)
# OpenAI function format - try to map to Perplexity tools
elif tool_type == "function":
function = tool.get("function", {})
function_name = function.get("name", "")
if function_name == "web_search" or "search" in function_name.lower():
perplexity_tools.append({"type": "web_search"})
elif function_name == "fetch_url" or "fetch" in function_name.lower():
perplexity_tools.append({"type": "fetch_url"})
return perplexity_tools
def transform_responses_api_request(
self,
model: str,
input: Union[str, ResponseInputParam],
response_api_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
"""
Transform request to Perplexity Responses API format
"""
# Check if the model is a preset (format: preset/preset-name)
if model.startswith("preset/"):
preset_name = model.replace("preset/", "")
data = {
"preset": preset_name,
"input": self._format_input(input),
}
# Check if preset is explicitly provided in params
elif response_api_optional_request_params.get("preset"):
data = {
"preset": response_api_optional_request_params.pop("preset"),
"input": self._format_input(input),
}
else:
# Full request format for third-party models
data = {
"model": model,
"input": self._format_input(input),
}
# Add all optional parameters
for key, value in response_api_optional_request_params.items():
data[key] = value
return data
def _format_input(self, input: Union[str, ResponseInputParam]) -> Union[str, List[Dict[str, Any]]]:
"""
Format input for Perplexity Responses API
The API accepts either:
- A simple string for single-turn queries
- An array of message objects for multi-turn conversations
"""
if isinstance(input, str):
return input
# Handle ResponseInputParam format
if isinstance(input, list):
formatted_messages = []
for item in input:
if isinstance(item, dict):
formatted_message = {
"type": "message",
"role": item.get("role"),
"content": item.get("content", ""),
}
formatted_messages.append(formatted_message)
return formatted_messages
return str(input)
def transform_response_api_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
"""
Transform Perplexity Responses API response to OpenAI Responses API format
"""
try:
raw_response_json = raw_response.json()
except Exception as e:
raise BaseLLMException(
status_code=raw_response.status_code,
message=f"Failed to parse response: {str(e)}",
)
# Check for error status
status = raw_response_json.get("status")
if status == "failed":
error = raw_response_json.get("error", {})
error_message = error.get("message", "Unknown error")
raise BaseLLMException(
status_code=raw_response.status_code,
message=error_message,
)
# Transform usage to handle Perplexity's cost structure
usage_data = raw_response_json.get("usage", {})
transformed_usage_dict = self._transform_usage(usage_data)
# Convert usage dict to ResponseAPIUsage object
usage_obj = ResponseAPIUsage(**transformed_usage_dict) if transformed_usage_dict else None
# Map Perplexity response to OpenAI Responses API format
response = ResponsesAPIResponse(
id=raw_response_json.get("id", ""),
object="response",
created_at=raw_response_json.get("created_at", 0),
status=raw_response_json.get("status", "completed"),
model=raw_response_json.get("model", model),
output=raw_response_json.get("output", []),
usage=usage_obj,
)
return response
def _transform_usage(self, usage_data: Dict[str, Any]) -> Dict[str, Any]:
"""
Transform Perplexity usage data to OpenAI format
Perplexity returns:
{
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost": {
"currency": "USD",
"input_cost": 0.0001,
"output_cost": 0.0002,
"total_cost": 0.0003
}
}
OpenAI expects:
{
"input_tokens": 100,
"output_tokens": 200,
"total_tokens": 300,
"cost": 0.0003
}
"""
transformed = {
"input_tokens": usage_data.get("input_tokens", 0),
"output_tokens": usage_data.get("output_tokens", 0),
"total_tokens": usage_data.get("total_tokens", 0),
}
# Transform cost from Perplexity format (dict) to OpenAI format (float)
cost_obj = usage_data.get("cost")
if isinstance(cost_obj, dict) and "total_cost" in cost_obj:
transformed["cost"] = cost_obj["total_cost"]
verbose_logger.debug(
"Transformed Perplexity cost object to float: %s -> %s",
cost_obj,
cost_obj["total_cost"]
)
elif cost_obj is not None:
# If cost is already a float/number, use it as-is
transformed["cost"] = cost_obj
# Add input_tokens_details if present
if "input_tokens_details" in usage_data:
transformed["input_tokens_details"] = usage_data["input_tokens_details"]
# Add output_tokens_details if present
if "output_tokens_details" in usage_data:
transformed["output_tokens_details"] = usage_data["output_tokens_details"]
return transformed
def transform_streaming_response(
self,
model: str,
parsed_chunk: dict,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIStreamingResponse:
"""
Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse
"""
# Get the event type from the chunk
verbose_logger.debug("Raw Perplexity Chunk=%s", parsed_chunk)
event_type = str(parsed_chunk.get("type"))
event_pydantic_model = PerplexityResponsesConfig.get_event_model_class(
event_type=event_type
)
# Transform Perplexity-specific fields to OpenAI format
parsed_chunk = self._transform_perplexity_chunk(parsed_chunk)
# Defensive: Handle error.code being null (similar to OpenAI implementation)
try:
error_obj = parsed_chunk.get("error")
if isinstance(error_obj, dict) and error_obj.get("code") is None:
# Preserve other fields, but ensure `code` is a non-null string
parsed_chunk = dict(parsed_chunk)
parsed_chunk["error"] = dict(error_obj)
parsed_chunk["error"]["code"] = "unknown_error"
except Exception:
# If anything unexpected happens here, fall back to attempting
# instantiation and let higher-level handlers manage errors.
verbose_logger.debug("Failed to coalesce error.code in parsed_chunk")
return event_pydantic_model(**parsed_chunk)
def _transform_perplexity_chunk(self, chunk: dict) -> dict:
"""
Transform Perplexity-specific fields in a streaming chunk to OpenAI format.
This handles:
- Converting Perplexity's cost object to a simple float
"""
# Make a copy to avoid modifying the original
chunk = dict(chunk)
# Transform usage.cost from Perplexity format to OpenAI format
# Perplexity: {"currency": "USD", "input_cost": 0.0001, "output_cost": 0.0002, "total_cost": 0.0003}
# OpenAI: 0.0003 (just the total_cost as a float)
try:
response_obj = chunk.get("response")
if isinstance(response_obj, dict):
usage_obj = response_obj.get("usage")
if isinstance(usage_obj, dict):
cost_obj = usage_obj.get("cost")
if isinstance(cost_obj, dict) and "total_cost" in cost_obj:
# Replace the cost object with just the total_cost value
chunk = dict(chunk)
chunk["response"] = dict(response_obj)
chunk["response"]["usage"] = dict(usage_obj)
chunk["response"]["usage"]["cost"] = cost_obj["total_cost"]
verbose_logger.debug(
"Transformed Perplexity cost object to float: %s -> %s",
cost_obj,
cost_obj["total_cost"]
)
except Exception as e:
# If transformation fails, log and continue with original chunk
verbose_logger.debug("Failed to transform Perplexity cost object: %s", e)
return chunk

37
litellm/llms/vertex_ai/common_utils.py
View file

@ -1,4 +1,5 @@
import re
from copy import deepcopy
from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, get_type_hints
@ -684,7 +685,7 @@ def convert_anyof_null_to_nullable(schema, depth=0):
if anyof is not None:
contains_null = False
for atype in anyof:
if atype == {"type": "null"}:
if isinstance(atype, dict) and atype.get("type") == "null":
# remove null type
anyof.remove(atype)
contains_null = True
@ -801,8 +802,38 @@ def _convert_schema_types(schema, depth=0):
if "type" in schema:
type_val = schema["type"]
if isinstance(type_val, list) and len(type_val) > 1:
# Convert ["string", "number"] -> {"anyOf": [{"type": "STRING"}, {"type": "NUMBER"}]}
schema["anyOf"] = [{"type": t} for t in type_val if isinstance(t, str)]
# Convert type arrays to anyOf format
# Fields that are specific to object/array types and should move into anyOf
type_specific_fields = {"properties", "required", "additionalProperties", "items", "minItems", "maxItems", "minProperties", "maxProperties"}
any_of: List[Dict[str, Any]] = []
for t in type_val:
if not isinstance(t, str):
continue
if t == "null":
# Keep null entry minimal so we can strip it later.
any_of.append({"type": "null"})
continue
# For object/array types, include type-specific fields
if t in ("object", "array"):
item_schema = {"type": t}
# Move type-specific fields into this anyOf item
for field in type_specific_fields:
if field in schema:
item_schema[field] = deepcopy(schema[field])
any_of.append(item_schema)
else:
# For primitive types, only include the type
any_of.append({"type": t})
# Remove type-specific fields from parent if we moved them into anyOf
has_object_or_array = any(t in ("object", "array") for t in type_val if isinstance(t, str))
if has_object_or_array:
for field in type_specific_fields:
schema.pop(field, None)
schema["anyOf"] = any_of
schema.pop("type")
elif isinstance(type_val, list) and len(type_val) == 1:
schema["type"] = type_val[0]

21
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -437,6 +437,27 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
else:
assistant_content.append(PartType(text=assistant_text)) # type: ignore
## HANDLE ASSISTANT IMAGES FIELD
# Process images field if present (for generated images from assistant)
assistant_images = assistant_msg.get("images")
if assistant_images is not None and isinstance(assistant_images, list):
for image_item in assistant_images:
if isinstance(image_item, dict):
image_url_obj = image_item.get("image_url")
if isinstance(image_url_obj, dict):
assistant_image_url = image_url_obj.get("url")
format = image_url_obj.get("format")
detail = image_url_obj.get("detail")
media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
if assistant_image_url:
_part = _process_gemini_media(
image_url=assistant_image_url,
format=format,
media_resolution_enum=media_resolution_enum,
model=model,
)
assistant_content.append(_part)
## HANDLE ASSISTANT FUNCTION CALL
if (
assistant_msg.get("tool_calls", []) is not None

23
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -68,6 +68,29 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
# Check for context management
context_management_param = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
# Check for web search tool
for tool in tools:
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):

46
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -56,34 +56,36 @@ class VertexAIAnthropicConfig(AnthropicConfig):
) -> None:
"""
Add context_management beta headers to the beta_set.
- If any edit has type "compact_20260112", add compact-2026-01-12 header
- For all other edits, add context-management-2025-06-27 header
Args:
beta_set: Set of beta headers to modify in-place
context_management: The context_management dict from optional_params
"""
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
edits = context_management.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_set.add(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
def transform_request(
self,
@ -102,10 +104,10 @@ class VertexAIAnthropicConfig(AnthropicConfig):
)
data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter
# VertexAI doesn't support output_format parameter, remove it if present
data.pop("output_format", None)
tools = optional_params.get("tools")
tool_search_used = self.is_tool_search_used(tools)
auto_betas = self.get_anthropic_beta_list(
@ -119,16 +121,30 @@ class VertexAIAnthropicConfig(AnthropicConfig):
beta_set = set(auto_betas)
if tool_search_used:
beta_set.add("tool-search-tool-2025-10-19") # Vertex requires this header for tool search
beta_set.add(
"tool-search-tool-2025-10-19"
) # Vertex requires this header for tool search
# Add context_management beta headers (compact and/or context-management)
context_management = optional_params.get("context_management")
if context_management:
self._add_context_management_beta_headers(beta_set, context_management)
extra_headers = optional_params.get("extra_headers") or {}
anthropic_beta_value = extra_headers.get("anthropic-beta", "")
if isinstance(anthropic_beta_value, str) and anthropic_beta_value:
for beta in anthropic_beta_value.split(","):
beta = beta.strip()
if beta:
beta_set.add(beta)
elif isinstance(anthropic_beta_value, list):
beta_set.update(anthropic_beta_value)
data.pop("extra_headers", None)
if beta_set:
data["anthropic_beta"] = list(beta_set)
return data
def map_openai_params(
@ -148,7 +164,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
original_model = model
if "response_format" in non_default_params:
model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach
# Call parent method with potentially modified model name
optional_params = super().map_openai_params(
non_default_params=non_default_params,
@ -156,10 +172,10 @@ class VertexAIAnthropicConfig(AnthropicConfig):
model=model,
drop_params=drop_params,
)
# Restore original model name for any other processing
model = original_model
return optional_params
def transform_response(

7
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -107,6 +107,11 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
vertex_project = self.get_vertex_ai_project(litellm_params)
vertex_location = self.get_vertex_ai_location(litellm_params)
# Map empty location/cluade models to a supported region for count-tokens endpoint
# https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
if not vertex_location or "claude" in model.lower():
vertex_location = "us-central1"
# Get access token and resolved project ID
access_token, project_id = await self._ensure_access_token_async(
credentials=vertex_credentials,
@ -118,7 +123,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
endpoint_url = self._build_count_tokens_endpoint(
model=model,
project_id=project_id,
vertex_location=vertex_location or "us-central1",
vertex_location=vertex_location,
api_base=litellm_params.get("api_base"),
)

553
litellm/model_prices_and_context_window_backup.json
View file

@ -993,66 +993,6 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"global.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
@ -1143,66 +1083,6 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"eu.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"apac.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.75e-05,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"apac.anthropic.claude-opus-4-6-v1": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
@ -5968,6 +5848,19 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"azure_ai/kimi-k2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/kimi-k2-5-now-in-microsoft-foundry/4492321",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/ministral-3b": {
"input_cost_per_token": 4e-08,
"litellm_provider": "azure_ai",
@ -6211,6 +6104,28 @@
"output_cost_per_token": 2.4e-05,
"supports_tool_choice": true
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.03e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.03e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ap-south-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.18e-06,
"litellm_provider": "bedrock",
@ -6229,6 +6144,17 @@
"mode": "chat",
"output_cost_per_token": 7.2e-07
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.94e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 3.05e-06,
"litellm_provider": "bedrock",
@ -6434,6 +6360,17 @@
"mode": "chat",
"output_cost_per_token": 1.01e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.03e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": {
"input_cost_per_second": 0.011,
"litellm_provider": "bedrock",
@ -6570,6 +6507,28 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us-gov-east-1/amazon.nova-pro-v1:0": {
"input_cost_per_token": 9.6e-07,
"litellm_provider": "bedrock",
@ -6976,6 +6935,17 @@
"output_cost_per_token": 7e-07,
"supports_tool_choice": true
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": true,
"supports_reasoning": true
},
"bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": {
"cache_creation_input_token_cost": 1e-06,
"cache_read_input_token_cost": 8e-08,
@ -7783,6 +7753,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
@ -7814,6 +7815,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/us/claude-opus-4-6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1.1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1.1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 4.125e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
@ -7845,6 +7877,37 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"fast/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_200k_tokens": 1e-06,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_200k_tokens": 1e-05,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.00015,
"output_cost_per_token_above_200k_tokens": 3.75e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
},
"us/claude-opus-4-6-20260205": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_200k_tokens": 1.375e-05,
@ -25641,6 +25704,66 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
"perplexity/preset/pro-search": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_preset": true
},
"perplexity/openai/gpt-4o": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/openai/gpt-4o-mini": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/openai/gpt-5.2": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": true
},
"perplexity/anthropic/claude-3-5-sonnet-20241022": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/anthropic/claude-3-5-haiku-20241022": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/google/gemini-2.0-flash-exp": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/google/gemini-2.0-flash-thinking-exp": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": true
},
"perplexity/xai/grok-2-1212": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"perplexity/xai/grok-2-vision-1212": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
"supports_reasoning": false
},
"publicai/aisingapore/Qwen-SEA-LION-v4-32B-IT": {
"input_cost_per_token": 0.0,
"litellm_provider": "publicai",
@ -28567,6 +28690,193 @@
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/anthropic/claude-3-5-sonnet": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-3-5-sonnet-20241022": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-3-7-sonnet": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-opus-4": {
"cache_creation_input_token_cost": 1.875e-05,
"cache_read_input_token_cost": 1.5e-06,
"input_cost_per_token": 1.5e-05,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-opus-4.1": {
"cache_creation_input_token_cost": 1.875e-05,
"cache_read_input_token_cost": 1.5e-06,
"input_cost_per_token": 1.5e-05,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-opus-4.6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-sonnet-4": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/anthropic/claude-sonnet-4.5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "vercel_ai_gateway",
"max_input_tokens": 1000000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"vercel_ai_gateway/cohere/command-a": {
"input_cost_per_token": 2.5e-06,
"litellm_provider": "vercel_ai_gateway",
@ -28576,7 +28886,8 @@
"mode": "chat",
"output_cost_per_token": 1e-05,
"supports_function_calling": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/cohere/command-r": {
"input_cost_per_token": 1.5e-07,

54
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -16,6 +16,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
)
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.utils import get_server_root_path
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPServer
router = APIRouter(
@ -125,6 +126,29 @@ def decode_state_hash(encrypted_state: str) -> dict:
return state_data
def _resolve_oauth2_server_for_root_endpoints(
client_ip: Optional[str] = None,
) -> Optional[MCPServer]:
"""
Resolve the MCP server for root-level OAuth endpoints (no server name in path).
When the MCP SDK hits root-level endpoints like /register, /authorize, /token
without a server name prefix, we try to find the right server automatically.
Returns the server if exactly one OAuth2 server is configured, else None.
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
registry = global_mcp_server_manager.get_filtered_registry(client_ip=client_ip)
oauth2_servers = [
s for s in registry.values() if s.auth_type == MCPAuth.oauth2
]
if len(oauth2_servers) == 1:
return oauth2_servers[0]
return None
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -305,6 +329,8 @@ async def authorize(
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
lookup_name, client_ip=client_ip
)
if mcp_server is None and mcp_server_name is None:
mcp_server = _resolve_oauth2_server_for_root_endpoints()
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
return await authorize_with_server(
@ -350,6 +376,8 @@ async def token_endpoint(
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
lookup_name, client_ip=client_ip
)
if mcp_server is None and mcp_server_name is None:
mcp_server = _resolve_oauth2_server_for_root_endpoints()
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
return await exchange_token_with_server(
@ -430,6 +458,13 @@ def _build_oauth_protected_resource_response(
)
request_base_url = get_request_base_url(request)
# When no server name provided, try to resolve the single OAuth2 server
if mcp_server_name is None:
resolved = _resolve_oauth2_server_for_root_endpoints()
if resolved:
mcp_server_name = resolved.server_name or resolved.name
mcp_server: Optional[MCPServer] = None
if mcp_server_name:
client_ip = IPAddressUtils.get_mcp_client_ip(request)
@ -535,6 +570,12 @@ def _build_oauth_authorization_server_response(
request_base_url = get_request_base_url(request)
# When no server name provided, try to resolve the single OAuth2 server
if mcp_server_name is None:
resolved = _resolve_oauth2_server_for_root_endpoints()
if resolved:
mcp_server_name = resolved.server_name or resolved.name
authorization_endpoint = (
f"{request_base_url}/{mcp_server_name}/authorize"
if mcp_server_name
@ -640,6 +681,19 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
"redirect_uris": [f"{request_base_url}/callback"],
}
if not mcp_server_name:
resolved = _resolve_oauth2_server_for_root_endpoints()
if resolved:
return await register_client_with_server(
request=request,
mcp_server=resolved,
client_name=data.get("client_name", ""),
grant_types=data.get("grant_types", []),
response_types=data.get("response_types", []),
token_endpoint_auth_method=data.get(
"token_endpoint_auth_method", ""
),
fallback_client_id=resolved.server_name or resolved.name,
)
return dummy_return
client_ip = IPAddressUtils.get_mcp_client_ip(request)

56
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -36,6 +36,7 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mcp_auth
from litellm.proxy._experimental.mcp_server.utils import (
MCP_TOOL_PREFIX_SEPARATOR,
add_server_prefix_to_name,
@ -340,7 +341,7 @@ class MCPServerManager:
verbose_logger.info(
f"Loading OpenAPI spec from {spec_path} for server {server_name}"
)
self._register_openapi_tools(
await self._register_openapi_tools(
spec_path=spec_path,
server=new_server,
base_url=server_config.get("url", ""),
@ -352,7 +353,9 @@ class MCPServerManager:
self.initialize_tool_name_to_mcp_server_name_mapping()
def _register_openapi_tools(self, spec_path: str, server: MCPServer, base_url: str):
async def _register_openapi_tools(
self, spec_path: str, server: MCPServer, base_url: str
):
"""
Register tools from an OpenAPI specification for a given server.
@ -374,15 +377,15 @@ class MCPServerManager:
get_base_url as get_openapi_base_url,
)
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
load_openapi_spec,
load_openapi_spec_async,
)
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
try:
# Load OpenAPI spec
spec = load_openapi_spec(spec_path)
# Load OpenAPI spec (async to avoid "called from within a running event loop")
spec = await load_openapi_spec_async(spec_path)
# Use base_url from config if provided, otherwise extract from spec
if not base_url:
@ -833,7 +836,7 @@ class MCPServerManager:
return resolved_env
def _create_mcp_client(
async def _create_mcp_client(
self,
server: MCPServer,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
@ -843,13 +846,22 @@ class MCPServerManager:
"""
Create an MCPClient instance for the given server.
Auth resolution (single place for all auth logic):
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
Args:
server (MCPServer): The server configuration
mcp_auth_header: MCP auth header to be passed to the MCP server. This is optional and will be used if provided.
server: The server configuration.
mcp_auth_header: Optional per-request auth override.
extra_headers: Additional headers to forward.
stdio_env: Environment variables for stdio transport.
Returns:
MCPClient: Configured MCP client instance
Configured MCP client instance.
"""
auth_value = await resolve_mcp_auth(server, mcp_auth_header)
transport = server.transport or MCPTransport.sse
# Handle stdio transport
@ -868,7 +880,7 @@ class MCPServerManager:
server_url="", # Not used for stdio
transport_type=transport,
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
auth_value=auth_value,
timeout=60.0,
stdio_config=stdio_config,
extra_headers=extra_headers,
@ -880,7 +892,7 @@ class MCPServerManager:
server_url=server_url,
transport_type=transport,
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
auth_value=auth_value,
timeout=60.0,
extra_headers=extra_headers,
)
@ -920,7 +932,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -980,7 +992,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -1024,7 +1036,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -1068,7 +1080,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -1109,7 +1121,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -1139,7 +1151,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
@ -1943,7 +1955,7 @@ class MCPServerManager:
stdio_env = self._build_stdio_env(mcp_server, raw_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
@ -2119,8 +2131,8 @@ class MCPServerManager:
Note: This now handles prefixed tool names
"""
for server in self.get_registry().values():
if server.auth_type == MCPAuth.oauth2:
# Skip OAuth2 servers for now as they may require user-specific tokens
if server.needs_user_oauth_token:
# Skip OAuth2 servers that rely on user-provided tokens
continue
tools = await self._get_tools_from_server(server)
for tool in tools:
@ -2414,7 +2426,7 @@ class MCPServerManager:
should_skip_health_check = False
# Skip if auth_type is oauth2
if server.auth_type == MCPAuth.oauth2:
if server.needs_user_oauth_token:
should_skip_health_check = True
# Skip if auth_type is not none and authentication_token is missing
elif (
@ -2429,7 +2441,7 @@ class MCPServerManager:
if server.static_headers:
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
client = await self._create_mcp_client(
server=server,
mcp_auth_header=None,
extra_headers=extra_headers,

163
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py Normal file
View file

@ -0,0 +1,163 @@
"""
OAuth2 client_credentials token cache for MCP servers.
Automatically fetches and refreshes access tokens for MCP servers configured
with ``client_id``, ``client_secret``, and ``token_url``.
"""
import asyncio
from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
import httpx
from litellm._logging import verbose_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import (
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
from litellm.types.mcp_server.mcp_server_manager import MCPServer
class MCPOAuth2TokenCache(InMemoryCache):
"""
In-memory cache for OAuth2 client_credentials tokens, keyed by server_id.
Inherits from ``InMemoryCache`` for TTL-based storage and eviction.
Adds per-server ``asyncio.Lock`` to prevent duplicate concurrent fetches.
"""
def __init__(self) -> None:
super().__init__(
max_size_in_memory=MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE,
default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
)
self._locks: Dict[str, asyncio.Lock] = {}
def _get_lock(self, server_id: str) -> asyncio.Lock:
return self._locks.setdefault(server_id, asyncio.Lock())
async def async_get_token(self, server: "MCPServer") -> Optional[str]:
"""Return a valid access token, fetching or refreshing as needed.
Returns ``None`` when the server lacks client credentials config.
"""
if not server.has_client_credentials:
return None
server_id = server.server_id
# Fast path — cached token is still valid
cached = self.get_cache(server_id)
if cached is not None:
return cached
# Slow path — acquire per-server lock then double-check
async with self._get_lock(server_id):
cached = self.get_cache(server_id)
if cached is not None:
return cached
token, ttl = await self._fetch_token(server)
self.set_cache(server_id, token, ttl=ttl)
return token
async def _fetch_token(self, server: "MCPServer") -> Tuple[str, int]:
"""POST to ``token_url`` with ``grant_type=client_credentials``.
Returns ``(access_token, ttl_seconds)`` where ttl accounts for the
expiry buffer so the cache entry expires before the real token does.
"""
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
if not server.client_id or not server.client_secret or not server.token_url:
raise ValueError(
f"MCP server '{server.server_id}' missing required OAuth2 fields: "
f"client_id={bool(server.client_id)}, "
f"client_secret={bool(server.client_secret)}, "
f"token_url={bool(server.token_url)}"
)
data: Dict[str, str] = {
"grant_type": "client_credentials",
"client_id": server.client_id,
"client_secret": server.client_secret,
}
if server.scopes:
data["scope"] = " ".join(server.scopes)
verbose_logger.debug(
"Fetching OAuth2 client_credentials token for MCP server %s",
server.server_id,
)
try:
response = await client.post(server.token_url, data=data)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
raise ValueError(
f"OAuth2 token request for MCP server '{server.server_id}' "
f"failed with status {exc.response.status_code}"
) from exc
body = response.json()
if not isinstance(body, dict):
raise ValueError(
f"OAuth2 token response for MCP server '{server.server_id}' "
f"returned non-object JSON (got {type(body).__name__})"
)
access_token = body.get("access_token")
if not access_token:
raise ValueError(
f"OAuth2 token response for MCP server '{server.server_id}' "
f"missing 'access_token'"
)
# Safely parse expires_in — providers may return null or non-numeric values
raw_expires_in = body.get("expires_in")
try:
expires_in = int(raw_expires_in) if raw_expires_in is not None else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
except (TypeError, ValueError):
expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
ttl = max(expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, MCP_OAUTH2_TOKEN_CACHE_MIN_TTL)
verbose_logger.info(
"Fetched OAuth2 token for MCP server %s (expires in %ds)",
server.server_id,
expires_in,
)
return access_token, ttl
def invalidate(self, server_id: str) -> None:
"""Remove a cached token (e.g. after a 401)."""
self.delete_cache(server_id)
mcp_oauth2_token_cache = MCPOAuth2TokenCache()
async def resolve_mcp_auth(
server: "MCPServer",
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
) -> Optional[Union[str, Dict[str, str]]]:
"""Resolve the auth value for an MCP server.
Priority:
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
"""
if mcp_auth_header:
return mcp_auth_header
if server.has_client_credentials:
return await mcp_oauth2_token_cache.async_get_token(server)
return server.authentication_token

34
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -3,6 +3,8 @@ This module is used to generate MCP tools from OpenAPI specs.
"""
import json
import asyncio
import os
from pathlib import PurePosixPath
from typing import Any, Dict, Optional
from urllib.parse import quote
@ -45,8 +47,36 @@ def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
def load_openapi_spec(filepath: str) -> Dict[str, Any]:
"""Load OpenAPI specification from JSON file."""
with open(filepath, "r") as f:
"""
Sync wrapper. For URL specs, use the shared/custom MCP httpx client.
"""
try:
# If we're already inside an event loop, prefer the async function.
asyncio.get_running_loop()
raise RuntimeError(
"load_openapi_spec() was called from within a running event loop. "
"Use 'await load_openapi_spec_async(...)' instead."
)
except RuntimeError as e:
# "no running event loop" is fine; other RuntimeErrors we re-raise
if "no running event loop" not in str(e).lower():
raise
return asyncio.run(load_openapi_spec_async(filepath))
async def load_openapi_spec_async(filepath: str) -> Dict[str, Any]:
if filepath.startswith("http://") or filepath.startswith("https://"):
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
# NOTE: do not close shared client if get_async_httpx_client returns a shared singleton.
# If it returns a new client each time, consider wrapping it in an async context manager.
r = await client.get(filepath)
r.raise_for_status()
return r.json()
# fallback: local file
# Local filesystem path
if not os.path.exists(filepath):
raise FileNotFoundError(f"OpenAPI spec not found at {filepath}")
with open(filepath, "r", encoding="utf-8") as f:
return json.load(f)

65
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,6 +1,6 @@
import importlib
from datetime import datetime
from typing import Dict, List, Optional, Union
from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
from fastapi import APIRouter, Depends, HTTPException, Query, Request
@ -501,24 +501,50 @@ if MCP_AVAILABLE:
NewMCPServerRequest,
)
def _extract_credentials(
request: NewMCPServerRequest,
) -> tuple:
"""
Extract OAuth credentials from the nested ``request.credentials`` dict.
Returns:
(client_id, client_secret, scopes) — any value may be ``None``.
"""
creds = request.credentials if isinstance(request.credentials, dict) else {}
client_id: Optional[str] = creds.get("client_id")
client_secret: Optional[str] = creds.get("client_secret")
scopes_raw = creds.get("scopes")
scopes: Optional[List[str]] = scopes_raw if isinstance(scopes_raw, list) else None
return client_id, client_secret, scopes
async def _execute_with_mcp_client(
request: NewMCPServerRequest,
operation,
operation: Callable[..., Awaitable[Any]],
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
):
) -> dict:
"""
Common helper to create MCP client, execute operation, and ensure proper cleanup.
Create a temporary MCP client from *request*, run *operation*, and return the result.
For M2M OAuth servers (those with ``client_id``, ``client_secret``, and
``token_url``), the incoming ``oauth2_headers`` are dropped so that
``resolve_mcp_auth`` can auto-fetch a token via ``client_credentials``.
Args:
request: MCP server configuration
operation: Async function that takes a client and returns the operation result
request: MCP server configuration submitted by the UI.
operation: Async callable that receives the created client and returns a result dict.
mcp_auth_header: Pre-resolved credential header (API-key / bearer token).
oauth2_headers: Headers extracted from the incoming request (may contain the
litellm API key — must NOT be forwarded for M2M servers).
raw_headers: Raw request headers forwarded for stdio env construction.
Returns:
Operation result or error response
The dict returned by *operation*, or an error dict on failure.
"""
try:
client_id, client_secret, scopes = _extract_credentials(request)
server_model = MCPServer(
server_id=request.server_id or "",
name=request.alias or request.server_name or "",
@ -530,18 +556,30 @@ if MCP_AVAILABLE:
args=request.args,
env=request.env,
static_headers=request.static_headers,
client_id=client_id,
client_secret=client_secret,
token_url=request.token_url,
scopes=scopes,
authorization_url=request.authorization_url,
registration_url=request.registration_url,
)
stdio_env = global_mcp_server_manager._build_stdio_env(
server_model, raw_headers
)
# For M2M OAuth servers, drop the incoming Authorization header so that
# resolve_mcp_auth can auto-fetch a token via client_credentials.
effective_oauth2_headers = (
None if server_model.has_client_credentials else oauth2_headers
)
merged_headers = merge_mcp_headers(
extra_headers=oauth2_headers,
extra_headers=effective_oauth2_headers,
static_headers=request.static_headers,
)
client = global_mcp_server_manager._create_mcp_client(
client = await global_mcp_server_manager._create_mcp_client(
server=server_model,
mcp_auth_header=mcp_auth_header,
extra_headers=merged_headers,
@ -550,11 +588,14 @@ if MCP_AVAILABLE:
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
except (KeyboardInterrupt, SystemExit):
raise
except BaseException as e:
verbose_logger.error("Error in MCP operation: %s", e, exc_info=True)
return {
"status": "error",
"message": "An internal error has occurred while testing the MCP server.",
"error": True,
"message": "Failed to connect to MCP server. Check proxy logs for details.",
}
@router.post("/test/connection", dependencies=[Depends(user_api_key_auth)])

5
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -31,6 +31,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
get_request_base_url,
)
from litellm.proxy._experimental.mcp_server.utils import (
LITELLM_MCP_SERVER_DESCRIPTION,
LITELLM_MCP_SERVER_NAME,
@ -1972,7 +1975,7 @@ if MCP_AVAILABLE:
)
if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers:
request = StarletteRequest(scope)
base_url = str(request.base_url).rstrip("/")
base_url = get_request_base_url(request)
authorization_uri = (
f"Bearer authorization_uri="

0
litellm/proxy/_experimental/out/404.html → litellm/proxy/_experimental/out/404/index.html
View file

0
litellm/proxy/_experimental/out/_not-found.html → litellm/proxy/_experimental/out/_not-found/index.html
View file

0
litellm/proxy/_experimental/out/api-reference.html → litellm/proxy/_experimental/out/api-reference/index.html
View file

0
litellm/proxy/_experimental/out/experimental/api-playground.html → litellm/proxy/_experimental/out/experimental/api-playground/index.html
View file

0
litellm/proxy/_experimental/out/experimental/budgets.html → litellm/proxy/_experimental/out/experimental/budgets/index.html
View file

0
litellm/proxy/_experimental/out/experimental/caching.html → litellm/proxy/_experimental/out/experimental/caching/index.html
View file

0
litellm/proxy/_experimental/out/experimental/claude-code-plugins.html → litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
View file

0
litellm/proxy/_experimental/out/experimental/old-usage.html → litellm/proxy/_experimental/out/experimental/old-usage/index.html
View file

0
litellm/proxy/_experimental/out/experimental/prompts.html → litellm/proxy/_experimental/out/experimental/prompts/index.html
View file

0
litellm/proxy/_experimental/out/experimental/tag-management.html → litellm/proxy/_experimental/out/experimental/tag-management/index.html
View file

0
litellm/proxy/_experimental/out/guardrails.html → litellm/proxy/_experimental/out/guardrails/index.html
View file

0
litellm/proxy/_experimental/out/login.html → litellm/proxy/_experimental/out/login/index.html
View file

0
litellm/proxy/_experimental/out/logs.html → litellm/proxy/_experimental/out/logs/index.html
View file

0
litellm/proxy/_experimental/out/mcp/oauth/callback.html → litellm/proxy/_experimental/out/mcp/oauth/callback/index.html
View file

0
litellm/proxy/_experimental/out/model-hub.html → litellm/proxy/_experimental/out/model-hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub.html → litellm/proxy/_experimental/out/model_hub/index.html
View file

Some files were not shown because too many files have changed in this diff Show more