scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge pull request #18962 from BerriAI/main

Browse source 
merge main
This commit is contained in:
Sameer Kankute 2026-01-12 18:30:28 +05:30 • committed by GitHub
commit 4c0ca53153
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
452 changed files with 19697 additions and 3510 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.circleci/config.yml
View file

@ -2038,6 +2038,39 @@ jobs:
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
memory_leak_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- setup_litellm_test_deps
- run:
name: Install Memory Test Dependencies
command: |
pip install "psutil>=5.9.0"
pip install "fastapi>=0.100.0"
pip install "httpx>=0.24.0"
pip install "uvicorn>=0.23.0"
- run:
name: Run Linear Memory Growth Tests
command: |
echo "Running memory leak tests individually to avoid baseline drift..."
echo "Running test_memory_baseline_1k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_1k -v -s --tb=short
echo "Running test_memory_baseline_2k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_2k -v -s --tb=short
echo "Running test_memory_baseline_4k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_4k -v -s --tb=short
echo "Running test_memory_baseline_10k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_10k -v -s --tb=short
echo "Running test_memory_baseline_30k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_30k -v -s --tb=short
no_output_timeout: 60m
db_migration_disable_update_check:
machine:
image: ubuntu-2204:2023.10.1
@ -2102,10 +2135,11 @@ jobs:
name: Check container logs for expected message
command: |
echo "=== Printing Full Container Startup Logs ==="
docker logs my-app
LOG_OUTPUT="$(docker logs my-app 2>&1)"
printf '%s\n' "$LOG_OUTPUT"
echo "=== End of Full Container Startup Logs ==="
if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
echo "Expected message found in logs. Test passed."
else
echo "Expected message not found in logs. Test failed."
@ -3557,12 +3591,34 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7} \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7} \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$SMALL_DATABASE_URL \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e UI_USERNAME="admin" \
@ -3765,6 +3821,12 @@ workflows:
only:
- main
- /litellm_.*/
- memory_leak_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
@ -3792,6 +3854,7 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
context: e2e_ui_tests
requires:
- ui_build
- build_docker_database_image

15
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -16,6 +16,21 @@ body:
value: "A bug happened!"
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to Reproduce
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
placeholder: |
1. config.yaml file/ .env file/ etc.
2. Run the following code...
3. Observe the error...
value: |
1.
2.
3.
validations:
required: true
- type: textarea
id: logs
attributes:

29
.github/workflows/ghcr_deploy.yml vendored
View file

@ -5,6 +5,7 @@ on:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
@ -336,9 +337,9 @@ jobs:
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
else
# Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
# Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
fi
@ -350,28 +351,42 @@ jobs:
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
version-fragment: 'bug'
# Add suffix for non-stable releases (semantic versioning)
- name: Calculate chart version with prerelease suffix
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
INPUT_TAG="${{ github.event.inputs.tag }}"
# Chart version (independent Helm chart versioning with release type suffix)
if [ "$RELEASE_TYPE" = "stable" ]; then
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
else
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
fi
# App version (must match Docker tags)
# stable/rc releases: Docker creates main-{tag}, so use the tag
# latest/dev releases: Docker only creates main-{release_type}, so use release_type
if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
APP_VERSION="${INPUT_TAG}"
else
APP_VERSION="${RELEASE_TYPE}"
fi
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}

1
.github/workflows/publish-migrations.yml vendored
View file

@ -13,6 +13,7 @@ on:
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:

2
README.md
View file

@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged.
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>

4
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,13 +18,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.10
version: 1.0.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.50.2
appVersion: v1.80.12
dependencies:
- name: "postgresql"

42
docs/my-website/docs/completion/input.md
View file

@ -142,7 +142,47 @@ def completion(
- `tool_call_id`: *str (optional)* - Tool call that this message is responding to.
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392)
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664)
#### Content Types
`content` can be a string (text only) or a list of content blocks (multimodal):
| Type | Description | Docs |
|------|-------------|------|
| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) |
| `image_url` | Images | [Vision](./vision.md) |
| `input_audio` | Audio input | [Audio](./audio.md) |
| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) |
| `file` | Files | [Document Understanding](./document_understanding.md) |
| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) |
**Examples:**
```python
# Text
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}]
# Image
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}]
# Audio
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "<base64>", "format": "wav"}}]}]
# Video
messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}]
# File
messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}]
# Document
messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": "<base64>"}}]}]
# Combining multiple types (multimodal)
messages=[{"role": "user", "content": [
{"type": "text", "text": "Generate a product description based on this image"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]}]
```
## Optional Fields

13
docs/my-website/docs/mcp_control.md
View file

@ -649,3 +649,16 @@ general_settings:
```
This is useful when you want discoverability for MCP offerings without granting additional execution privileges.
## Publish MCP Registry
If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry).
1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy.
2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`.
3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL.
:::note Permissions still apply
The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions.
:::

122
docs/my-website/docs/observability/qualifire_integration.md Normal file
View file

@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
# Qualifire - LLM Evaluation, Guardrails & Observability
[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications.
**Key Features:**
- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities
- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches
- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents
- **Prompt Management** - Centralized prompt management with versioning and no-code studio
:::tip
Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more.
:::
## Pre-Requisites
1. Create an account on [Qualifire](https://app.qualifire.ai/)
2. Get your API key and webhook URL from the Qualifire dashboard
```bash
pip install litellm
```
## Quick Start
Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire.
```python
litellm.callbacks = ["qualifire_eval"]
```
```python
import litellm
import os
# Set Qualifire credentials
os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key"
os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = "your-openai-api-key"
# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire
litellm.callbacks = ["qualifire_eval"]
# OpenAI call
response = litellm.completion(
model="gpt-5",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
]
)
```
## Using with LiteLLM Proxy
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["qualifire_eval"]
general_settings:
master_key: "sk-1234"
environment_variables:
QUALIFIRE_API_KEY: "your-qualifire-api-key"
QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations"
```
2. Start the proxy
```bash
litellm --config config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
```
## Environment Variables
| Variable | Description |
| ----------------------- | ------------------------------------------------------ |
| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication |
| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard |
## What Gets Logged?
The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call.
This includes:
- Request messages and parameters
- Response content and metadata
- Token usage statistics
- Latency metrics
- Model information
- Cost data
Once data is in Qualifire, you can:
- Run evaluations to detect hallucinations, toxicity, and policy violations
- Set up guardrails to block or modify responses in real-time
- View traces across your entire AI pipeline
- Track performance and quality metrics over time

109
docs/my-website/docs/providers/abliteration.md Normal file
View file

@ -0,0 +1,109 @@
# Abliteration
## Overview
| Property | Details |
|-------|-------|
| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. |
| Provider Route on LiteLLM | `abliteration/` |
| Link to Provider Doc | [Abliteration](https://abliteration.ai) |
| Base URL | `https://api.abliteration.ai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key
```
## Sample Usage
```python showLineNumbers title="Abliteration Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
)
print(response)
```
## Sample Usage - Streaming
```python showLineNumbers title="Abliteration Streaming Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Stream a short reply"}],
stream=True,
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy Server
1. Add the model to your proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: abliteration-chat
litellm_params:
model: abliteration/abliterated-model
api_key: os.environ/ABLITERATION_API_KEY
```
2. Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
## Direct API Usage (Bearer Token)
Use the environment variable as a Bearer token against the OpenAI-compatible endpoint:
`https://api.abliteration.ai/v1/chat/completions`.
```bash showLineNumbers title="cURL"
export ABLITERATION_API_KEY=""
curl https://api.abliteration.ai/v1/chat/completions \
-H "Authorization: Bearer ${ABLITERATION_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}]
}'
```
```python showLineNumbers title="Python (requests)"
import os
import requests
api_key = os.environ["ABLITERATION_API_KEY"]
response = requests.post(
"https://api.abliteration.ai/v1/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}],
},
timeout=60,
)
print(response.json())
```

24
docs/my-website/docs/providers/bedrock.md
View file

@ -967,6 +967,30 @@ Control the processing tier for your Bedrock requests using `serviceTier`. Valid
[Bedrock ServiceTier API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ServiceTier.html)
### OpenAI-compatible `service_tier` parameter
LiteLLM also supports the OpenAI-style `service_tier` parameter, which is automatically translated to Bedrock's native `serviceTier` format:
| OpenAI `service_tier` | Bedrock `serviceTier` |
|-----------------------|----------------------|
| `"priority"` | `{"type": "priority"}` |
| `"default"` | `{"type": "default"}` |
| `"flex"` | `{"type": "flex"}` |
| `"auto"` | `{"type": "default"}` |
```python
from litellm import completion
# Using OpenAI-style service_tier parameter
response = completion(
model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
service_tier="priority" # Automatically translated to serviceTier={"type": "priority"}
)
```
### Native Bedrock `serviceTier` parameter
<Tabs>
<TabItem value="sdk" label="SDK">

177
docs/my-website/docs/providers/manus.md
View file

@ -9,7 +9,7 @@ Use Manus AI agents through LiteLLM's OpenAI-compatible Responses API.
|----------|---------|
| Description | Manus is an AI agent platform for complex reasoning tasks, document analysis, and multi-step workflows with asynchronous task execution. |
| Provider Route on LiteLLM | `manus/{agent_profile}` |
| Supported Operations | `/responses` (Responses API) |
| Supported Operations | `/responses` (Responses API), `/files` (Files API) |
| Provider Doc | [Manus API ↗](https://open.manus.im/docs/openai-compatibility) |
## Model Format
@ -188,7 +188,182 @@ For production applications, use [webhooks](https://open.manus.im/docs/webhooks)
| `max_output_tokens` | ✅ | Limits response length |
| `previous_response_id` | ✅ | For multi-turn conversations |
## Files API
Manus supports file uploads for document analysis and processing. Files can be uploaded and then referenced in Responses API calls.
### LiteLLM Python SDK
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import litellm
import os
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Upload file
file_content = b"This is a document for analysis."
created_file = await litellm.acreate_file(
file=("document.txt", file_content),
purpose="assistants",
custom_llm_provider="manus",
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = await litellm.aresponses(
model="manus/manus-1.6",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id},
],
},
],
extra_body={"task_mode": "agent", "agent_profile": "manus-1.6-agent"},
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = await litellm.afile_retrieve(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"File details: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = await litellm.afile_delete(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"Deleted: {deleted_file.deleted}")
```
### LiteLLM AI Gateway
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Upload File"
# Upload file
curl -X POST http://localhost:4000/v1/files \
-H "Authorization: Bearer your-proxy-key" \
-F "file=@document.txt" \
-F "purpose=assistants" \
-F "custom_llm_provider=manus"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Use File with Responses API"
# Create response with file
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": [
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": "file_abc123"}
]
}
]
}'
```
```bash showLineNumbers title="Retrieve File"
# Get file details
curl http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Delete File"
# Delete file
curl -X DELETE http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"deleted": true
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import openai
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Upload file
with open("document.txt", "rb") as f:
created_file = client.files.create(
file=f,
purpose="assistants",
extra_body={"custom_llm_provider": "manus"}
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = client.responses.create(
model="manus-agent",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id}
]
}
]
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = client.files.retrieve(created_file.id)
print(f"File: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = client.files.delete(created_file.id)
print(f"Deleted: {deleted_file.deleted}")
```
</TabItem>
</Tabs>
## Related Documentation
- [LiteLLM Responses API](/docs/response_api)
- [LiteLLM Files API](/docs/proxy/litellm_managed_files)
- [Manus OpenAI Compatibility](https://open.manus.im/docs/openai-compatibility)

17
docs/my-website/docs/providers/openrouter.md
View file

@ -1,5 +1,5 @@
# OpenRouter
LiteLLM supports all the text / chat / vision models from [OpenRouter](https://openrouter.ai/docs)
LiteLLM supports all the text / chat / vision / embedding models from [OpenRouter](https://openrouter.ai/docs)
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_OpenRouter.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
@ -78,3 +78,18 @@ response = completion(
route= ""
)
```
## Embedding
```python
from litellm import embedding
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = embedding(
model="openrouter/openai/text-embedding-3-small",
input=["good morning from litellm", "this is another item"],
)
print(response)
```

6
docs/my-website/docs/proxy/db_deadlocks.md
View file

@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# High Availability Setup (Resolve DB Deadlocks)
:::tip Essential for Production
This configuration is **required** for production deployments handling 1000+ requests per second. Without Redis configured, you may experience PostgreSQL connection exhaustion (`FATAL: sorry, too many clients already`).
:::
Resolve any Database Deadlocks you see in high traffic by using this setup
## What causes the problem?

20
docs/my-website/docs/proxy/deploy.md
View file

@ -359,6 +359,26 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M
### Deploy with Database
##### Docker, Kubernetes, Helm Chart
:::warning High Traffic Deployments (1000+ RPS)
If you expect high traffic (1000+ requests per second), **Redis is required** to prevent database connection exhaustion and deadlocks.
Add this to your config:
```yaml
general_settings:
use_redis_transaction_buffer: true
litellm_settings:
cache: true
cache_params:
type: redis
host: your-redis-host
```
See [Resolve DB Deadlocks](/docs/proxy/db_deadlocks) for details.
:::
Requirements:
- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
- Set a `LITELLM_MASTER_KEY`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`)

117
docs/my-website/docs/proxy/endpoint_activity.md Normal file
View file

@ -0,0 +1,117 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Endpoint Activity
Track and visualize API endpoint usage directly in the dashboard. Monitor endpoint-level activity analytics, spend breakdowns, and performance metrics to understand which endpoints are receiving the most traffic and how they're performing.
## Overview
Endpoint Activity enables you to track spend and usage for individual API endpoints automatically. Every time you call an endpoint through the LiteLLM proxy, activity is automatically tracked and aggregated. This allows you to:
- Track spend per endpoint automatically
- View endpoint-level usage analytics in the Admin UI
- Monitor token consumption by endpoint
- Analyze success and failure rates per endpoint
- Identify which endpoints are getting the most activity
- View trend data showing endpoint usage over time
<Image img={require('../../img/ui_endpoint_activity.png')} />
## How Endpoint Activity Works
Endpoint activity is **automatically tracked** whenever you make API calls through the LiteLLM proxy. No additional configuration is required - simply call your endpoints as usual and activity will be tracked.
### Example API Call
When you make a request to any endpoint, activity is automatically recorded:
```bash showLineNumbers title="Endpoint activity is automatically tracked"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \ # 👈 ENDPOINT AUTOMATICALLY TRACKED
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
The endpoint (`/chat/completions`) will be automatically tracked with:
- Token counts (prompt tokens, completion tokens, total tokens)
- Spend for the request
- Request status (success or failure)
- Timestamp and other metadata
## How to View Endpoint Activity
### View Activity in Admin UI
Navigate to the Endpoint Activity tab in the Admin UI to view endpoint-level analytics:
#### 1. Access Endpoint Activity
Go to the Usage page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=new_usage`) and click on the **Endpoint Activity** tab.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/67601fc0-8415-49b4-8e55-0673d37540c2/ascreenshot_f609a506dfe745c5aadccd332681c32d_text_export.jpeg)
#### 2. View Endpoint Analytics
The Endpoint Activity dashboard provides:
- **Endpoint usage table**: View all endpoints with aggregated metrics including:
- Total requests (successful and failed)
- Success rate percentage
- Total tokens consumed
- Total spend per endpoint
- **Success vs Failed requests chart**: Visualize request success and failure rates by endpoint
- **Usage trends**: See how endpoint activity changes over time with daily trend data
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/41b2b158-3ab3-4154-a0d0-7233451d3f2b/ascreenshot_ff46db6e09b54ea9bf34ae9028aff58a_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/bce32f99-f0ba-4502-8a3a-76257ff5e47a/ascreenshot_2273d3a94acd42e983ad7d6436722c2a_text_export.jpeg)
#### 3. Understand Endpoint Metrics
Each endpoint displays the following metrics:
- **Successful Requests**: Number of requests that completed successfully
- **Failed Requests**: Number of requests that encountered errors
- **Total Requests**: Sum of successful and failed requests
- **Success Rate**: Percentage of successful requests
- **Total Tokens**: Sum of prompt and completion tokens
- **Spend**: Total cost for all requests to that endpoint
## Use Cases
### Performance Monitoring
Monitor endpoint health and performance:
- Identify endpoints with high failure rates
- Track which endpoints are receiving the most traffic
- Monitor token consumption patterns by endpoint
- Detect anomalies in endpoint usage
### Cost Optimization
Understand spend distribution across endpoints:
- Identify high-cost endpoints
- Optimize expensive endpoints
- Allocate budget based on endpoint usage
- Track cost trends over time
---
## Related Features
- [Customer Usage](./customer_usage.md) - Track spend and usage for individual customers
- [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics
- [Spend Logs](./spend_logs.md) - Detailed request-level spend logs

41
docs/my-website/docs/proxy/guardrails/qualifire.md
View file

@ -8,13 +8,7 @@ Use [Qualifire](https://qualifire.ai) to evaluate LLM outputs for quality, safet
## Quick Start
### 1. Install the Qualifire SDK
```bash
pip install qualifire
```
### 2. Define Guardrails on your LiteLLM config.yaml
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
@ -61,13 +55,13 @@ guardrails:
- `post_call` Run **after** LLM call, on **input & output**
- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes
### 3. Start LiteLLM Gateway
### 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 4. Test request
### 3. Test request
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
@ -142,7 +136,7 @@ guardrails:
evaluation_id: eval_abc123 # Your evaluation ID from Qualifire dashboard
```
When `evaluation_id` is provided, LiteLLM will use `invoke_evaluation()` instead of `evaluate()`, running the pre-configured evaluation from your dashboard.
When `evaluation_id` is provided, LiteLLM will use the invoke evaluation API endpoint instead of the evaluate endpoint, running the pre-configured evaluation from your dashboard.
## Available Checks
@ -213,19 +207,19 @@ guardrails:
### Parameter Reference
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | --------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `None` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | ---------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `https://proxy.qualifire.ai` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
### Default Behavior
@ -261,4 +255,3 @@ This evaluates whether the LLM selected the appropriate tools and provided corre
- [Qualifire Documentation](https://docs.qualifire.ai)
- [Qualifire Dashboard](https://app.qualifire.ai)
- [Qualifire Python SDK](https://github.com/qualifire-dev/qualifire-python-sdk)

7
docs/my-website/docs/proxy/load_balancing.md
View file

@ -264,8 +264,15 @@ model_list:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
```
:::important
The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`.
:::
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
### When You'll See Load Balancing in Action

2
docs/my-website/docs/proxy/logging.md
View file

@ -67,7 +67,7 @@ Set `litellm.turn_off_message_logging=True` This will prevent the messages and r
<TabItem value="global" label="Global">
**1. Setup config.yaml **
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo

18
docs/my-website/docs/proxy/pass_through.md
View file

@ -165,6 +165,7 @@ general_settings:
target: string # Target URL for forwarding
auth: boolean # Enable LiteLLM authentication (Enterprise)
forward_headers: boolean # Forward all incoming headers
include_subpath: boolean # If true, forwards requests to sub-paths (default: false)
headers: # Custom headers to add
Authorization: string # Auth header for target API
content-type: string # Request content type
@ -181,6 +182,23 @@ general_settings:
- **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration
- **Custom headers**: Any additional key-value pairs
### Sub-path Routing
By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`:
```yaml
general_settings:
pass_through_endpoints:
- path: "/custom-api" # Any path prefix you choose
target: "https://api.example.com"
include_subpath: true # Forward /custom-api/*, not just /custom-api
```
| Setting | Behavior |
|---------|----------|
| `include_subpath: false` (default) | Only `/custom-api` is forwarded |
| `include_subpath: true` | `/custom-api`, `/custom-api/v1/chat`, `/custom-api/anything` are all forwarded |
---
## Advanced: Custom Adapters

9
docs/my-website/docs/routing.md
View file

@ -861,9 +861,13 @@ model_list = [
},
]
router = Router(model_list=model_list)
router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work
```
:::important
The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router.
:::
</TabItem>
<TabItem value="proxy" label="PROXY">
@ -880,6 +884,9 @@ model_list:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
```
</TabItem>

BIN
docs/my-website/img/ui_endpoint_activity.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 503 KiB

4
docs/my-website/release_notes/v1.80.11-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.80.11 - Google Interactions API"
title: "v1.80.11 - Google Interactions API"
slug: "v1-80-11"
date: 2025-12-20T10:00:00
authors:
@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.80.11.rc.1
docker.litellm.ai/berriai/litellm:v1.80.11-stable
```
</TabItem>

643
docs/my-website/release_notes/v1.80.15/index.md Normal file
View file

@ -0,0 +1,643 @@
---
title: "v1.80.15 - Manus API Support"
slug: "v1-80-15"
date: 2026-01-10T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.80.15.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.15
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Manus API Support** - [New provider support for Manus API on /responses and GET /responses endpoints](../../docs/providers/manus)
- **MiniMax Provider** - [Full support for MiniMax chat completions, TTS, and Anthropic native endpoint](../../docs/providers/minimax)
- **AWS Polly TTS** - [New TTS provider using AWS Polly API](../../docs/providers/aws_polly)
- **SSO Role Mapping** - Configure role mappings for SSO providers directly in the UI
- **Cost Estimator** - New UI tool for estimating costs across multiple models and requests
- **MCP Global Mode** - [Configure MCP servers globally with visibility controls](../../docs/mcp)
- **Interactions API Bridge** - [Use all LiteLLM providers with the Interactions API](../../docs/interactions)
- **RAG Query Endpoint** - [New RAG Search/Query endpoint for retrieval-augmented generation](../../docs/search/index)
- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity.md)
- **50% Overhead Reduction** - LiteLLM now sends 2.5× more requests to LLM providers
---
## Performance - 50% Overhead Reduction
LiteLLM now sends 2.5× more requests to LLM providers by replacing sequential if/elif chains with O(1) dictionary lookups for provider configuration resolution (92.7% faster). This optimization has a high impact because it runs inside the client decorator, which is invoked on every HTTP request made to the proxy server.
### Before
> **Note:** Worse-looking provider metrics are a good sign here—they indicate requests spend less time inside LiteLLM.
```
============================================================
Fake LLM Provider Stats (When called by LiteLLM)
============================================================
Total Time: 0.56s
Requests/Second: 10746.68
Latency Statistics (seconds):
Mean: 0.2039s
Median (p50): 0.2310s
Min: 0.0323s
Max: 0.3928s
Std Dev: 0.1166s
p95: 0.3574s
p99: 0.3748s
Status Codes:
200: 6000
```
### After
```
============================================================
Fake LLM Provider Stats (When called by LiteLLM)
============================================================
Total Time: 1.42s
Requests/Second: 4224.49
Latency Statistics (seconds):
Mean: 0.5300s
Median (p50): 0.5871s
Min: 0.0885s
Max: 1.0482s
Std Dev: 0.3065s
p95: 0.9750s
p99: 1.0444s
Status Codes:
200: 6000
```
> The benchmarks run LiteLLM locally with a lightweight LLM provider to eliminate network latency, isolating internal overhead and bottlenecks so we can focus on reducing pure LiteLLM overhead on a single instance.
---
### UI Usage - Endpoint Activity
<Image
img={require('../../img/ui_endpoint_activity.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
Users can now see Endpoint Activity Metrics in the UI.
---
## New Providers and Endpoints
### New Providers (11 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | ------------------- | ----------- |
| [Manus](../../docs/providers/manus) | `/responses` | Manus API for agentic workflows |
| [Manus](../../docs/providers/manus) | `GET /responses` | Manus API for retrieving responses |
| [Manus](../../docs/providers/manus) | `/files` | Manus API for file management |
| [MiniMax](../../docs/providers/minimax) | `/chat/completions` | MiniMax chat completions |
| [MiniMax](../../docs/providers/minimax) | `/audio/speech` | MiniMax text-to-speech |
| [AWS Polly](../../docs/providers/aws_polly) | `/audio/speech` | AWS Polly text-to-speech API |
| [GigaChat](../../docs/providers/gigachat) | `/chat/completions` | GigaChat provider for Russian language AI |
| [LlamaGate](../../docs/providers/llamagate) | `/chat/completions` | LlamaGate chat completions |
| [LlamaGate](../../docs/providers/llamagate) | `/embeddings` | LlamaGate embeddings |
| [Abliteration AI](../../docs/providers/abliteration) | `/chat/completions` | Abliteration.ai provider support |
| [Bedrock](../../docs/providers/bedrock) | `/v1/messages/count_tokens` | Bedrock as new provider for token counting |
### New LLM API Endpoints (3 new endpoints)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
| `/responses/compact` | POST | Compact responses API endpoint | [Docs](../../docs/response_api) |
| `/rag/query` | POST | RAG Search/Query endpoint | [Docs](../../docs/search/index) |
| `/containers/{id}/files` | POST | Upload files to containers | [Docs](../../docs/container_files) |
---
## New Models / Updated Models
#### New Model Support (100+ new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Azure | `azure/gpt-5.2` | 400K | $1.75 | $14.00 | Reasoning, vision, caching |
| Azure | `azure/gpt-5.2-chat` | 128K | $1.75 | $14.00 | Reasoning, vision |
| Azure | `azure/gpt-5.2-pro` | 400K | $21.00 | $168.00 | Reasoning, vision, web search |
| Azure | `azure/gpt-image-1.5` | - | Token-based | Token-based | Image generation/editing |
| Azure AI | `azure_ai/gpt-oss-120b` | 131K | $0.15 | $0.60 | Function calling |
| Azure AI | `azure_ai/flux.2-pro` | - | - | $0.04/image | Image generation |
| Azure AI | `azure_ai/deepseek-v3.2` | 164K | $0.58 | $1.68 | Reasoning, function calling |
| Bedrock | `amazon.nova-2-multimodal-embeddings-v1:0` | 8K | $0.135 | - | Multimodal embeddings |
| Bedrock | `writer.palmyra-x4-v1:0` | 128K | $2.50 | $10.00 | Function calling, PDF |
| Bedrock | `writer.palmyra-x5-v1:0` | 1M | $0.60 | $6.00 | Function calling, PDF |
| Bedrock | `moonshot.kimi-k2-v1:0` | - | - | - | Kimi K2 model |
| Cerebras | `cerebras/zai-glm-4.6` | 128K | $2.25 | $2.75 | Reasoning, function calling |
| GigaChat | `gigachat/GigaChat-2-Lite` | - | - | - | Chat completions |
| GigaChat | `gigachat/GigaChat-2-Max` | - | - | - | Chat completions |
| GigaChat | `gigachat/GigaChat-2-Pro` | - | - | - | Chat completions |
| Gemini | `gemini/veo-3.1-generate-001` | - | - | - | Video generation |
| Gemini | `gemini/veo-3.1-fast-generate-001` | - | - | - | Video generation |
| GitHub Copilot | 25+ models | Various | - | - | Chat completions |
| LlamaGate | 15+ models | Various | - | - | Chat, vision, embeddings |
| MiniMax | `minimax/abab7-chat-preview` | - | - | - | Chat completions |
| Novita | 80+ models | Various | Various | Various | Chat, vision, embeddings |
| OpenRouter | `openrouter/google/gemini-3-flash-preview` | - | - | - | Chat completions |
| Together AI | Multiple models | Various | Various | Various | Response schema support |
| Vertex AI | `vertex_ai/zai-glm-4.7` | - | - | - | GLM 4.7 support |
#### Features
- **[Gemini](../../docs/providers/gemini)**
- Add image tokens in chat completion - [PR #18327](https://github.com/BerriAI/litellm/pull/18327)
- Add usage object in image generation - [PR #18328](https://github.com/BerriAI/litellm/pull/18328)
- Add thought signature support via tool call id - [PR #18374](https://github.com/BerriAI/litellm/pull/18374)
- Add thought signature for non tool call requests - [PR #18581](https://github.com/BerriAI/litellm/pull/18581)
- Preserve system instructions - [PR #18585](https://github.com/BerriAI/litellm/pull/18585)
- Fix Gemini 3 images in tool response - [PR #18190](https://github.com/BerriAI/litellm/pull/18190)
- Support snake_case for google_search tool parameters - [PR #18451](https://github.com/BerriAI/litellm/pull/18451)
- Google GenAI adapter inline data support - [PR #18477](https://github.com/BerriAI/litellm/pull/18477)
- Add deprecation_date for discontinued Google models - [PR #18550](https://github.com/BerriAI/litellm/pull/18550)
- **[Vertex AI](../../docs/providers/vertex)**
- Add centralized get_vertex_base_url() helper for global location support - [PR #18410](https://github.com/BerriAI/litellm/pull/18410)
- Convert image URLs to base64 for Vertex AI Anthropic - [PR #18497](https://github.com/BerriAI/litellm/pull/18497)
- Separate Tool objects for each tool type per API spec - [PR #18514](https://github.com/BerriAI/litellm/pull/18514)
- Add thought_signatures to VertexGeminiConfig - [PR #18853](https://github.com/BerriAI/litellm/pull/18853)
- Add support for Vertex AI API keys - [PR #18806](https://github.com/BerriAI/litellm/pull/18806)
- Add zai glm-4.7 model support - [PR #18782](https://github.com/BerriAI/litellm/pull/18782)
- **[Azure](../../docs/providers/azure/azure)**
- Add Azure gpt-image-1.5 pricing to cost map - [PR #18347](https://github.com/BerriAI/litellm/pull/18347)
- Add azure/gpt-5.2-chat model - [PR #18361](https://github.com/BerriAI/litellm/pull/18361)
- Add support for image generation via Azure AD token - [PR #18413](https://github.com/BerriAI/litellm/pull/18413)
- Add logprobs support for Azure OpenAI GPT-5.2 model - [PR #18856](https://github.com/BerriAI/litellm/pull/18856)
- Add Azure BFL Flux 2 models for image generation and editing - [PR #18764](https://github.com/BerriAI/litellm/pull/18764), [PR #18766](https://github.com/BerriAI/litellm/pull/18766)
- **[Bedrock](../../docs/providers/bedrock)**
- Add Bedrock Kimi K2 model support - [PR #18797](https://github.com/BerriAI/litellm/pull/18797)
- Add support for model id in bedrock passthrough - [PR #18800](https://github.com/BerriAI/litellm/pull/18800)
- Fix Nova model detection for Bedrock provider - [PR #18250](https://github.com/BerriAI/litellm/pull/18250)
- Ensure toolUse.input is always a dict when converting from OpenAI format - [PR #18414](https://github.com/BerriAI/litellm/pull/18414)
- **[Databricks](../../docs/providers/databricks)**
- Add enhanced authentication, security features, and custom user-agent support - [PR #18349](https://github.com/BerriAI/litellm/pull/18349)
- **[MiniMax](../../docs/providers/minimax)**
- Add MiniMax chat completion support - [PR #18380](https://github.com/BerriAI/litellm/pull/18380)
- Add Anthropic native endpoint support for MiniMax - [PR #18377](https://github.com/BerriAI/litellm/pull/18377)
- Add support for MiniMax TTS - [PR #18334](https://github.com/BerriAI/litellm/pull/18334)
- Add MiniMax provider support to UI dashboard - [PR #18496](https://github.com/BerriAI/litellm/pull/18496)
- **[Together AI](../../docs/providers/togetherai)**
- Add supports_response_schema to all supported Together AI models - [PR #18368](https://github.com/BerriAI/litellm/pull/18368)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add OpenRouter embeddings API support - [PR #18391](https://github.com/BerriAI/litellm/pull/18391)
- **[Anthropic](../../docs/providers/anthropic)**
- Pass server_tool_use and tool_search_tool_result blocks - [PR #18770](https://github.com/BerriAI/litellm/pull/18770)
- Add Anthropic cache control option to image tool call results - [PR #18674](https://github.com/BerriAI/litellm/pull/18674)
- **[Ollama](../../docs/providers/ollama)**
- Add dimensions for ollama embedding - [PR #18536](https://github.com/BerriAI/litellm/pull/18536)
- Extract pure base64 data from data URLs for Ollama - [PR #18465](https://github.com/BerriAI/litellm/pull/18465)
- **[Watsonx](../../docs/providers/watsonx/index)**
- Add Watsonx fields support - [PR #18569](https://github.com/BerriAI/litellm/pull/18569)
- Fix Watsonx Audio Transcription - filter model field - [PR #18810](https://github.com/BerriAI/litellm/pull/18810)
- **[SAP](../../docs/providers/sap)**
- Add SAP creds for list in proxy UI - [PR #18375](https://github.com/BerriAI/litellm/pull/18375)
- Pass through extra params from allowed_openai_params - [PR #18432](https://github.com/BerriAI/litellm/pull/18432)
- Add client header for SAP AI Core Tracking - [PR #18714](https://github.com/BerriAI/litellm/pull/18714)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Correct deepseek-v3p2 pricing - [PR #18483](https://github.com/BerriAI/litellm/pull/18483)
- **[ZAI](../../docs/providers/zai)**
- Add GLM-4.7 model with reasoning support - [PR #18476](https://github.com/BerriAI/litellm/pull/18476)
- **[Codestral](../../docs/providers/codestral)**
- Correctly route codestral chat and FIM endpoints - [PR #18467](https://github.com/BerriAI/litellm/pull/18467)
- **[Azure AI](../../docs/providers/azure_ai)**
- Fix authentication errors at messages API via azure_ai - [PR #18500](https://github.com/BerriAI/litellm/pull/18500)
#### New Provider Support
- **[AWS Polly](../../docs/providers/aws_polly)** - Add AWS Polly API for TTS - [PR #18326](https://github.com/BerriAI/litellm/pull/18326)
- **[GigaChat](../../docs/providers/gigachat)** - Add GigaChat provider support - [PR #18564](https://github.com/BerriAI/litellm/pull/18564)
- **[LlamaGate](../../docs/providers/llamagate)** - Add LlamaGate as a new provider - [PR #18673](https://github.com/BerriAI/litellm/pull/18673)
- **[Abliteration AI](../../docs/providers/abliteration)** - Add abliteration.ai provider - [PR #18678](https://github.com/BerriAI/litellm/pull/18678)
- **[Manus](../../docs/providers/manus)** - Add Manus API support on /responses, GET /responses - [PR #18804](https://github.com/BerriAI/litellm/pull/18804)
- **5 AI Providers via openai_like** - Add 5 AI providers using openai_like - [PR #18362](https://github.com/BerriAI/litellm/pull/18362)
### Bug Fixes
- **[Gemini](../../docs/providers/gemini)**
- Properly catch context window exceeded errors - [PR #18283](https://github.com/BerriAI/litellm/pull/18283)
- Remove prompt caching headers as support has been removed - [PR #18579](https://github.com/BerriAI/litellm/pull/18579)
- Fix generate content request with audio file id - [PR #18745](https://github.com/BerriAI/litellm/pull/18745)
- Fix google_genai streaming adapter provider handling - [PR #18845](https://github.com/BerriAI/litellm/pull/18845)
- **[Groq](../../docs/providers/groq)**
- Remove deprecated Groq models and update model registry - [PR #18062](https://github.com/BerriAI/litellm/pull/18062)
- **[Vertex AI](../../docs/providers/vertex)**
- Handle unsupported region for Vertex AI count tokens endpoint - [PR #18665](https://github.com/BerriAI/litellm/pull/18665)
- **General**
- Fix request body for image embedding request - [PR #18336](https://github.com/BerriAI/litellm/pull/18336)
- Fix lost tool_calls when streaming has both text and tool_calls - [PR #18316](https://github.com/BerriAI/litellm/pull/18316)
- Add all resolution for gpt-image-1.5 - [PR #18586](https://github.com/BerriAI/litellm/pull/18586)
- Fix gpt-image-1 cost calculation using token-based pricing - [PR #17906](https://github.com/BerriAI/litellm/pull/17906)
- Fix response_format leaking into extra_body - [PR #18859](https://github.com/BerriAI/litellm/pull/18859)
- Align max_tokens with max_output_tokens for consistency - [PR #18820](https://github.com/BerriAI/litellm/pull/18820)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Add new compact endpoint (v1/responses/compact) - [PR #18697](https://github.com/BerriAI/litellm/pull/18697)
- Support more streaming callback hooks - [PR #18513](https://github.com/BerriAI/litellm/pull/18513)
- Add mapping for reasoning effort to summary param - [PR #18635](https://github.com/BerriAI/litellm/pull/18635)
- Add output_text property to ResponsesAPIResponse - [PR #18491](https://github.com/BerriAI/litellm/pull/18491)
- Add annotations to completions responses API bridge - [PR #18754](https://github.com/BerriAI/litellm/pull/18754)
- **[Interactions API](../../docs/interactions)**
- Allow using all LiteLLM providers (interactions -> responses API bridge) - [PR #18373](https://github.com/BerriAI/litellm/pull/18373)
- **[RAG Search API](../../docs/search/index)**
- Add RAG Search/Query endpoint - [PR #18376](https://github.com/BerriAI/litellm/pull/18376)
- **[CountTokens API](../../docs/anthropic_count_tokens)**
- Add Bedrock as a new provider for `/v1/messages/count_tokens` - [PR #18858](https://github.com/BerriAI/litellm/pull/18858)
- **[Generate Content](../../docs/providers/gemini)**
- Add generate content in LLM route - [PR #18405](https://github.com/BerriAI/litellm/pull/18405)
- **General**
- Enable async_post_call_failure_hook to transform error responses - [PR #18348](https://github.com/BerriAI/litellm/pull/18348)
- Calculate total_tokens manually if missing and can be calculated - [PR #18445](https://github.com/BerriAI/litellm/pull/18445)
- Add custom llm provider to get_llm_provider when sent via UI - [PR #18638](https://github.com/BerriAI/litellm/pull/18638)
#### Bugs
- **General**
- Handle empty error objects in response conversion - [PR #18493](https://github.com/BerriAI/litellm/pull/18493)
- Preserve client error status codes in streaming mode - [PR #18698](https://github.com/BerriAI/litellm/pull/18698)
- Return json error response instead of SSE format for initial streaming errors - [PR #18757](https://github.com/BerriAI/litellm/pull/18757)
- Fix auth header for custom api base in generateContent request - [PR #18637](https://github.com/BerriAI/litellm/pull/18637)
- Tool content should be string for Deepinfra - [PR #18739](https://github.com/BerriAI/litellm/pull/18739)
- Fix incomplete usage in response object passed - [PR #18799](https://github.com/BerriAI/litellm/pull/18799)
- Unify model names to provider-defined names - [PR #18573](https://github.com/BerriAI/litellm/pull/18573)
---
## Management Endpoints / UI
#### Features
- **SSO Configuration**
- Add SSO Role Mapping feature - [PR #18090](https://github.com/BerriAI/litellm/pull/18090)
- Add SSO Settings Page - [PR #18600](https://github.com/BerriAI/litellm/pull/18600)
- Allow adding role mappings for SSO - [PR #18593](https://github.com/BerriAI/litellm/pull/18593)
- SSO Settings Page Add Role Mappings - [PR #18677](https://github.com/BerriAI/litellm/pull/18677)
- SSO Settings Loading State + Deprecate Previous SSO Flow - [PR #18617](https://github.com/BerriAI/litellm/pull/18617)
- **Virtual Keys**
- Allow deleting key expiry - [PR #18278](https://github.com/BerriAI/litellm/pull/18278)
- Add optional query param "expand" to /key/list - [PR #18502](https://github.com/BerriAI/litellm/pull/18502)
- Key Table Loading Skeleton - [PR #18527](https://github.com/BerriAI/litellm/pull/18527)
- Allow column resizing on Keys Table - [PR #18424](https://github.com/BerriAI/litellm/pull/18424)
- Virtual Keys Table Loading State Between Pages - [PR #18619](https://github.com/BerriAI/litellm/pull/18619)
- Key and Team Router Setting - [PR #18790](https://github.com/BerriAI/litellm/pull/18790)
- Allow router_settings on Keys and Teams - [PR #18675](https://github.com/BerriAI/litellm/pull/18675)
- Use timedelta to calculate key expiry on generate - [PR #18666](https://github.com/BerriAI/litellm/pull/18666)
- **Models + Endpoints**
- Add Model Clearer Flow For Team Admins - [PR #18532](https://github.com/BerriAI/litellm/pull/18532)
- Model Page Loading State - [PR #18574](https://github.com/BerriAI/litellm/pull/18574)
- Model Page Model Provider Select Performance - [PR #18425](https://github.com/BerriAI/litellm/pull/18425)
- Model Page Sorting Sorts Entire Set - [PR #18420](https://github.com/BerriAI/litellm/pull/18420)
- Refactor Model Hub Page - [PR #18568](https://github.com/BerriAI/litellm/pull/18568)
- Add request provider form on UI - [PR #18704](https://github.com/BerriAI/litellm/pull/18704)
- **Organizations & Teams**
- Allow Organization Admins to See Organization Tab - [PR #18400](https://github.com/BerriAI/litellm/pull/18400)
- Resolve Organization Alias on Team Table - [PR #18401](https://github.com/BerriAI/litellm/pull/18401)
- Resolve Team Alias in Organization Info View - [PR #18404](https://github.com/BerriAI/litellm/pull/18404)
- Allow Organization Admins to View Their Organization Info - [PR #18417](https://github.com/BerriAI/litellm/pull/18417)
- Allow editing team_member_budget_duration in /team/update - [PR #18735](https://github.com/BerriAI/litellm/pull/18735)
- Reusable Duration Select + Team Update Member Budget Duration - [PR #18736](https://github.com/BerriAI/litellm/pull/18736)
- **Usage & Spend**
- Add Error Code Filtering on Spend Logs - [PR #18359](https://github.com/BerriAI/litellm/pull/18359)
- Add Error Code Filtering on UI - [PR #18366](https://github.com/BerriAI/litellm/pull/18366)
- Usage Page User Max Budget fix - [PR #18555](https://github.com/BerriAI/litellm/pull/18555)
- Add endpoint to Daily Activity Tables - [PR #18729](https://github.com/BerriAI/litellm/pull/18729)
- Endpoint Activity in Usage - [PR #18798](https://github.com/BerriAI/litellm/pull/18798)
- **Cost Estimator**
- Add Cost Estimator for AI Gateway - [PR #18643](https://github.com/BerriAI/litellm/pull/18643)
- Add view for estimating costs across requests - [PR #18645](https://github.com/BerriAI/litellm/pull/18645)
- Allow selecting many models for cost estimator - [PR #18653](https://github.com/BerriAI/litellm/pull/18653)
- **CloudZero**
- Improve Create and Delete Path for CloudZero - [PR #18263](https://github.com/BerriAI/litellm/pull/18263)
- Add CloudZero UI Docs - [PR #18350](https://github.com/BerriAI/litellm/pull/18350)
- **Playground**
- Add MCP test support to completions on Playground - [PR #18440](https://github.com/BerriAI/litellm/pull/18440)
- Add selectable MCP servers to the playground - [PR #18578](https://github.com/BerriAI/litellm/pull/18578)
- Add custom proxy base URL support to Playground - [PR #18661](https://github.com/BerriAI/litellm/pull/18661)
- **General UI**
- UI styling improvements and fixes - [PR #18310](https://github.com/BerriAI/litellm/pull/18310)
- Add reusable "New" badge component for feature highlights - [PR #18537](https://github.com/BerriAI/litellm/pull/18537)
- Hide New Badges - [PR #18547](https://github.com/BerriAI/litellm/pull/18547)
- Change Budget page to Have Tabs - [PR #18576](https://github.com/BerriAI/litellm/pull/18576)
- Clicking on Logo Directs to Correct URL - [PR #18575](https://github.com/BerriAI/litellm/pull/18575)
- Add UI support for configuring meta URLs - [PR #18580](https://github.com/BerriAI/litellm/pull/18580)
- Expire Previous UI Session Tokens on Login - [PR #18557](https://github.com/BerriAI/litellm/pull/18557)
- Add license endpoint - [PR #18311](https://github.com/BerriAI/litellm/pull/18311)
- Router Fields Endpoint + React Query for Router Fields - [PR #18880](https://github.com/BerriAI/litellm/pull/18880)
#### Bugs
- **UI Fixes**
- Fix Key Creation MCP Settings Submit Form Unintentionally - [PR #18355](https://github.com/BerriAI/litellm/pull/18355)
- Fix UI Disappears in Development Environments - [PR #18399](https://github.com/BerriAI/litellm/pull/18399)
- Fix Disable Admin UI Flag - [PR #18397](https://github.com/BerriAI/litellm/pull/18397)
- Remove Model Analytics From Model Page - [PR #18552](https://github.com/BerriAI/litellm/pull/18552)
- Useful Links Remove Modal on Adding Links - [PR #18602](https://github.com/BerriAI/litellm/pull/18602)
- SSO Edit Modal Clear Role Mapping Values on Provider Change - [PR #18680](https://github.com/BerriAI/litellm/pull/18680)
- UI Login Case Sensitivity fix - [PR #18877](https://github.com/BerriAI/litellm/pull/18877)
- **API Fixes**
- Fix User Invite & Key Generation Email Notification Logic - [PR #18524](https://github.com/BerriAI/litellm/pull/18524)
- Normalize Proxy Config Callback - [PR #18775](https://github.com/BerriAI/litellm/pull/18775)
- Return empty data array instead of 500 when no models configured - [PR #18556](https://github.com/BerriAI/litellm/pull/18556)
- Enforce org level max budget - [PR #18813](https://github.com/BerriAI/litellm/pull/18813)
---
## AI Integrations
### New Integrations (4 new integrations)
| Integration | Type | Description |
| ----------- | ---- | ----------- |
| [Focus](../../docs/observability/focus) | Logging | Focus export support for observability - [PR #18802](https://github.com/BerriAI/litellm/pull/18802) |
| [SigNoz](../../docs/observability/signoz) | Logging | SigNoz integration for observability - [PR #18726](https://github.com/BerriAI/litellm/pull/18726) |
| [Qualifire](../../docs/proxy/guardrails/qualifire) | Guardrails | Qualifire guardrails and eval webhook - [PR #18594](https://github.com/BerriAI/litellm/pull/18594) |
| [Levo AI](../../docs/observability/levo_integration) | Guardrails | Levo AI integration for security - [PR #18529](https://github.com/BerriAI/litellm/pull/18529) |
### Logging
- **[DataDog](../../docs/proxy/logging#datadog)**
- Fix span kind fallback when parent_id missing - [PR #18418](https://github.com/BerriAI/litellm/pull/18418)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Map Gemini cached_tokens to Langfuse cache_read_input_tokens - [PR #18614](https://github.com/BerriAI/litellm/pull/18614)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Align prometheus metric names with DEFINED_PROMETHEUS_METRICS - [PR #18463](https://github.com/BerriAI/litellm/pull/18463)
- Add Prometheus metrics for request queue time and guardrails - [PR #17973](https://github.com/BerriAI/litellm/pull/17973)
- Add caching metrics for cache hits, misses, and tokens - [PR #18755](https://github.com/BerriAI/litellm/pull/18755)
- Skip metrics for invalid API key requests - [PR #18788](https://github.com/BerriAI/litellm/pull/18788)
- **[Braintrust](../../docs/proxy/logging#braintrust)**
- Pass span_attributes in async logging and skip tags on non-root spans - [PR #18409](https://github.com/BerriAI/litellm/pull/18409)
- **[CloudZero](../../docs/proxy/logging#cloudzero)**
- Add user email to CloudZero - [PR #18584](https://github.com/BerriAI/litellm/pull/18584)
- **[OpenTelemetry](../../docs/proxy/logging#opentelemetry)**
- Use already configured opentelemetry providers - [PR #18279](https://github.com/BerriAI/litellm/pull/18279)
- Prevent LiteLLM from closing external OTEL spans - [PR #18553](https://github.com/BerriAI/litellm/pull/18553)
- Allow configuring arize project name for OpenTelemetry service name - [PR #18738](https://github.com/BerriAI/litellm/pull/18738)
- **[LangSmith](../../docs/proxy/logging#langsmith)**
- Add support for LangSmith organization-scoped API keys with tenant ID - [PR #18623](https://github.com/BerriAI/litellm/pull/18623)
- **[Generic API Logger](../../docs/proxy/logging#generic-api-logger)**
- Add log_format option to GenericAPILogger - [PR #18587](https://github.com/BerriAI/litellm/pull/18587)
### Guardrails
- **[Content Filter](../../docs/proxy/guardrails/litellm_content_filter)**
- Add content filter logs page - [PR #18335](https://github.com/BerriAI/litellm/pull/18335)
- Log actual event type for guardrails - [PR #18489](https://github.com/BerriAI/litellm/pull/18489)
- **[Qualifire](../../docs/proxy/guardrails/qualifire)**
- Add Qualifire eval webhook - [PR #18836](https://github.com/BerriAI/litellm/pull/18836)
- **[Lasso Security](../../docs/proxy/guardrails/lasso_security)**
- Add Lasso guardrail API docs - [PR #18652](https://github.com/BerriAI/litellm/pull/18652)
- **[Noma Security](../../docs/proxy/guardrails/noma_security)**
- Add MCP guardrail support for Noma - [PR #18668](https://github.com/BerriAI/litellm/pull/18668)
- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)**
- Remove redundant Bedrock guardrail block handling - [PR #18634](https://github.com/BerriAI/litellm/pull/18634)
- **General**
- Generic guardrail API update - [PR #18647](https://github.com/BerriAI/litellm/pull/18647)
- Prevent proxy startup failures from case-sensitive tool permission guardrail validation - [PR #18662](https://github.com/BerriAI/litellm/pull/18662)
- Extend case normalization to ALL guardrail types - [PR #18664](https://github.com/BerriAI/litellm/pull/18664)
- Fix MCP handling in unified guardrail - [PR #18630](https://github.com/BerriAI/litellm/pull/18630)
- Fix embeddings calltype for guardrail precallhook - [PR #18740](https://github.com/BerriAI/litellm/pull/18740)
---
## Spend Tracking, Budgets and Rate Limiting
- **Platform Fee / Margins** - Add support for Platform Fee / Margins - [PR #18427](https://github.com/BerriAI/litellm/pull/18427)
- **Negative Budget Validation** - Add validation for negative budget - [PR #18583](https://github.com/BerriAI/litellm/pull/18583)
- **Cost Calculation Fixes**
- Correct cost calculation when reasoning_tokens are without text_tokens - [PR #18607](https://github.com/BerriAI/litellm/pull/18607)
- Fix background cost tracking tests - [PR #18588](https://github.com/BerriAI/litellm/pull/18588)
- **Tag Routing** - Support toggling tag matching between ANY and ALL - [PR #18776](https://github.com/BerriAI/litellm/pull/18776)
---
## MCP Gateway
- **MCP Global Mode** - Add MCP global mode - [PR #18639](https://github.com/BerriAI/litellm/pull/18639)
- **MCP Server Visibility** - Add configurable MCP server visibility - [PR #18681](https://github.com/BerriAI/litellm/pull/18681)
- **MCP Registry** - Add MCP registry - [PR #18850](https://github.com/BerriAI/litellm/pull/18850)
- **MCP Stdio Header** - Support MCP stdio header env overrides - [PR #18324](https://github.com/BerriAI/litellm/pull/18324)
- **Parallel Tool Fetching** - Parallelize tool fetching from multiple MCP servers - [PR #18627](https://github.com/BerriAI/litellm/pull/18627)
- **Optimize MCP Server Listing** - Separate health checks for optimized listing - [PR #18530](https://github.com/BerriAI/litellm/pull/18530)
- **Auth Improvements**
- Require auth for MCP connection test endpoint - [PR #18290](https://github.com/BerriAI/litellm/pull/18290)
- Fix MCP gateway OAuth2 auth issues and ClosedResourceError - [PR #18281](https://github.com/BerriAI/litellm/pull/18281)
- **Bug Fixes**
- Fix MCP server health status reporting - [PR #18443](https://github.com/BerriAI/litellm/pull/18443)
- Fix OpenAPI to MCP tool conversion - [PR #18597](https://github.com/BerriAI/litellm/pull/18597)
- Remove exec() usage and handle invalid OpenAPI parameter names for security - [PR #18480](https://github.com/BerriAI/litellm/pull/18480)
- Fix MCP error when using multiple servers simultaneously - [PR #18855](https://github.com/BerriAI/litellm/pull/18855)
- **Migrate MCP Fetching Logic to React Query** - [PR #18352](https://github.com/BerriAI/litellm/pull/18352)
---
## Performance / Loadbalancing / Reliability improvements
- **92.7% Faster Provider Config Lookup** - LiteLLM now stresses LLM providers 2.5x more - [PR #18867](https://github.com/BerriAI/litellm/pull/18867)
- **Lazy Loading Improvements**
- Consolidate lazy import handlers with registry pattern - [PR #18389](https://github.com/BerriAI/litellm/pull/18389)
- Complete lazy loading migration for all 180+ LLM config classes - [PR #18392](https://github.com/BerriAI/litellm/pull/18392)
- Lazy load additional components (types, callbacks, utilities) - [PR #18396](https://github.com/BerriAI/litellm/pull/18396)
- Add lazy loading for get_llm_provider - [PR #18591](https://github.com/BerriAI/litellm/pull/18591)
- Lazy-load heavy audio library and loggers - [PR #18592](https://github.com/BerriAI/litellm/pull/18592)
- Lazy load 9 heavy imports in litellm/utils.py - [PR #18595](https://github.com/BerriAI/litellm/pull/18595)
- Lazy load heavy imports to improve import time and memory usage - [PR #18610](https://github.com/BerriAI/litellm/pull/18610)
- Implement lazy loading for provider configs, model info classes, streaming handlers - [PR #18611](https://github.com/BerriAI/litellm/pull/18611)
- Lazy load 15 additional imports - [PR #18613](https://github.com/BerriAI/litellm/pull/18613)
- Lazy load 15+ unused imports - [PR #18616](https://github.com/BerriAI/litellm/pull/18616)
- Lazy load DatadogLLMObsInitParams - [PR #18658](https://github.com/BerriAI/litellm/pull/18658)
- Migrate utils.py lazy imports to registry pattern - [PR #18657](https://github.com/BerriAI/litellm/pull/18657)
- Lazy load get_llm_provider and remove_index_from_tool_calls - [PR #18608](https://github.com/BerriAI/litellm/pull/18608)
- **Router Improvements**
- Validate routing_strategy at startup to fail fast with helpful error - [PR #18624](https://github.com/BerriAI/litellm/pull/18624)
- Correct num_retries tracking in retry logic - [PR #18712](https://github.com/BerriAI/litellm/pull/18712)
- Improve error messages and validation for wildcard routing with multiple credentials - [PR #18629](https://github.com/BerriAI/litellm/pull/18629)
- **Memory Improvements**
- Add memory pattern detection test and fix bad memory patterns - [PR #18589](https://github.com/BerriAI/litellm/pull/18589)
- Add unbounded data structure detection to memory test - [PR #18590](https://github.com/BerriAI/litellm/pull/18590)
- Add memory leak detection tests with CI integration - [PR #18881](https://github.com/BerriAI/litellm/pull/18881)
- **Database**
- Add idx on LOWER(user_email) for faster duplicate email checks - [PR #18828](https://github.com/BerriAI/litellm/pull/18828)
- Proactive RDS IAM token refresh to prevent 15-min connection failed - [PR #18795](https://github.com/BerriAI/litellm/pull/18795)
- Clarify database_connection_pool_limit applies per worker - [PR #18780](https://github.com/BerriAI/litellm/pull/18780)
- Make base_connection_pool_limit default value the same - [PR #18721](https://github.com/BerriAI/litellm/pull/18721)
- **Docker**
- Add libsndfile to database Docker image for audio processing - [PR #18612](https://github.com/BerriAI/litellm/pull/18612)
- Add line_profiler support for performance analysis and fix Windows CRLF issues - [PR #18773](https://github.com/BerriAI/litellm/pull/18773)
- **Helm**
- Add lifecycle support to Helm charts - [PR #18517](https://github.com/BerriAI/litellm/pull/18517)
- **Authentication**
- Add Kubernetes ServiceAccount JWT authentication support - [PR #18055](https://github.com/BerriAI/litellm/pull/18055)
- Use async anthropic client to prevent event loop blocking - [PR #18435](https://github.com/BerriAI/litellm/pull/18435)
- **Logging Worker**
- Handle event loop changes in multiprocessing - [PR #18423](https://github.com/BerriAI/litellm/pull/18423)
- **Security**
- Prevent expired key plaintext leak in error response - [PR #18860](https://github.com/BerriAI/litellm/pull/18860)
- Mask extra header secrets in model info - [PR #18822](https://github.com/BerriAI/litellm/pull/18822)
- Prevent duplicate User-Agent tags in request_tags - [PR #18723](https://github.com/BerriAI/litellm/pull/18723)
- Properly use litellm api keys - [PR #18832](https://github.com/BerriAI/litellm/pull/18832)
- **Misc**
- Remove double imports in main.py - [PR #18406](https://github.com/BerriAI/litellm/pull/18406)
- Add LITELLM_DISABLE_LAZY_LOADING env var to fix VCR cassette creation issue - [PR #18725](https://github.com/BerriAI/litellm/pull/18725)
- Add xiaomi_mimo to LlmProviders enum to fix router support - [PR #18819](https://github.com/BerriAI/litellm/pull/18819)
- Allow installation with current grpcio on old Python - [PR #18473](https://github.com/BerriAI/litellm/pull/18473)
- Add Custom CA certificates to boto3 clients - [PR #18852](https://github.com/BerriAI/litellm/pull/18852)
- Fix bedrock_cache, metadata and max_model_budget - [PR #18872](https://github.com/BerriAI/litellm/pull/18872)
- Fix LiteLLM SDK embedding headers missing field - [PR #18844](https://github.com/BerriAI/litellm/pull/18844)
- Put automatic reasoning summary inclusion behind feat flag - [PR #18688](https://github.com/BerriAI/litellm/pull/18688)
- turn_off_message_logging Does Not Redact Request Messages in proxy_server_request Field - [PR #18897](https://github.com/BerriAI/litellm/pull/18897)
---
## Documentation Updates
- **Provider Documentation**
- Update MiniMax docs to be in proper format - [PR #18403](https://github.com/BerriAI/litellm/pull/18403)
- Add docs for 5 AI providers - [PR #18388](https://github.com/BerriAI/litellm/pull/18388)
- Fix gpt-5-mini reasoning_effort supported values - [PR #18346](https://github.com/BerriAI/litellm/pull/18346)
- Fix PDF documentation inconsistency in Anthropic page - [PR #18816](https://github.com/BerriAI/litellm/pull/18816)
- Update OpenRouter docs to include embedding support - [PR #18874](https://github.com/BerriAI/litellm/pull/18874)
- Add LITELLM_REASONING_AUTO_SUMMARY in doc - [PR #18705](https://github.com/BerriAI/litellm/pull/18705)
- **MCP Documentation**
- Agentcore MCP server docs - [PR #18603](https://github.com/BerriAI/litellm/pull/18603)
- Mention MCP prompt/resources types in overview - [PR #18669](https://github.com/BerriAI/litellm/pull/18669)
- Add Focus docs - [PR #18837](https://github.com/BerriAI/litellm/pull/18837)
- **Guardrails Documentation**
- Qualifire docs hotfix - [PR #18724](https://github.com/BerriAI/litellm/pull/18724)
- **Infrastructure Documentation**
- IAM Roles Anywhere docs - [PR #18559](https://github.com/BerriAI/litellm/pull/18559)
- Fix formatting in proxy configs documentation - [PR #18498](https://github.com/BerriAI/litellm/pull/18498)
- Fix GCS cache docs missing for proxy mode - [PR #13328](https://github.com/BerriAI/litellm/pull/13328)
- Fix how to execute cloudzero sql - [PR #18841](https://github.com/BerriAI/litellm/pull/18841)
- **General**
- LiteLLM adopters section - [PR #18605](https://github.com/BerriAI/litellm/pull/18605)
- Remove redundant comments about setting litellm.callbacks - [PR #18711](https://github.com/BerriAI/litellm/pull/18711)
- Update header to be markdown bold by removing space - [PR #18846](https://github.com/BerriAI/litellm/pull/18846)
- Manus docs - new provider - [PR #18817](https://github.com/BerriAI/litellm/pull/18817)
---
## New Contributors
* @prasadkona made their first contribution in [PR #18349](https://github.com/BerriAI/litellm/pull/18349)
* @lucasrothman made their first contribution in [PR #18283](https://github.com/BerriAI/litellm/pull/18283)
* @aggeentik made their first contribution in [PR #18317](https://github.com/BerriAI/litellm/pull/18317)
* @mihidumh made their first contribution in [PR #18361](https://github.com/BerriAI/litellm/pull/18361)
* @Prazeina made their first contribution in [PR #18498](https://github.com/BerriAI/litellm/pull/18498)
* @systec-dk made their first contribution in [PR #18500](https://github.com/BerriAI/litellm/pull/18500)
* @xuan07t2 made their first contribution in [PR #18514](https://github.com/BerriAI/litellm/pull/18514)
* @RensDimmendaal made their first contribution in [PR #18190](https://github.com/BerriAI/litellm/pull/18190)
* @yurekami made their first contribution in [PR #18483](https://github.com/BerriAI/litellm/pull/18483)
* @agertz7 made their first contribution in [PR #18556](https://github.com/BerriAI/litellm/pull/18556)
* @yudelevi made their first contribution in [PR #18550](https://github.com/BerriAI/litellm/pull/18550)
* @smallp made their first contribution in [PR #18536](https://github.com/BerriAI/litellm/pull/18536)
* @kevinpauer made their first contribution in [PR #18569](https://github.com/BerriAI/litellm/pull/18569)
* @cansakiroglu made their first contribution in [PR #18517](https://github.com/BerriAI/litellm/pull/18517)
* @dee-walia20 made their first contribution in [PR #18432](https://github.com/BerriAI/litellm/pull/18432)
* @luxinfeng made their first contribution in [PR #18477](https://github.com/BerriAI/litellm/pull/18477)
* @cantalupo555 made their first contribution in [PR #18476](https://github.com/BerriAI/litellm/pull/18476)
* @andersk made their first contribution in [PR #18473](https://github.com/BerriAI/litellm/pull/18473)
* @majiayu000 made their first contribution in [PR #18467](https://github.com/BerriAI/litellm/pull/18467)
* @amangupta-20 made their first contribution in [PR #18529](https://github.com/BerriAI/litellm/pull/18529)
* @hamzaq453 made their first contribution in [PR #18480](https://github.com/BerriAI/litellm/pull/18480)
* @ktsaou made their first contribution in [PR #18627](https://github.com/BerriAI/litellm/pull/18627)
* @FlibbertyGibbitz made their first contribution in [PR #18624](https://github.com/BerriAI/litellm/pull/18624)
* @drorIvry made their first contribution in [PR #18594](https://github.com/BerriAI/litellm/pull/18594)
* @urainshah made their first contribution in [PR #18524](https://github.com/BerriAI/litellm/pull/18524)
* @mangabits made their first contribution in [PR #18279](https://github.com/BerriAI/litellm/pull/18279)
* @0717376 made their first contribution in [PR #18564](https://github.com/BerriAI/litellm/pull/18564)
* @nmgarza5 made their first contribution in [PR #17330](https://github.com/BerriAI/litellm/pull/17330)
* @wileykestner made their first contribution in [PR #18445](https://github.com/BerriAI/litellm/pull/18445)
* @minijeong-log made their first contribution in [PR #14440](https://github.com/BerriAI/litellm/pull/14440)
* @Isaac4real made their first contribution in [PR #18710](https://github.com/BerriAI/litellm/pull/18710)
* @marukaz made their first contribution in [PR #18711](https://github.com/BerriAI/litellm/pull/18711)
* @rohitravirane made their first contribution in [PR #18712](https://github.com/BerriAI/litellm/pull/18712)
* @lizzzcai made their first contribution in [PR #18714](https://github.com/BerriAI/litellm/pull/18714)
* @hkd987 made their first contribution in [PR #18673](https://github.com/BerriAI/litellm/pull/18673)
* @Mr-Pepe made their first contribution in [PR #18674](https://github.com/BerriAI/litellm/pull/18674)
* @gkarthi-signoz made their first contribution in [PR #18726](https://github.com/BerriAI/litellm/pull/18726)
* @Tianduo16 made their first contribution in [PR #18723](https://github.com/BerriAI/litellm/pull/18723)
* @wilsonjr made their first contribution in [PR #18721](https://github.com/BerriAI/litellm/pull/18721)
* @abliteration-ai made their first contribution in [PR #18678](https://github.com/BerriAI/litellm/pull/18678)
* @danialkhan02 made their first contribution in [PR #18770](https://github.com/BerriAI/litellm/pull/18770)
* @ihower made their first contribution in [PR #18409](https://github.com/BerriAI/litellm/pull/18409)
* @elkkhan made their first contribution in [PR #18391](https://github.com/BerriAI/litellm/pull/18391)
* @runixer made their first contribution in [PR #18435](https://github.com/BerriAI/litellm/pull/18435)
* @choby-shun made their first contribution in [PR #18776](https://github.com/BerriAI/litellm/pull/18776)
* @jutaz made their first contribution in [PR #18853](https://github.com/BerriAI/litellm/pull/18853)
* @sjmatta made their first contribution in [PR #18250](https://github.com/BerriAI/litellm/pull/18250)
* @andres-ortizl made their first contribution in [PR #18856](https://github.com/BerriAI/litellm/pull/18856)
* @gauthiermartin made their first contribution in [PR #18844](https://github.com/BerriAI/litellm/pull/18844)
* @mel2oo made their first contribution in [PR #18845](https://github.com/BerriAI/litellm/pull/18845)
* @DominikHallab made their first contribution in [PR #18846](https://github.com/BerriAI/litellm/pull/18846)
* @ji-chuan-che made their first contribution in [PR #18540](https://github.com/BerriAI/litellm/pull/18540)
* @raghav-stripe made their first contribution in [PR #18858](https://github.com/BerriAI/litellm/pull/18858)
* @akraines made their first contribution in [PR #18629](https://github.com/BerriAI/litellm/pull/18629)
* @otaviofbrito made their first contribution in [PR #18665](https://github.com/BerriAI/litellm/pull/18665)
* @chetanchoudhary-sumo made their first contribution in [PR #18587](https://github.com/BerriAI/litellm/pull/18587)
* @pascalwhoop made their first contribution in [PR #13328](https://github.com/BerriAI/litellm/pull/13328)
* @orgersh92 made their first contribution in [PR #18652](https://github.com/BerriAI/litellm/pull/18652)
* @DevajMody made their first contribution in [PR #18497](https://github.com/BerriAI/litellm/pull/18497)
* @matt-greathouse made their first contribution in [PR #18247](https://github.com/BerriAI/litellm/pull/18247)
* @emerzon made their first contribution in [PR #18290](https://github.com/BerriAI/litellm/pull/18290)
* @Eric84626 made their first contribution in [PR #18281](https://github.com/BerriAI/litellm/pull/18281)
* @LukasdeBoer made their first contribution in [PR #18055](https://github.com/BerriAI/litellm/pull/18055)
* @LingXuanYin made their first contribution in [PR #18513](https://github.com/BerriAI/litellm/pull/18513)
* @krisxia0506 made their first contribution in [PR #18698](https://github.com/BerriAI/litellm/pull/18698)
* @LouisShark made their first contribution in [PR #18414](https://github.com/BerriAI/litellm/pull/18414)
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.14.rc.1)**

14
docs/my-website/sidebars.js
View file

@ -55,6 +55,7 @@ const sidebars = {
"proxy/guardrails/test_playground",
"proxy/guardrails/litellm_content_filter",
...[
"proxy/guardrails/qualifire",
"proxy/guardrails/aim_security",
"proxy/guardrails/onyx_security",
"proxy/guardrails/aporia_api",
@ -653,12 +654,13 @@ const sidebars = {
"providers/bedrock_writer",
"providers/bedrock_batches",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/ai21",
"providers/aiml",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.21-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.21.tar.gz vendored Normal file
View file

Binary file not shown.

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.20"
version = "0.4.21"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.20"
version = "0.4.21"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

2
litellm/__init__.py
View file

@ -9,7 +9,7 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
warnings.filterwarnings(
"ignore", message=".*Accessing the.*attribute on the instance is deprecated.*"
)
### INIT VARIABLES #######################
### INIT VARIABLES ########################
import threading
import os
from typing import (

198
litellm/files/main.py
View file

@ -8,6 +8,7 @@ https://platform.openai.com/docs/api-reference/files
import asyncio
import contextvars
import os
import time
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
@ -60,7 +61,7 @@ async def acreate_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -105,7 +106,7 @@ def create_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -274,7 +275,7 @@ def create_file(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai'] are supported.".format(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai', 'manus'] are supported.".format(
custom_llm_provider
),
model="n/a",
@ -293,7 +294,7 @@ def create_file(
@client
async def afile_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -334,7 +335,7 @@ async def afile_retrieve(
@client
def file_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -428,18 +429,60 @@ def file_retrieve(
file_id=file_id,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_retrieve'. Only 'openai' and 'azure' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
# Try using provider config pattern (for Manus, Bedrock, etc.)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict = get_litellm_params(**kwargs)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_retrieve" if _is_async else "file_retrieve",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id") or ""),
)
client = kwargs.get("client")
response = base_llm_http_handler.retrieve_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_retrieve'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
return cast(FileObject, response)
except Exception as e:
@ -450,7 +493,7 @@ def file_retrieve(
@client
async def afile_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -494,7 +537,7 @@ async def afile_delete(
def file_delete(
file_id: str,
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
custom_llm_provider: Union[Literal["openai", "azure", "manus"], str] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -596,18 +639,58 @@ def file_delete(
litellm_params=litellm_params_dict,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
# Try using provider config pattern (for Manus, Bedrock, etc.)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_delete" if _is_async else "file_delete",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id") or ""),
)
response = base_llm_http_handler.delete_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_delete'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
return cast(FileDeleted, response)
except Exception as e:
raise e
@ -616,7 +699,7 @@ def file_delete(
# List files
@client
async def afile_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
purpose: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -657,7 +740,7 @@ async def afile_list(
@client
def file_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
purpose: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -687,7 +770,50 @@ def file_list(
timeout = 600.0
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# Check if provider has a custom files config (e.g., Manus, Bedrock, Vertex AI)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict = get_litellm_params(**kwargs)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_list" if _is_async else "file_list",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id", "")),
)
client = kwargs.get("client")
response = base_llm_http_handler.list_files(
purpose=purpose,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
return response
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
@ -752,7 +878,7 @@ def file_list(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_list'. Only 'openai' and 'azure' are supported.".format(
message="LiteLLM doesn't support {} for 'file_list'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
@ -771,7 +897,7 @@ def file_list(
@client
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -816,7 +942,7 @@ def file_content(
file_id: str,
model: Optional[str] = None,
custom_llm_provider: Optional[
Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"], str]
Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"], str]
] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -977,7 +1103,7 @@ def file_content(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai', 'bedrock'.".format(
message="LiteLLM doesn't support {} for 'file_content'. Supported providers are 'openai', 'azure', 'vertex_ai', 'bedrock', 'manus'.".format(
custom_llm_provider
),
model="n/a",

11
litellm/google_genai/adapters/handler.py
View file

@ -37,9 +37,14 @@ class GenerateContentToCompletionHandler:
completion_kwargs: Dict[str, Any] = dict(completion_request)
# feed metadata for custom callback
if extra_kwargs is not None and "metadata" in extra_kwargs:
completion_kwargs["metadata"] = extra_kwargs["metadata"]
# Forward extra_kwargs that should be passed to completion call
if extra_kwargs is not None:
# Forward metadata for custom callback
if "metadata" in extra_kwargs:
completion_kwargs["metadata"] = extra_kwargs["metadata"]
# Forward extra_headers for providers that require custom headers (e.g., github_copilot)
if "extra_headers" in extra_kwargs:
completion_kwargs["extra_headers"] = extra_kwargs["extra_headers"]
if stream:
completion_kwargs["stream"] = stream

12
litellm/google_genai/main.py
View file

@ -130,6 +130,9 @@ class GenerateContentHelper:
api_key=litellm_params.api_key,
)
if litellm_params.custom_llm_provider is None:
litellm_params.custom_llm_provider = custom_llm_provider
# get provider config
generate_content_provider_config: Optional[
BaseGoogleGenAIGenerateContentConfig
@ -327,6 +330,7 @@ def generate_content(
tools=tools,
_is_async=_is_async,
litellm_params=setup_result.litellm_params,
extra_headers=extra_headers,
**kwargs,
)
@ -407,6 +411,9 @@ async def agenerate_content_stream(
# Check if we should use the adapter (when provider config is None)
if setup_result.generate_content_provider_config is None:
if "stream" in kwargs:
kwargs.pop("stream", None)
# Use the adapter to convert to completion format
return (
await GenerateContentToCompletionHandler.async_generate_content_handler(
@ -416,6 +423,7 @@ async def agenerate_content_stream(
litellm_params=setup_result.litellm_params,
tools=tools,
stream=True,
extra_headers=extra_headers,
**kwargs,
)
)
@ -490,6 +498,9 @@ def generate_content_stream(
# Check if we should use the adapter (when provider config is None)
if setup_result.generate_content_provider_config is None:
if "stream" in kwargs:
kwargs.pop("stream", None)
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
@ -498,6 +509,7 @@ def generate_content_stream(
_is_async=_is_async,
litellm_params=setup_result.litellm_params,
stream=True,
extra_headers=extra_headers,
**kwargs,
)

35
litellm/integrations/braintrust_logging.py
View file

@ -225,10 +225,13 @@ class BraintrustLogger(CustomLogger):
"id": litellm_call_id,
"input": prompt["messages"],
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
@ -351,14 +354,37 @@ class BraintrustLogger(CustomLogger):
# Allow metadata override for span name
span_name = dynamic_metadata.get("span_name", "Chat Completion")
# Span parents is a special case
span_parents = dynamic_metadata.get("span_parents")
# Convert comma-separated string to list if present
if span_parents:
span_parents = [s.strip() for s in span_parents.split(",") if s.strip()]
# Add optional span attributes only if present
span_attributes = {
"span_id": dynamic_metadata.get("span_id"),
"root_span_id": dynamic_metadata.get("root_span_id"),
"span_parents": span_parents,
}
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"output": output,
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
request_data[key] = value
if choices is not None:
request_data["output"] = [choice.dict() for choice in choices]
else:
@ -367,9 +393,6 @@ class BraintrustLogger(CustomLogger):
if metrics is not None:
request_data["metrics"] = metrics
if metrics is not None:
request_data["metrics"] = metrics
try:
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",

182
litellm/integrations/cloudzero/database.py
View file

@ -19,7 +19,7 @@
"""Database connection and data extraction for LiteLLM."""
from datetime import datetime
from typing import Any, Dict, Optional
from typing import Any, Optional, List
import polars as pl
@ -46,19 +46,9 @@ class LiteLLMDatabase:
"""Retrieve usage data from LiteLLM daily user spend table."""
client = self._ensure_prisma_client()
# Build WHERE clause for time filtering
where_conditions = []
if start_time_utc:
where_conditions.append(f"dus.updated_at >= '{start_time_utc.isoformat()}'")
if end_time_utc:
where_conditions.append(f"dus.updated_at <= '{end_time_utc.isoformat()}'")
where_clause = ""
if where_conditions:
where_clause = "WHERE " + " AND ".join(where_conditions)
# Query to get user spend data with team information
query = f"""
# Query to get user spend data with team information. Use parameter binding to
# avoid SQL injection from user-supplied timestamps or limits.
query = """
SELECT
dus.id,
dus.date,
@ -85,163 +75,27 @@ class LiteLLMDatabase:
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
{where_clause}
WHERE ($1::timestamptz IS NULL OR dus.updated_at >= $1::timestamptz)
AND ($2::timestamptz IS NULL OR dus.updated_at <= $2::timestamptz)
ORDER BY dus.date DESC, dus.created_at DESC
"""
if limit:
query += f" LIMIT {limit}"
params: List[Any] = [
start_time_utc,
end_time_utc,
]
if limit is not None:
try:
params.append(int(limit))
except (TypeError, ValueError):
raise ValueError("limit must be an integer")
query += " LIMIT $3"
try:
db_response = await client.db.query_raw(query)
db_response = await client.db.query_raw(query, *params)
# Convert the response to polars DataFrame with full schema inference
# This prevents schema mismatch errors when data types vary across rows
return pl.DataFrame(db_response, infer_schema_length=None)
except Exception as e:
raise Exception(f"Error retrieving usage data: {str(e)}")
async def get_table_info(self) -> Dict[str, Any]:
"""Get information about the daily user spend table."""
client = self._ensure_prisma_client()
try:
# Get row count from user spend table
user_count = await self._get_table_row_count("LiteLLM_DailyUserSpend")
# Get column structure from user spend table
query = """
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = 'LiteLLM_DailyUserSpend'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(query)
return {
"columns": columns_response,
"row_count": user_count,
"table_name": "LiteLLM_DailyUserSpend",
}
except Exception as e:
raise Exception(f"Error getting table info: {str(e)}")
async def _get_table_row_count(self, table_name: str) -> int:
"""Get row count from specified table."""
client = self._ensure_prisma_client()
try:
query = f'SELECT COUNT(*) as count FROM "{table_name}"'
response = await client.db.query_raw(query)
if response and len(response) > 0:
return response[0].get("count", 0)
return 0
except Exception:
return 0
async def discover_all_tables(self) -> Dict[str, Any]:
"""Discover all tables in the LiteLLM database and their schemas."""
client = self._ensure_prisma_client()
try:
# Get all LiteLLM tables
litellm_tables_query = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name LIKE 'LiteLLM_%'
ORDER BY table_name;
"""
tables_response = await client.db.query_raw(litellm_tables_query)
table_names = [row["table_name"] for row in tables_response]
# Get detailed schema for each table
tables_info = {}
for table_name in table_names:
# Get column information
columns_query = """
SELECT
column_name,
data_type,
is_nullable,
column_default,
character_maximum_length,
numeric_precision,
numeric_scale,
ordinal_position
FROM information_schema.columns
WHERE table_name = $1
AND table_schema = 'public'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(columns_query, table_name)
# Get primary key information
pk_query = """
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = $1::regclass AND i.indisprimary;
"""
pk_response = await client.db.query_raw(pk_query, f'"{table_name}"')
primary_keys = (
[row["attname"] for row in pk_response] if pk_response else []
)
# Get foreign key information
fk_query = """
SELECT
tc.constraint_name,
kcu.column_name,
ccu.table_name AS foreign_table_name,
ccu.column_name AS foreign_column_name
FROM information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_name = $1;
"""
fk_response = await client.db.query_raw(fk_query, table_name)
foreign_keys = fk_response if fk_response else []
# Get indexes
indexes_query = """
SELECT
i.relname AS index_name,
array_agg(a.attname ORDER BY a.attnum) AS column_names,
ix.indisunique AS is_unique
FROM pg_class t
JOIN pg_index ix ON t.oid = ix.indrelid
JOIN pg_class i ON i.oid = ix.indexrelid
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
WHERE t.relname = $1
AND t.relkind = 'r'
GROUP BY i.relname, ix.indisunique
ORDER BY i.relname;
"""
indexes_response = await client.db.query_raw(indexes_query, table_name)
indexes = indexes_response if indexes_response else []
# Get row count
try:
row_count = await self._get_table_row_count(table_name)
except Exception:
row_count = 0
tables_info[table_name] = {
"columns": columns_response,
"primary_keys": primary_keys,
"foreign_keys": foreign_keys,
"indexes": indexes,
"row_count": row_count,
}
return {
"tables": tables_info,
"table_count": len(table_names),
"table_names": table_names,
}
except Exception as e:
raise Exception(f"Error discovering tables: {str(e)}")

59
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -1,28 +1,37 @@
{
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
}
}
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
},
"qualifire_eval": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.QUALIFIRE_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}"
},
"environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"]
}
}

200
litellm/integrations/prometheus.py
View file

@ -45,6 +45,7 @@ def _get_cached_end_user_id_for_cost_tracking():
global _get_end_user_id_for_cost_tracking
if _get_end_user_id_for_cost_tracking is None:
from litellm.utils import get_end_user_id_for_cost_tracking
_get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking
return _get_end_user_id_for_cost_tracking
@ -238,6 +239,36 @@ class PrometheusLogger(CustomLogger):
),
buckets=LATENCY_BUCKETS,
)
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time spent in request queue before processing starts (seconds)",
labelnames=self.get_labels_for_metric(
"litellm_request_queue_time_seconds"
),
buckets=LATENCY_BUCKETS,
)
# Guardrail metrics
self.litellm_guardrail_latency_metric = self._histogram_factory(
"litellm_guardrail_latency_seconds",
"Latency (seconds) for guardrail execution",
labelnames=["guardrail_name", "status", "error_type", "hook_type"],
buckets=LATENCY_BUCKETS,
)
self.litellm_guardrail_errors_total = self._counter_factory(
"litellm_guardrail_errors_total",
"Total number of errors encountered during guardrail execution",
labelnames=["guardrail_name", "error_type", "hook_type"],
)
self.litellm_guardrail_requests_total = self._counter_factory(
"litellm_guardrail_requests_total",
"Total number of guardrail invocations",
labelnames=["guardrail_name", "status", "hook_type"],
)
# llm api provider budget metrics
self.litellm_provider_remaining_budget_metric = self._gauge_factory(
"litellm_provider_remaining_budget_metric",
@ -330,6 +361,25 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_requests_metric"),
)
# Cache metrics
self.litellm_cache_hits_metric = self._counter_factory(
name="litellm_cache_hits_metric",
documentation="Total number of LiteLLM cache hits",
labelnames=self.get_labels_for_metric("litellm_cache_hits_metric"),
)
self.litellm_cache_misses_metric = self._counter_factory(
name="litellm_cache_misses_metric",
documentation="Total number of LiteLLM cache misses",
labelnames=self.get_labels_for_metric("litellm_cache_misses_metric"),
)
self.litellm_cached_tokens_metric = self._counter_factory(
name="litellm_cached_tokens_metric",
documentation="Total tokens served from LiteLLM cache",
labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
@ -801,7 +851,7 @@ class PrometheusLogger(CustomLogger):
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {})
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
end_user_id = get_end_user_id_for_cost_tracking(
litellm_params, service_type="prometheus"
)
@ -821,20 +871,8 @@ class PrometheusLogger(CustomLogger):
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
# Include top-level metadata fields (excluding nested dictionaries)
# This allows accessing fields like requester_ip_address from top-level metadata
top_level_metadata = standard_logging_payload.get("metadata", {})
top_level_fields: Dict[str, Any] = {}
if isinstance(top_level_metadata, dict):
top_level_fields = {
k: v
for k, v in top_level_metadata.items()
if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts
}
combined_metadata: Dict[str, Any] = {
**top_level_fields, # Include top-level fields first
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
}
@ -951,6 +989,12 @@ class PrometheusLogger(CustomLogger):
kwargs, start_time, end_time, enum_values, output_tokens
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
)
if (
standard_logging_payload["stream"] is True
): # log successful streaming requests from logging event hook.
@ -1020,6 +1064,54 @@ class PrometheusLogger(CustomLogger):
standard_logging_payload["completion_tokens"]
)
def _increment_cache_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
):
"""
Increment cache-related Prometheus metrics based on cache hit/miss status.
Args:
standard_logging_payload: Contains cache_hit field (True/False/None)
enum_values: Label values for Prometheus metrics
"""
cache_hit = standard_logging_payload.get("cache_hit")
# Only track if cache_hit has a definite value (True or False)
if cache_hit is None:
return
if cache_hit is True:
# Increment cache hits counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_hits_metric"
),
enum_values=enum_values,
)
self.litellm_cache_hits_metric.labels(**_labels).inc()
# Increment cached tokens counter
total_tokens = standard_logging_payload.get("total_tokens", 0)
if total_tokens > 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cached_tokens_metric"
),
enum_values=enum_values,
)
self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens)
else:
# cache_hit is False - increment cache misses counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_misses_metric"
),
enum_values=enum_values,
)
self.litellm_cache_misses_metric.labels(**_labels).inc()
async def _increment_remaining_budget_metrics(
self,
user_api_team: Optional[str],
@ -1188,6 +1280,22 @@ class PrometheusLogger(CustomLogger):
total_time_seconds
)
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
queue_time_seconds = _litellm_params.get("metadata", {}).get(
"queue_time_seconds"
)
if queue_time_seconds is not None and queue_time_seconds >= 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_request_queue_time_seconds"
),
enum_values=enum_values,
)
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
from litellm.types.utils import StandardLoggingPayload
@ -1208,7 +1316,7 @@ class PrometheusLogger(CustomLogger):
litellm_params = kwargs.get("litellm_params", {}) or {}
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
end_user_id = get_end_user_id_for_cost_tracking(
litellm_params, service_type="prometheus"
)
@ -1562,7 +1670,6 @@ class PrometheusLogger(CustomLogger):
api_provider=llm_provider or "",
)
if exception is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_failure_responses"
@ -1595,12 +1702,11 @@ class PrometheusLogger(CustomLogger):
enum_values: UserAPIKeyLabelValues,
output_tokens: float = 1.0,
):
try:
verbose_logger.debug("setting remaining tokens requests metric")
standard_logging_payload: Optional[StandardLoggingPayload] = (
request_kwargs.get("standard_logging_object")
)
standard_logging_payload: Optional[
StandardLoggingPayload
] = request_kwargs.get("standard_logging_object")
if standard_logging_payload is None:
return
@ -1743,6 +1849,50 @@ class PrometheusLogger(CustomLogger):
)
return
def _record_guardrail_metrics(
self,
guardrail_name: str,
latency_seconds: float,
status: str,
error_type: Optional[str],
hook_type: str,
):
"""
Record guardrail metrics for prometheus.
Args:
guardrail_name: Name of the guardrail
latency_seconds: Execution latency in seconds
status: "success" or "error"
error_type: Type of error if any, None otherwise
hook_type: "pre_call", "during_call", or "post_call"
"""
try:
# Record latency
self.litellm_guardrail_latency_metric.labels(
guardrail_name=guardrail_name,
status=status,
error_type=error_type or "none",
hook_type=hook_type,
).observe(latency_seconds)
# Record request count
self.litellm_guardrail_requests_total.labels(
guardrail_name=guardrail_name,
status=status,
hook_type=hook_type,
).inc()
# Record error count if there was an error
if status == "error" and error_type:
self.litellm_guardrail_errors_total.labels(
guardrail_name=guardrail_name,
error_type=error_type,
hook_type=hook_type,
).inc()
except Exception as e:
verbose_logger.debug(f"Error recording guardrail metrics: {str(e)}")
@staticmethod
def _get_exception_class_name(exception: Exception) -> str:
exception_class_name = ""
@ -2380,10 +2530,10 @@ class PrometheusLogger(CustomLogger):
from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
prometheus_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
)
prometheus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
)
# we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers))
@ -2455,7 +2605,7 @@ def prometheus_label_factory(
if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
filtered_labels["end_user"] = get_end_user_id_for_cost_tracking(
litellm_params={"user_api_key_end_user_id": enum_values.end_user},
service_type="prometheus",

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4839,9 +4839,9 @@ class StandardLoggingPayloadSetup:
metadata = litellm_params.get("metadata") or {}
litellm_metadata = litellm_params.get("litellm_metadata") or {}
if metadata.get("tags", []):
request_tags = metadata.get("tags", [])
request_tags = metadata.get("tags", []).copy()
elif litellm_metadata.get("tags", []):
request_tags = litellm_metadata.get("tags", [])
request_tags = litellm_metadata.get("tags", []).copy()
else:
request_tags = []
user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags(

11
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -485,9 +485,14 @@ def _calculate_input_cost(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### IMAGE TOKEN COST (for gpt-image-1 and similar models)
### IMAGE TOKEN COST
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
if model_info.get(image_token_cost_key) is None:
image_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"]
model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
@ -521,7 +526,7 @@ def _calculate_input_cost(
return prompt_cost
def generic_cost_per_token(
def generic_cost_per_token( # noqa: PLR0915
model: str,
usage: Usage,
custom_llm_provider: str,

102
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -95,7 +95,9 @@ def handle_messages_with_content_list_to_str_conversion(
return messages
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[str] = ["user"]) -> AllMessageValues:
def strip_name_from_message(
message: AllMessageValues, allowed_name_roles: List[str] = ["user"]
) -> AllMessageValues:
"""
Removes 'name' from message
"""
@ -104,6 +106,7 @@ def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[
msg_copy.pop("name", None) # type: ignore
return msg_copy
def strip_name_from_messages(
messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"]
) -> List[AllMessageValues]:
@ -444,7 +447,7 @@ def update_responses_input_with_model_file_ids(
"""
Updates responses API input with provider-specific file IDs.
File IDs are always inside the content array, not as direct input_file items.
For managed files (unified file IDs), decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly.
"""
@ -452,25 +455,28 @@ def update_responses_input_with_model_file_ids(
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
if isinstance(input, str):
return input
if not isinstance(input, list):
return input
updated_input = []
for item in input:
if not isinstance(item, dict):
updated_input.append(item)
continue
updated_item = item.copy()
content = item.get("content")
if isinstance(content, list):
updated_content = []
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
if (
isinstance(content_item, dict)
and content_item.get("type") == "input_file"
):
file_id = content_item.get("file_id")
if file_id:
# Check if this is a managed file ID (base64-encoded unified file ID)
@ -478,7 +484,9 @@ def update_responses_input_with_model_file_ids(
if is_unified_file_id:
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
else:
# Fallback: keep original if we can't extract
provider_file_id = file_id
@ -492,9 +500,9 @@ def update_responses_input_with_model_file_ids(
else:
updated_content.append(content_item)
updated_item["content"] = updated_content
updated_input.append(updated_item)
return updated_input
@ -697,9 +705,9 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]:
video/flv
"""
from urllib.parse import urlparse
url = url.lower()
# Parse URL to extract path without query parameters
# This handles URLs like: https://example.com/image.jpg?signature=...
parsed = urlparse(url)
@ -744,28 +752,28 @@ def infer_content_type_from_url_and_content(
) -> str:
"""
Infer content type from URL extension and binary content when content-type header is missing or generic.
This helper implements a fallback strategy for determining MIME types when HTTP headers
are missing or provide generic values (like binary/octet-stream). It's commonly used
when processing images and documents from various sources (S3, URLs, etc.).
Fallback Strategy:
1. If current_content_type is valid (not None and not generic octet-stream), return it
2. Try to infer from URL extension (handles query parameters)
3. Try to detect from binary content signature (magic bytes)
4. Raise ValueError if all methods fail
Args:
url: The URL of the content (used to extract file extension)
content: The binary content (first ~100 bytes are sufficient for detection)
current_content_type: The current content-type from headers (may be None or generic)
Returns:
str: The inferred MIME type (e.g., "image/png", "application/pdf")
Raises:
ValueError: If content type cannot be determined by any method
Example:
>>> content_type = infer_content_type_from_url_and_content(
... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123",
@ -776,14 +784,14 @@ def infer_content_type_from_url_and_content(
"image/png"
"""
from litellm.litellm_core_utils.token_counter import get_image_type
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types
extension_to_mime = {
@ -804,14 +812,14 @@ def infer_content_type_from_url_and_content(
"txt": "text/plain",
"md": "text/markdown",
}
# Try to infer from URL extension
if url:
extension = url.split(".")[-1].lower().split("?")[0] # Remove query params
inferred_type = extension_to_mime.get(extension)
if inferred_type:
return inferred_type
# Try to detect from binary content signature (magic bytes)
if content:
detected_type = get_image_type(content[:100])
@ -825,7 +833,7 @@ def infer_content_type_from_url_and_content(
}
if detected_type in type_to_mime:
return type_to_mime[detected_type]
# If all fallbacks failed, raise error
raise ValueError(
f"Unable to determine content type from URL: {url}. "
@ -1085,7 +1093,9 @@ def _parse_content_for_reasoning(
return None, message_text
reasoning_match = re.match(
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)", message_text, re.DOTALL
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)",
message_text,
re.DOTALL,
)
if reasoning_match:
@ -1135,3 +1145,47 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
elif isinstance(image_url, dict) and "url" in image_url:
images.append(_extract_base64_data(image_url["url"]))
return images
def parse_tool_call_arguments(
arguments: Optional[str],
tool_name: Optional[str] = None,
context: Optional[str] = None,
) -> Dict[str, Any]:
"""
Parse tool call arguments from a JSON string.
This function handles malformed JSON gracefully by raising a ValueError
with context about what failed and what the problematic input was.
Args:
arguments: The JSON string containing tool arguments, or None.
tool_name: Optional name of the tool (for error messages).
context: Optional context string (e.g., "Anthropic Messages API").
Returns:
Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty.
Raises:
ValueError: If the arguments string is not valid JSON.
"""
import json
if not arguments:
return {}
try:
return json.loads(arguments)
except json.JSONDecodeError as e:
error_parts = ["Failed to parse tool call arguments"]
if tool_name:
error_parts.append(f"for tool '{tool_name}'")
if context:
error_parts.append(f"({context})")
error_message = (
" ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}"
)
raise ValueError(error_message) from e

171
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -44,6 +44,7 @@ from .common_utils import (
convert_content_list_to_str,
infer_content_type_from_url_and_content,
is_non_content_values_set,
parse_tool_call_arguments,
)
from .image_handling import convert_url_to_base64
@ -911,13 +912,13 @@ def convert_to_anthropic_image_obj(
def create_anthropic_image_param(
image_url_input: Union[str, dict],
image_url_input: Union[str, dict],
format: Optional[str] = None,
is_bedrock_invoke: bool = False
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
# Extract URL and format from input
@ -927,7 +928,7 @@ def create_anthropic_image_param(
image_url = image_url_input.get("url", "")
if format is None:
format = image_url_input.get("format")
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
@ -1031,9 +1032,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
tool_function = get_attribute_or_key(tool, "function")
tool_name = get_attribute_or_key(tool_function, "name")
tool_arguments = get_attribute_or_key(tool_function, "arguments")
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
)
parameters = "".join(
f"<{param}>{val}</{param}>\n"
for param, val in json.loads(tool_arguments).items()
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
)
invokes += (
"<invoke>\n"
@ -1071,8 +1074,14 @@ def anthropic_messages_pt_xml(messages: list):
if isinstance(messages[msg_i]["content"], list):
for m in messages[msg_i]["content"]:
if m.get("type", "") == "image_url":
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
image_param = create_anthropic_image_param(m["image_url"], format=format)
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
image_param = create_anthropic_image_param(
m["image_url"], format=format
)
# Convert to dict format for XML version
source = image_param["source"]
if isinstance(source, dict) and source.get("type") == "url":
@ -1381,10 +1390,10 @@ def convert_to_gemini_tool_call_invoke(
if tool_calls is not None:
for idx, tool in enumerate(tool_calls):
if "function" in tool:
gemini_function_call: Optional[
VertexFunctionCall
] = _gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
gemini_function_call: Optional[VertexFunctionCall] = (
_gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
)
)
if gemini_function_call is not None:
part_dict: VertexPartType = {
@ -1484,10 +1493,10 @@ def convert_to_gemini_tool_call_result(
}
"""
from litellm.types.llms.vertex_ai import BlobType
content_str: str = ""
inline_data: Optional[BlobType] = None
if "content" in message:
if isinstance(message["content"], str):
content_str = message["content"]
@ -1500,15 +1509,21 @@ def convert_to_gemini_tool_call_result(
elif content_type in ("input_image", "image_url"):
# Extract image for inline_data (for Computer Use screenshots and tool results)
image_url_data = content.get("image_url", "")
image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data
image_url = (
image_url_data.get("url", "")
if isinstance(image_url_data, dict)
else image_url_data
)
if image_url:
# Convert image to base64 blob format for Gemini
try:
image_obj = convert_to_anthropic_image_obj(image_url, format=None)
image_obj = convert_to_anthropic_image_obj(
image_url, format=None
)
inline_data = BlobType(
data=image_obj["data"],
mime_type=image_obj["media_type"]
mime_type=image_obj["media_type"],
)
except Exception as e:
verbose_logger.warning(
@ -1541,6 +1556,7 @@ def convert_to_gemini_tool_call_result(
response_data: dict
try:
import json
if content_str.strip().startswith("{") or content_str.strip().startswith("["):
# Try to parse as JSON (for Computer Use structured responses)
parsed = json.loads(content_str)
@ -1553,7 +1569,7 @@ def convert_to_gemini_tool_call_result(
except (json.JSONDecodeError, ValueError):
# Not valid JSON, wrap in content field
response_data = {"content": content_str}
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
_function_response = VertexFunctionResponse(
@ -1562,7 +1578,7 @@ def convert_to_gemini_tool_call_result(
# Create part with function_response, and optionally inline_data for images (Computer Use)
_part: VertexPartType = {"function_response": _function_response}
# For Computer Use, if we have an image, we need separate parts:
# - One part with function_response
# - One part with inline_data
@ -1570,19 +1586,19 @@ def convert_to_gemini_tool_call_result(
if inline_data:
image_part: VertexPartType = {"inline_data": inline_data}
return [_part, image_part]
return _part
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
This function replaces any invalid characters with underscores.
"""
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
sanitized = re.sub(r'[^a-zA-Z0-9_-]', '_', tool_use_id)
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
# Ensure it's not empty (fallback to a default if needed)
if not sanitized:
sanitized = "tool_use_id"
@ -1644,10 +1660,19 @@ def convert_to_anthropic_tool_result(
)
)
elif content["type"] == "image_url":
format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None
anthropic_content_list.append(
create_anthropic_image_param(content["image_url"], format=format)
format = (
content["image_url"].get("format")
if isinstance(content["image_url"], dict)
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(_anthropic_image_param)
anthropic_content = anthropic_content_list
anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@ -1662,7 +1687,9 @@ def convert_to_anthropic_tool_result(
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if message["role"] == "function":
@ -1671,7 +1698,9 @@ def convert_to_anthropic_tool_result(
# Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if anthropic_tool_result is None:
@ -1687,12 +1716,17 @@ def convert_function_to_anthropic_tool_invoke(
try:
_name = get_attribute_or_key(function_call, "name") or ""
_arguments = get_attribute_or_key(function_call, "arguments")
tool_input = parse_tool_call_arguments(
_arguments, tool_name=_name, context="Anthropic function to tool invoke"
)
anthropic_tool_invoke = [
AnthropicMessagesToolUseParam(
type="tool_use",
id=str(uuid.uuid4()),
name=_name,
input=json.loads(_arguments) if _arguments else {},
input=tool_input,
)
]
return anthropic_tool_invoke
@ -1746,7 +1780,9 @@ def convert_to_anthropic_tool_invoke(
Fixes: https://github.com/BerriAI/litellm/issues/17737
"""
anthropic_tool_invoke: List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]] = []
anthropic_tool_invoke: List[
Union[AnthropicMessagesToolUseParam, Dict[str, Any]]
] = []
for tool in tool_calls:
if not get_attribute_or_key(tool, "type") == "function":
@ -1757,10 +1793,10 @@ def convert_to_anthropic_tool_invoke(
str,
get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
)
tool_input = json.loads(
get_attribute_or_key(
get_attribute_or_key(tool, "function"), "arguments"
)
tool_input = parse_tool_call_arguments(
get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"),
tool_name=tool_name,
context="Anthropic tool invoke",
)
# Check if this is a server-side tool (web_search, tool_search, etc.)
@ -2012,11 +2048,17 @@ def anthropic_messages_pt( # noqa: PLR0915
for m in user_message_types_block["content"]:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
# Convert ChatCompletionImageUrlObject to dict if needed
image_url_value = m["image_url"]
if isinstance(image_url_value, str):
image_url_input: Union[str, dict[str, Any]] = image_url_value
image_url_input: Union[str, dict[str, Any]] = (
image_url_value
)
else:
# ChatCompletionImageUrlObject or dict case - convert to dict
image_url_input = {
@ -2026,20 +2068,26 @@ def anthropic_messages_pt( # noqa: PLR0915
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
is_vertex_ai = (
llm_provider.startswith("vertex_ai")
if llm_provider
else False
)
force_base64 = is_bedrock_invoke or is_vertex_ai
_anthropic_content_element = create_anthropic_image_param(
image_url_input, format=format, is_bedrock_invoke=force_base64
)
image_url_input,
format=format,
is_bedrock_invoke=force_base64,
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_element,
original_content_element=dict(m),
)
if "cache_control" in _content_element:
_anthropic_content_element[
"cache_control"
] = _content_element["cache_control"]
_anthropic_content_element["cache_control"] = (
_content_element["cache_control"]
)
user_content.append(_anthropic_content_element)
elif m.get("type", "") == "text":
m = cast(ChatCompletionTextObject, m)
@ -2077,9 +2125,9 @@ def anthropic_messages_pt( # noqa: PLR0915
)
if "cache_control" in _content_element:
_anthropic_content_text_element[
"cache_control"
] = _content_element["cache_control"]
_anthropic_content_text_element["cache_control"] = (
_content_element["cache_control"]
)
user_content.append(_anthropic_content_text_element)
@ -2175,18 +2223,27 @@ def anthropic_messages_pt( # noqa: PLR0915
): # support assistant tool invoke conversion
# Get web_search_results from provider_specific_fields for server_tool_use reconstruction
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields_raw = assistant_content_block.get(
"provider_specific_fields"
)
_provider_specific_fields: Dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(Dict[str, Any], _provider_specific_fields_raw)
_web_search_results = _provider_specific_fields.get("web_search_results")
_provider_specific_fields = cast(
Dict[str, Any], _provider_specific_fields_raw
)
_web_search_results = _provider_specific_fields.get(
"web_search_results"
)
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
)
# AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
assistant_content.extend(
cast(List[AnthropicMessagesAssistantMessageValues], tool_invoke_results)
cast(
List[AnthropicMessagesAssistantMessageValues],
tool_invoke_results,
)
)
assistant_function_call = assistant_content_block.get("function_call")
@ -3249,14 +3306,18 @@ def _convert_to_bedrock_tool_call_result(
"""
-
"""
tool_result_content_blocks:List[BedrockToolResultContentBlock] = []
tool_result_content_blocks: List[BedrockToolResultContentBlock] = []
if isinstance(message["content"], str):
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=message["content"])
)
elif isinstance(message["content"], List):
content_list = message["content"]
for content in content_list:
if content["type"] == "text":
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=content["text"])
)
elif content["type"] == "image_url":
format: Optional[str] = None
if isinstance(content["image_url"], dict):
@ -3264,12 +3325,14 @@ def _convert_to_bedrock_tool_call_result(
format = content["image_url"].get("format")
else:
image_url = content["image_url"]
_block:BedrockContentBlock = BedrockImageProcessor.process_image_sync(
_block: BedrockContentBlock = BedrockImageProcessor.process_image_sync(
image_url=image_url,
format=format,
)
if "image" in _block:
tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_block["image"])
)
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))

13
litellm/llms/anthropic/chat/handler.py
View file

@ -732,6 +732,19 @@ class ModelResponseIterator:
provider_specific_fields["web_search_results"] = (
self.web_search_results
)
elif (
content_block_start["content_block"]["type"]
== "web_fetch_tool_result"
):
# Capture web_fetch_tool_result for multi-turn reconstruction
# The full content comes in content_block_start, not in deltas
# Fixes: https://github.com/BerriAI/litellm/issues/18137
self.web_search_results.append(
content_block_start["content_block"]
)
provider_specific_fields["web_search_results"] = (
self.web_search_results
)
elif type_chunk == "content_block_stop":
ContentBlockStop(**chunk) # type: ignore
# check if tool call content block - only for tool_use and server_tool_use blocks

13
litellm/llms/anthropic/chat/transformation.py
View file

@ -62,6 +62,7 @@ from litellm.utils import (
ModelResponse,
Usage,
add_dummy_tool,
any_assistant_message_has_thinking_blocks,
get_max_tokens,
has_tool_call_blocks,
last_assistant_with_tool_calls_has_no_thinking_blocks,
@ -1013,10 +1014,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Drop thinking param if thinking is enabled but thinking_blocks are missing
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
#
# IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
# If any message has thinking_blocks, we must keep thinking enabled, otherwise
# Anthropic errors with: "When thinking is disabled, an assistant message cannot contain thinking"
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -1162,6 +1169,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is None:
web_search_results = []
web_search_results.append(content)
## WEB FETCH TOOL RESULT - preserve web fetch results for multi-turn conversations
## Fixes: https://github.com/BerriAI/litellm/issues/18137
elif content["type"] == "web_fetch_tool_result":
if web_search_results is None:
web_search_results = []
web_search_results.append(content)
elif content.get("thinking", None) is not None:
if thinking_blocks is None:
thinking_blocks = []

20
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -14,6 +14,10 @@ from typing import (
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
)
from litellm.types.llms.anthropic import (
AllAnthropicToolsValues,
AnthopicMessagesAssistantMessageParam,
@ -425,15 +429,15 @@ class LiteLLMAnthropicMessagesAdapter:
) -> Optional[str]:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int}
OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default'
"""
if not isinstance(thinking, dict):
return None
thinking_type = thinking.get("type", "disabled")
if thinking_type == "disabled":
return None
elif thinking_type == "enabled":
@ -446,7 +450,7 @@ class LiteLLMAnthropicMessagesAdapter:
return "low"
else:
return "minimal"
return None
def translate_anthropic_tool_choice_to_openai(
@ -676,10 +680,10 @@ class LiteLLMAnthropicMessagesAdapter:
type="tool_use",
id=tool_call.id,
name=tool_call.function.name or "",
input=(
json.loads(tool_call.function.arguments)
if tool_call.function.arguments
else {}
input=parse_tool_call_arguments(
tool_call.function.arguments,
tool_name=tool_call.function.name,
context="Anthropic pass-through adapter",
),
)
# Add provider_specific_fields if signature is present

19
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -25,7 +25,24 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
return "gpt-5" in model or "gpt5_series" in model
def get_supported_openai_params(self, model: str) -> List[str]:
return OpenAIGPT5Config.get_supported_openai_params(self, model=model)
"""Get supported parameters for Azure OpenAI GPT-5 models.
Azure OpenAI GPT-5.2 models support logprobs, unlike OpenAI's GPT-5.
This overrides the parent class to add logprobs support back for gpt-5.2.
Reference:
- Tested with Azure OpenAI GPT-5.2 (api-version: 2025-01-01-preview)
- Azure returns logprobs successfully despite Microsoft's general
documentation stating reasoning models don't support it.
"""
params = OpenAIGPT5Config.get_supported_openai_params(self, model=model)
# Only gpt-5.2 has been verified to support logprobs on Azure
if self.is_model_gpt_5_2_model(model):
azure_supported_params = ["logprobs", "top_logprobs"]
params.extend(azure_supported_params)
return params
def map_openai_params(
self,

4
litellm/llms/base_llm/chat/transformation.py
View file

@ -132,10 +132,10 @@ class BaseConfig(ABC):
Checks 'non_default_params' for 'thinking' and 'max_tokens'
if 'thinking' is enabled and 'max_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
if 'thinking' is enabled and 'max_tokens' or 'max_completion_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
"""
is_thinking_enabled = self.is_thinking_enabled(optional_params)
if is_thinking_enabled and "max_tokens" not in non_default_params:
if is_thinking_enabled and ("max_tokens" not in non_default_params and "max_completion_tokens" not in non_default_params):
thinking_token_budget = cast(dict, optional_params["thinking"]).get(
"budget_tokens", None
)

93
litellm/llms/base_llm/files/transformation.py
View file

@ -2,11 +2,14 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from openai.types.file_deleted import FileDeleted
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.files import TwoStepFileUploadConfig
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileContentRequest,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
@ -75,7 +78,15 @@ class BaseFilesConfig(BaseConfig):
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
) -> Union[dict, str, bytes]:
) -> Union[dict, str, bytes, "TwoStepFileUploadConfig"]:
"""
Transform OpenAI-style file creation request into provider-specific format.
Returns:
- dict: For pre-signed single-step uploads (e.g., Bedrock S3)
- str/bytes: For traditional file uploads
- TwoStepFileUploadConfig: For two-step upload process (e.g., Manus, GCS)
"""
pass
@abstractmethod
@ -88,6 +99,86 @@ class BaseFilesConfig(BaseConfig):
) -> OpenAIFileObject:
pass
@abstractmethod
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file retrieve request into provider-specific format."""
pass
@abstractmethod
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""Transform file retrieve response into OpenAI format."""
pass
@abstractmethod
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file delete request into provider-specific format."""
pass
@abstractmethod
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> "FileDeleted":
"""Transform file delete response into OpenAI format."""
pass
@abstractmethod
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file list request into provider-specific format."""
pass
@abstractmethod
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
"""Transform file list response into OpenAI format."""
pass
@abstractmethod
def transform_file_content_request(
self,
file_content_request: "FileContentRequest",
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file content request into provider-specific format."""
pass
@abstractmethod
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> "HttpxBinaryResponseContent":
"""Transform file content response into OpenAI format."""
pass
def transform_request(
self,
model: str,

68
litellm/llms/bedrock/base_aws_llm.py
View file

@ -74,6 +74,41 @@ class BaseAWSLLM:
"aws_external_id",
]
def _get_ssl_verify(self):
"""
Get SSL verification setting for boto3 clients.
This ensures that custom CA certificates are properly used for all AWS API calls,
including STS and Bedrock services.
Returns:
Union[bool, str]: SSL verification setting - False to disable, True to enable,
or a string path to a CA bundle file
"""
import litellm
from litellm.secret_managers.main import str_to_bool
# Check environment variable first (highest priority)
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
# Convert string "False"/"True" to boolean
if isinstance(ssl_verify, str):
# Check if it's a file path
if os.path.exists(ssl_verify):
return ssl_verify
# Otherwise try to convert to boolean
ssl_verify_bool = str_to_bool(ssl_verify)
if ssl_verify_bool is not None:
ssl_verify = ssl_verify_bool
# Check SSL_CERT_FILE environment variable for custom CA bundle
if ssl_verify is True or ssl_verify == "True":
ssl_cert_file = os.getenv("SSL_CERT_FILE")
if ssl_cert_file and os.path.exists(ssl_cert_file):
return ssl_cert_file
return ssl_verify
def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
"""
Generate a unique cache key based on the credential arguments.
@ -314,6 +349,12 @@ class BaseAWSLLM:
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model = model.split(".")[0]
if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
@ -323,13 +364,9 @@ class BaseAWSLLM:
if provider is not None:
return provider
# check if provider == "nova"
if "nova" in model:
return "nova"
else:
for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider
for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider
return None
@staticmethod
@ -364,7 +401,7 @@ class BaseAWSLLM:
elif provider == "qwen3" and "qwen3/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="qwen3"
)
)
elif provider == "stability" and "stability/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="stability"
@ -416,7 +453,7 @@ class BaseAWSLLM:
if "nova" in model.lower():
if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova")
# Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
if "." in model:
parts = model.split(".")
@ -567,6 +604,7 @@ class BaseAWSLLM:
"sts",
region_name=aws_region_name,
endpoint_url=sts_endpoint,
verify=self._get_ssl_verify(),
)
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
@ -623,7 +661,7 @@ class BaseAWSLLM:
# Create an STS client without credentials
with tracer.trace("boto3.client(sts) for manual IRSA"):
sts_client = boto3.client("sts", region_name=region)
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
# Manually assume the IRSA role with the session name
verbose_logger.debug(
@ -646,6 +684,7 @@ class BaseAWSLLM:
aws_access_key_id=irsa_creds["AccessKeyId"],
aws_secret_access_key=irsa_creds["SecretAccessKey"],
aws_session_token=irsa_creds["SessionToken"],
verify=self._get_ssl_verify(),
)
# Get current caller identity for debugging
@ -684,7 +723,7 @@ class BaseAWSLLM:
verbose_logger.debug("Same account role assumption, using automatic IRSA")
with tracer.trace("boto3.client(sts) with automatic IRSA"):
sts_client = boto3.client("sts", region_name=region)
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
# Get current caller identity for debugging
try:
@ -807,7 +846,7 @@ class BaseAWSLLM:
# This allows the web identity token to work automatically
if aws_access_key_id is None and aws_secret_access_key is None:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client("sts")
sts_client = boto3.client("sts", verify=self._get_ssl_verify())
else:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client(
@ -815,6 +854,7 @@ class BaseAWSLLM:
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
verify=self._get_ssl_verify(),
)
assume_role_params = {
@ -962,7 +1002,9 @@ class BaseAWSLLM:
return endpoint_url, proxy_endpoint_url
def _select_default_endpoint_url(
self, endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], aws_region_name: str
self,
endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]],
aws_region_name: str,
) -> str:
"""
Select the default endpoint url based on the endpoint type

104
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -339,6 +339,55 @@ class AmazonConverseConfig(BaseConfig):
}
}
def _handle_reasoning_effort_parameter(
self, model: str, reasoning_effort: str, optional_params: dict
) -> None:
"""
Handle the reasoning_effort parameter based on the model type.
Different model families handle reasoning effort differently:
- GPT-OSS models: Keep reasoning_effort as-is (passed to additionalModelRequestFields)
- Nova Lite 2 models: Transform to reasoningConfig structure
- Other models (Anthropic, etc.): Convert to thinking parameter
Args:
model: The model identifier
reasoning_effort: The reasoning effort value
optional_params: Dictionary of optional parameters to update in-place
Examples:
>>> config = AmazonConverseConfig()
>>> params = {}
>>> config._handle_reasoning_effort_parameter("gpt-oss-model", "high", params)
>>> params
{'reasoning_effort': 'high'}
>>> params = {}
>>> config._handle_reasoning_effort_parameter("amazon.nova-2-lite-v1:0", "high", params)
>>> params
{'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
>>> params = {}
>>> config._handle_reasoning_effort_parameter("anthropic.claude-3", "high", params)
>>> params
{'thinking': {'type': 'enabled', 'budget_tokens': 10000}}
"""
if "gpt-oss" in model:
# GPT-OSS models: keep reasoning_effort as-is
# It will be passed through to additionalModelRequestFields
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_nova_lite_2_model(model):
# Nova Lite 2 models: transform to reasoningConfig
reasoning_config = self._transform_reasoning_effort_to_reasoning_config(
reasoning_effort
)
optional_params.update(reasoning_config)
else:
# Anthropic and other models: convert to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
reasoning_effort
)
def get_supported_openai_params(self, model: str) -> List[str]:
from litellm.utils import supports_function_calling
@ -353,6 +402,7 @@ class AmazonConverseConfig(BaseConfig):
"extra_headers",
"response_format",
"requestMetadata",
"service_tier",
]
if (
@ -657,25 +707,22 @@ class AmazonConverseConfig(BaseConfig):
if param == "thinking":
optional_params["thinking"] = value
elif param == "reasoning_effort" and isinstance(value, str):
if "gpt-oss" in model:
# GPT-OSS models: keep reasoning_effort as-is
# It will be passed through to additionalModelRequestFields
optional_params["reasoning_effort"] = value
elif self._is_nova_lite_2_model(model):
# Nova Lite 2 models: transform to reasoningConfig
reasoning_config = (
self._transform_reasoning_effort_to_reasoning_config(value)
)
optional_params.update(reasoning_config)
else:
# Anthropic and other models: convert to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
value
)
self._handle_reasoning_effort_parameter(
model=model, reasoning_effort=value, optional_params=optional_params
)
if param == "requestMetadata":
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value) # type: ignore
optional_params["requestMetadata"] = value
if param == "service_tier" and isinstance(value, str):
# Map OpenAI service_tier (string) to Bedrock serviceTier (object)
# OpenAI values: "auto", "default", "flex", "priority"
# Bedrock values: "default", "flex", "priority" (no "auto")
bedrock_tier = value
if value == "auto":
bedrock_tier = "default" # Bedrock doesn't support "auto"
if bedrock_tier in ("default", "flex", "priority"):
optional_params["serviceTier"] = {"type": bedrock_tier}
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
# Nova Lite 2 handles token budgeting differently through reasoningConfig
@ -685,10 +732,7 @@ class AmazonConverseConfig(BaseConfig):
)
final_is_thinking_enabled = self.is_thinking_enabled(optional_params)
if (
final_is_thinking_enabled
and "tool_choice" in optional_params
):
if final_is_thinking_enabled and "tool_choice" in optional_params:
tool_choice_block = optional_params["tool_choice"]
if isinstance(tool_choice_block, dict):
if "any" in tool_choice_block or "tool" in tool_choice_block:
@ -912,20 +956,22 @@ class AmazonConverseConfig(BaseConfig):
inference_params = {
k: v for k, v in inference_params.items() if k in total_supported_params
}
# Only set the topK value in for models that support it
additional_request_params.update(
self._handle_top_k_value(model, inference_params)
)
# Filter out internal/MCP-related parameters that shouldn't be sent to the API
# These are LiteLLM internal parameters, not API parameters
additional_request_params = filter_internal_params(additional_request_params)
# Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
# from additional_request_params to prevent JSON serialization errors
# This filters: Exception objects, callable objects (functions), Logging objects, etc.
additional_request_params = filter_exceptions_from_params(additional_request_params)
additional_request_params = filter_exceptions_from_params(
additional_request_params
)
return inference_params, additional_request_params, request_metadata
@ -950,7 +996,10 @@ class AmazonConverseConfig(BaseConfig):
if original_tools:
for tool in original_tools:
tool_type = tool.get("type", "")
if tool_type in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
if tool_type in (
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
):
# Tool search not supported in Converse API - skip it
continue
filtered_tools.append(tool)
@ -1535,6 +1584,13 @@ class AmazonConverseConfig(BaseConfig):
if "trace" in completion_response:
setattr(model_response, "trace", completion_response["trace"])
# Add service_tier if present in Bedrock response
# Map Bedrock serviceTier (object) to OpenAI service_tier (string)
if "serviceTier" in completion_response:
service_tier_block = completion_response["serviceTier"]
if isinstance(service_tier_block, dict) and "type" in service_tier_block:
setattr(model_response, "service_tier", service_tier_block["type"])
return model_response
def get_error_class(

10
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -524,6 +524,12 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
@ -533,10 +539,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
if provider is not None:
return provider
# check if provider == "nova"
if "nova" in model:
return "nova"
for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider

175
litellm/llms/bedrock/common_utils.py
View file

@ -15,7 +15,7 @@ import litellm
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret
@ -132,6 +132,38 @@ def add_custom_header(headers):
return callback
def _get_bedrock_client_ssl_verify() -> Union[bool, str]:
"""
Get SSL verification setting for Bedrock client.
Returns the SSL verification setting which can be:
- True: Use default SSL verification
- False: Disable SSL verification
- str: Path to a custom CA bundle file
"""
from litellm.secret_managers.main import str_to_bool
ssl_verify: Union[bool, str, None] = os.getenv("SSL_VERIFY", litellm.ssl_verify)
# Convert string "False"/"True" to boolean
if isinstance(ssl_verify, str):
# Check if it's a file path
if os.path.exists(ssl_verify):
return ssl_verify # Keep the file path
# Otherwise try to convert to boolean
ssl_verify_bool = str_to_bool(ssl_verify)
if ssl_verify_bool is not None:
ssl_verify = ssl_verify_bool
# Check SSL_CERT_FILE environment variable for custom CA bundle
if ssl_verify is True or ssl_verify == "True":
ssl_cert_file = os.getenv("SSL_CERT_FILE")
if ssl_cert_file and os.path.exists(ssl_cert_file):
return ssl_cert_file
return ssl_verify if ssl_verify is not None else True
def init_bedrock_client(
region_name=None,
aws_access_key_id: Optional[str] = None,
@ -177,8 +209,7 @@ def init_bedrock_client(
aws_web_identity_token,
) = params_to_check
# SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts.
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
ssl_verify = _get_bedrock_client_ssl_verify()
### SET REGION NAME
if region_name:
@ -229,7 +260,7 @@ def init_bedrock_client(
status_code=401,
)
sts_client = boto3.client("sts")
sts_client = boto3.client("sts", verify=ssl_verify)
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
@ -359,6 +390,70 @@ def get_bedrock_tool_name(response_tool_name: str) -> str:
return response_tool_name
# Cache the global regions list at module level
_BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None
def _get_all_bedrock_regions() -> List[str]:
"""Get all Bedrock regions, cached at module level."""
global _BEDROCK_GLOBAL_REGIONS
if _BEDROCK_GLOBAL_REGIONS is None:
_BEDROCK_GLOBAL_REGIONS = AmazonBedrockGlobalConfig().get_all_regions()
return _BEDROCK_GLOBAL_REGIONS
def get_bedrock_cross_region_inference_regions() -> List[str]:
"""Abbreviations of regions AWS Bedrock supports for cross region inference."""
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
def extract_model_name_from_bedrock_arn(model: str) -> str:
"""
Extract the model name from an AWS Bedrock ARN.
Returns the string after the last '/' if 'arn' is in the input string.
"""
if "arn" in model.lower():
return model.split("/")[-1]
return model
def strip_bedrock_routing_prefix(model: str) -> str:
"""Strip LiteLLM routing prefixes from model name."""
for prefix in ["bedrock/", "converse/", "invoke/", "openai/"]:
if model.startswith(prefix):
model = model.split("/", 1)[1]
return model
def get_bedrock_base_model(model: str) -> str:
"""
Get the base model from the given model name.
Handle model names like:
- "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
- "bedrock/converse/model" -> "model"
"""
model = strip_bedrock_routing_prefix(model)
model = extract_model_name_from_bedrock_arn(model)
potential_region = model.split(".", 1)[0]
alt_potential_region = model.split("/", 1)[0]
if potential_region in get_bedrock_cross_region_inference_regions():
return model.split(".", 1)[1]
elif (
alt_potential_region in _get_all_bedrock_regions()
and len(model.split("/", 1)) > 1
):
return model.split("/", 1)[1]
return model
# Import after standalone functions to avoid circular imports
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
class BedrockModelInfo(BaseLLMModelInfo):
global_config = AmazonBedrockGlobalConfig()
all_global_regions = global_config.get_all_regions()
@ -394,76 +489,34 @@ class BedrockModelInfo(BaseLLMModelInfo):
) -> List[str]:
return []
@staticmethod
def extract_model_name_from_arn(model: str) -> str:
def get_token_counter(self) -> Optional[BaseTokenCounter]:
"""
Extract the model name from an AWS Bedrock ARN.
Returns the string after the last '/' if 'arn' is in the input string.
Args:
arn (str): The ARN string to parse
Factory method to create a Bedrock token counter.
Returns:
str: The extracted model name if 'arn' is in the string,
otherwise returns the original string
BedrockTokenCounter instance for this provider.
"""
if "arn" in model.lower():
return model.split("/")[-1]
return model
return BedrockTokenCounter()
@staticmethod
def extract_model_name_from_arn(model: str) -> str:
"""Wrapper for standalone function. See extract_model_name_from_bedrock_arn()."""
return extract_model_name_from_bedrock_arn(model)
@staticmethod
def get_non_litellm_routing_model_name(model: str) -> str:
if model.startswith("bedrock/"):
model = model.split("/", 1)[1]
if model.startswith("converse/"):
model = model.split("/", 1)[1]
if model.startswith("invoke/"):
model = model.split("/", 1)[1]
if model.startswith("openai/"):
model = model.split("/", 1)[1]
return model
"""Wrapper for standalone function. See strip_bedrock_routing_prefix()."""
return strip_bedrock_routing_prefix(model)
@staticmethod
def get_base_model(model: str) -> str:
"""
Get the base model from the given model name.
Handle model names like - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
"""
model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
model = BedrockModelInfo.extract_model_name_from_arn(model)
potential_region = model.split(".", 1)[0]
alt_potential_region = model.split("/", 1)[
0
] # in model cost map we store regional information like `/us-west-2/bedrock-model`
if (
potential_region
in BedrockModelInfo._supported_cross_region_inference_region()
):
return model.split(".", 1)[1]
elif (
alt_potential_region in BedrockModelInfo.all_global_regions
and len(model.split("/", 1)) > 1
):
return model.split("/", 1)[1]
return model
"""Wrapper for standalone function. See get_bedrock_base_model()."""
return get_bedrock_base_model(model)
@staticmethod
def _supported_cross_region_inference_region() -> List[str]:
"""
Abbreviations of regions AWS Bedrock supports for cross region inference
"""
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
"""Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
return get_bedrock_cross_region_inference_regions()
@staticmethod
def get_bedrock_route(

87
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py Normal file
View file

@ -0,0 +1,87 @@
"""
Bedrock Token Counter implementation using the CountTokens API.
"""
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.types.utils import LlmProviders, TokenCountResponse
class BedrockTokenCounter(BaseTokenCounter):
"""Token counter implementation for AWS Bedrock provider using the CountTokens API."""
def should_use_token_counting_api(
self,
custom_llm_provider: Optional[str] = None,
) -> bool:
"""
Returns True if we should use the Bedrock CountTokens API for token counting.
"""
return custom_llm_provider == LlmProviders.BEDROCK.value
async def count_tokens(
self,
model_to_use: str,
messages: Optional[List[Dict[str, Any]]],
contents: Optional[List[Dict[str, Any]]],
deployment: Optional[Dict[str, Any]] = None,
request_model: str = "",
) -> Optional[TokenCountResponse]:
"""
Count tokens using AWS Bedrock's CountTokens API.
This method calls the existing BedrockCountTokensHandler to make an API call
to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting.
Args:
model_to_use: The model identifier
messages: The messages to count tokens for
contents: Alternative content format (not used for Bedrock)
deployment: Deployment configuration containing litellm_params
request_model: The original request model name
Returns:
TokenCountResponse with token count, or None if counting fails
"""
if not messages:
return None
deployment = deployment or {}
litellm_params = deployment.get("litellm_params", {})
# Build request data in the format expected by BedrockCountTokensHandler
request_data = {
"model": model_to_use,
"messages": messages,
}
# Get the resolved model (strip prefixes like bedrock/, converse/, etc.)
resolved_model = get_bedrock_base_model(model_to_use)
try:
handler = BedrockCountTokensHandler()
result = await handler.handle_count_tokens_request(
request_data=request_data,
litellm_params=litellm_params,
resolved_model=resolved_model,
)
# Transform response to TokenCountResponse
if result is not None:
return TokenCountResponse(
total_tokens=result.get("input_tokens", 0),
request_model=request_model,
model_used=model_to_use,
tokenizer_type="bedrock_api",
original_response=result,
)
except Exception as e:
verbose_logger.warning(
f"Error calling Bedrock CountTokens API: {e}, falling back to default tokenizer"
)
return None

20
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -6,10 +6,9 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
from typing import Any, Dict
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@ -70,6 +69,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug(f"Making request to: {endpoint_url}")
# Use existing _sign_request method from BaseAWSLLM
# Extract api_key for bearer token auth if provided
api_key = litellm_params.get("api_key", None)
headers = {"Content-Type": "application/json"}
signed_headers, signed_body = self._sign_request(
service_name="bedrock",
@ -78,6 +79,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
request_data=bedrock_request,
api_base=endpoint_url,
model=resolved_model,
api_key=api_key,
)
async_client = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK)
@ -94,9 +96,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
if response.status_code != 200:
error_text = response.text
verbose_logger.error(f"AWS Bedrock error: {error_text}")
raise HTTPException(
status_code=400,
detail={"error": f"AWS Bedrock error: {error_text}"},
raise BedrockError(
status_code=response.status_code,
message=f"AWS Bedrock error: {error_text}",
)
bedrock_response = response.json()
@ -112,12 +114,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
return final_response
except HTTPException:
# Re-raise HTTP exceptions as-is
except BedrockError:
# Re-raise Bedrock exceptions as-is
raise
except Exception as e:
verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
raise HTTPException(
raise BedrockError(
status_code=500,
detail={"error": f"CountTokens processing error: {str(e)}"},
message=f"CountTokens processing error: {str(e)}",
)

4
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -8,7 +8,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
from typing import Any, Dict, List
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
class BedrockCountTokensConfig(BaseAWSLLM):
@ -141,7 +141,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
Complete endpoint URL for CountTokens API
"""
# Use existing LiteLLM function to get the base model ID (removes region prefix)
model_id = BedrockModelInfo.get_base_model(model)
model_id = get_bedrock_base_model(model)
# Remove bedrock/ prefix if present
if model_id.startswith("bedrock/"):

1
litellm/llms/bedrock/files/handler.py
View file

@ -142,6 +142,7 @@ class BedrockFilesHandler(BaseAWSLLM):
aws_secret_access_key=credentials.secret_key,
aws_session_token=credentials.token,
region_name=aws_region_name,
verify=self._get_ssl_verify(),
)
# Download file from S3

69
litellm/llms/bedrock/files/transformation.py
View file

@ -1,12 +1,14 @@
import json
import os
import time
from litellm._uuid import uuid
from typing import Any, Dict, List, Optional, Tuple, Union
import httpx
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -18,6 +20,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileTypes,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
@ -539,6 +542,70 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
status_code=status_code, message=error_message, headers=headers
)
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file listing")
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("BedrockFilesConfig does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("BedrockFilesConfig does not support file content retrieval")
class BedrockJsonlFilesTransformation:
"""

39
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -24,6 +24,37 @@ class BedrockPassthroughConfig(
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in endpoint
def _encode_model_id_for_endpoint(self, model_id: str) -> str:
"""
Encode model_id (especially ARNs) for use in Bedrock endpoints.
ARNs contain special characters like colons and slashes that need to be
properly URL-encoded when used in HTTP request paths. For example:
arn:aws:bedrock:us-east-1:123:application-inference-profile/abc123
becomes:
arn:aws:bedrock:us-east-1:123:application-inference-profile%2Fabc123
Args:
model_id: The model ID or ARN to encode
Returns:
The encoded model_id suitable for use in endpoint URLs
"""
from litellm.passthrough.utils import CommonUtils
import re
# Create a temporary endpoint with the model_id to check if encoding is needed
temp_endpoint = f"/model/{model_id}/converse"
encoded_temp_endpoint = CommonUtils.encode_bedrock_runtime_modelid_arn(temp_endpoint)
# Extract the encoded model_id from the temporary endpoint
encoded_model_id_match = re.search(r'/model/([^/]+)/', encoded_temp_endpoint)
if encoded_model_id_match:
return encoded_model_id_match.group(1)
else:
# Fallback to original model_id if extraction fails
return model_id
def get_complete_url(
self,
api_base: Optional[str],
@ -53,9 +84,13 @@ class BedrockPassthroughConfig(
# If model_id is provided (e.g., Application Inference Profile ARN), use it in the endpoint
# instead of the translated model name
if model_id is not None:
# Replace the model name in the endpoint with the model_id
import re
endpoint = re.sub(r'model/[^/]+/', f'model/{model_id}/', endpoint)
# Encode the model_id if it's an ARN to properly handle special characters
encoded_model_id = self._encode_model_id_for_endpoint(model_id)
# Replace the model name in the endpoint with the encoded model_id
endpoint = re.sub(r'model/[^/]+/', f'model/{encoded_model_id}/', endpoint)
return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url
def sign_request(

737
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -14,6 +14,7 @@ from typing import (
)
import httpx # type: ignore
from openai.types.file_deleted import FileDeleted
import litellm
import litellm.litellm_core_utils
@ -71,6 +72,7 @@ from litellm.types.containers.main import (
ContainerObject,
DeleteContainerResult,
)
from litellm.types.files import TwoStepFileUploadConfig
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -82,6 +84,7 @@ from litellm.types.llms.anthropic_skills import (
from litellm.types.llms.openai import (
CreateBatchRequest,
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAIFileObject,
ResponseInputParam,
@ -2782,6 +2785,38 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
def _extract_upload_url_from_response(
self,
response: httpx.Response,
upload_url_location: str,
upload_url_key: str = "upload_url",
) -> tuple[Optional[str], Optional[dict]]:
"""
Extract upload URL from initial file creation response.
Args:
response: HTTP response from initial file creation request
upload_url_location: Where to find URL ('headers' or 'body')
upload_url_key: Key name for URL in response body (default: 'upload_url')
Returns:
Tuple of (upload_url, response_data)
- upload_url: The extracted upload URL, or None if not found
- response_data: Parsed response body (for 'body' location), or None
"""
if upload_url_location == "headers":
# Google Cloud Storage style - URL in X-Goog-Upload-URL header
upload_url = response.headers.get("X-Goog-Upload-URL")
return upload_url, None
else:
# Response body style (e.g., Manus, S3 presigned URLs)
try:
response_data = response.json()
upload_url = response_data.get(upload_url_key)
return upload_url, response_data if upload_url else None
except Exception:
return None, None
def create_file(
self,
create_file_data: CreateFileRequest,
@ -2844,14 +2879,58 @@ class BaseLLMHTTPHandler:
else:
sync_httpx_client = client
if isinstance(transformed_request, dict) and "method" in transformed_request:
if isinstance(transformed_request, dict) and "initial_request" in transformed_request:
# Handle two-step uploads (TwoStepFileUploadConfig)
# Used by providers like Manus, Google Cloud Storage
try:
# Step 1: Initial request to get upload URL
initial_response = sync_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response
upload_url, initial_response_data = self._extract_upload_url_from_response(
response=initial_response,
upload_url_location=transformed_request.get("upload_url_location", "headers"),
upload_url_key=transformed_request.get("upload_url_key", "upload_url"),
)
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_method = transformed_request["upload_request"].get("method", "POST").lower()
upload_response = getattr(sync_httpx_client, upload_method)(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
# Store initial response for transformation
if initial_response_data:
litellm_params["initial_file_response"] = initial_response_data
except Exception as e:
raise self._handle_error(
e=e,
provider_config=provider_config,
)
elif isinstance(transformed_request, dict) and "method" in transformed_request and "initial_request" not in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request = cast(Dict[str, Any], transformed_request)
upload_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
sync_httpx_client, presigned_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
url=presigned_request["url"],
headers=presigned_request["headers"],
data=presigned_request["data"],
timeout=timeout,
)
elif isinstance(transformed_request, str) or isinstance(
@ -2879,36 +2958,7 @@ class BaseLLMHTTPHandler:
timeout=timeout,
)
else:
try:
# Step 1: Initial request to get upload URL
initial_response = sync_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response headers
upload_url = initial_response.headers.get("X-Goog-Upload-URL")
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_response = sync_httpx_client.post(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=provider_config,
)
raise ValueError(f"Unsupported transformed_request type: {type(transformed_request)}")
# Store the upload URL in litellm_params for the transformation method
litellm_params_with_url = dict(litellm_params)
@ -2923,7 +2973,7 @@ class BaseLLMHTTPHandler:
async def async_create_file(
self,
transformed_request: Union[bytes, str, dict],
transformed_request: Union[bytes, str, dict, "TwoStepFileUploadConfig"],
litellm_params: dict,
provider_config: BaseFilesConfig,
headers: dict,
@ -2955,14 +3005,59 @@ class BaseLLMHTTPHandler:
},
)
if isinstance(transformed_request, dict) and "method" in transformed_request:
if isinstance(transformed_request, dict) and "initial_request" in transformed_request:
# Handle two-step uploads (TwoStepFileUploadConfig)
# Used by providers like Manus, Google Cloud Storage
try:
# Step 1: Initial request to get upload URL
initial_response = await async_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response
upload_url, initial_response_data = self._extract_upload_url_from_response(
response=initial_response,
upload_url_location=transformed_request.get("upload_url_location", "headers"),
upload_url_key=transformed_request.get("upload_url_key", "upload_url"),
)
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_method = transformed_request["upload_request"].get("method", "POST").lower()
upload_response = await getattr(async_httpx_client, upload_method)(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
# Store initial response for transformation
if initial_response_data:
litellm_params["initial_file_response"] = initial_response_data
except Exception as e:
verbose_logger.exception(f"Error creating file: {e}")
raise self._handle_error(
e=e,
provider_config=provider_config,
)
elif isinstance(transformed_request, dict) and "method" in transformed_request and "initial_request" not in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request = cast(Dict[str, Any], transformed_request)
upload_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
async_httpx_client, presigned_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
url=presigned_request["url"],
headers=presigned_request["headers"],
data=presigned_request["data"],
timeout=timeout,
)
elif isinstance(transformed_request, str) or isinstance(
@ -2990,37 +3085,7 @@ class BaseLLMHTTPHandler:
timeout=timeout,
)
else:
try:
# Step 1: Initial request to get upload URL
initial_response = await async_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response headers
upload_url = initial_response.headers.get("X-Goog-Upload-URL")
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_response = await async_httpx_client.post(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
except Exception as e:
verbose_logger.exception(f"Error creating file: {e}")
raise self._handle_error(
e=e,
provider_config=provider_config,
)
raise ValueError(f"Unsupported transformed_request type: {type(transformed_request)}")
return provider_config.transform_create_file_response(
model=None,
@ -3734,29 +3799,525 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
def list_files(self):
def retrieve_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]:
"""
Lists all files
Retrieve file metadata by ID
"""
pass
if _is_async:
return self.async_retrieve_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
def delete_file(self):
"""
Deletes a file
"""
pass
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
def retrieve_file(self):
"""
Returns the metadata of the file
"""
pass
# Get URL and params from provider config
url, params = provider_config.transform_retrieve_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
def retrieve_file_content(self):
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_retrieve_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_retrieve_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> OpenAIFileObject:
"""
Returns the content of the file
Async retrieve file metadata by ID
"""
pass
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_retrieve_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_retrieve_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def delete_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union["FileDeleted", Coroutine[Any, Any, "FileDeleted"]]:
"""
Delete a file by ID
"""
if _is_async:
return self.async_delete_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_delete_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = sync_httpx_client.delete(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_delete_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_delete_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> "FileDeleted":
"""
Async delete a file by ID
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_delete_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = await async_httpx_client.delete(
url=url, headers=headers, params=params, timeout=timeout
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_delete_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def list_files(
self,
purpose: Optional[str],
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[List[OpenAIFileObject], Coroutine[Any, Any, List[OpenAIFileObject]]]:
"""
List all files
"""
if _is_async:
return self.async_list_files(
purpose=purpose,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_list_files_request(
purpose=purpose,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"purpose": purpose,
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_list_files_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_list_files(
self,
purpose: Optional[str],
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> List[OpenAIFileObject]:
"""
Async list all files
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_list_files_request(
purpose=purpose,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"purpose": purpose,
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_list_files_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def retrieve_file_content(
self,
file_content_request: "FileContentRequest",
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union["HttpxBinaryResponseContent", Coroutine[Any, Any, "HttpxBinaryResponseContent"]]:
"""
Retrieve file content by ID
"""
if _is_async:
return self.async_retrieve_file_content(
file_content_request=file_content_request,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_file_content_request(
file_content_request=file_content_request,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_content_request.get("file_id"),
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_file_content_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_retrieve_file_content(
self,
file_content_request: "FileContentRequest",
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> "HttpxBinaryResponseContent":
"""
Async retrieve file content by ID
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_file_content_request(
file_content_request=file_content_request,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_content_request.get("file_id"),
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_file_content_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def _prepare_fake_stream_request(
self,

1
litellm/llms/gemini/chat/transformation.py
View file

@ -87,6 +87,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"stop",
"logprobs",
"frequency_penalty",
"presence_penalty",
"modalities",
"parallel_tool_calls",
"web_search_options",

66
litellm/llms/gemini/files/transformation.py
View file

@ -7,6 +7,7 @@ import time
from typing import List, Optional
import httpx
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
@ -17,6 +18,7 @@ from litellm.llms.base_llm.files.transformation import (
from litellm.types.llms.gemini import GeminiCreateFilesResponseObject
from litellm.types.llms.openai import (
CreateFileRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
)
@ -171,3 +173,67 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
except Exception as e:
verbose_logger.exception(f"Error parsing file upload response: {str(e)}")
raise ValueError(f"Error parsing file upload response: {str(e)}")
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file listing")
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval")

1
litellm/llms/gemini/google_genai/transformation.py
View file

@ -89,6 +89,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"audio_timestamp",
"automatic_function_calling",
"thinking_config",
"image_config",
]
def map_generate_content_optional_params(

2
litellm/llms/manus/files/__init__.py Normal file
View file

@ -0,0 +1,2 @@
# Manus Files API implementation

439
litellm/llms/manus/files/transformation.py Normal file
View file

@ -0,0 +1,439 @@
"""
Manus Files API implementation.
Manus has an OpenAI-compatible Files API with some differences:
- Uses API_KEY header instead of Authorization: Bearer
- File upload is a two-step process:
1. Create file record to get upload URL
2. Upload file content to the upload URL
Reference: https://open.manus.im/docs/openai-compatibility#file-management
"""
import time
from typing import Any, Dict, List, Optional, Union
import httpx
from openai.types.file_deleted import FileDeleted
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
LiteLLMLoggingObj,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.secret_managers.main import get_secret_str
from litellm.types.files import TwoStepFileUploadConfig, TwoStepFileUploadRequest
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
)
from litellm.types.utils import LlmProviders
MANUS_API_BASE = "https://api.manus.im"
class ManusFilesConfig(BaseFilesConfig):
"""
Configuration for Manus Files API.
Manus uses:
- API_KEY header for authentication (not Authorization: Bearer)
- Two-step file upload process
- Content-Type: application/json for all requests
Reference: https://open.manus.im/docs/openai-compatibility#file-management
"""
def __init__(self):
pass
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.MANUS
def validate_environment(
self,
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for Manus API.
Manus uses API_KEY header instead of Authorization: Bearer.
For file uploads, don't set Content-Type - httpx will set it for multipart.
"""
api_key = (
api_key
or litellm.api_key
or get_secret_str("MANUS_API_KEY")
)
if not api_key:
raise ValueError(
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
)
# Manus uses API_KEY header, not Authorization: Bearer
# Manus requires Content-Type: application/json for all requests (even GET)
headers.update(
{
"API_KEY": api_key,
"Content-Type": "application/json",
}
)
return headers
def get_supported_openai_params(
self, model: str
) -> List[OpenAICreateFileRequestOptionalParams]:
"""
Return supported OpenAI file creation parameters for Manus.
Manus supports the standard 'purpose' parameter.
"""
return ["purpose"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to Manus-specific parameters.
Manus is OpenAI-compatible, so no special mapping needed.
"""
return optional_params
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for Manus Files API endpoint.
Returns:
str: The full URL for the Manus /v1/files endpoint
"""
api_base = (
api_base
or litellm.api_base
or get_secret_str("MANUS_API_BASE")
or MANUS_API_BASE
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
# Manus API uses /v1/files endpoint
if api_base.endswith("/v1"):
return f"{api_base}/files"
return f"{api_base}/v1/files"
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
"""
Return the appropriate error class for Manus API errors.
Uses OpenAIError since Manus is OpenAI-compatible.
"""
return OpenAIError(
status_code=status_code,
message=error_message,
headers=headers,
)
def transform_create_file_request(
self,
model: str,
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
) -> TwoStepFileUploadConfig:
"""
Transform OpenAI-style file creation request into Manus's two-step format.
Manus API spec (https://open.manus.im/docs/openai-compatibility#file-management):
1. POST /v1/files with JSON {"filename": "..."} → returns {"id": "...", "upload_url": "..."}
2. PUT to upload_url with raw file content
"""
# Extract file data
file_data = create_file_data.get("file")
if file_data is None:
raise ValueError("File data is required")
extracted_data = extract_file_data(file_data)
filename = extracted_data["filename"] or f"file_{int(time.time())}"
content = extracted_data["content"]
# Get API base URL
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
)
# Get API key
api_key = (
litellm_params.get("api_key")
or litellm.api_key
or get_secret_str("MANUS_API_KEY")
)
if not api_key:
raise ValueError(
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
)
# Build typed two-step upload config
return TwoStepFileUploadConfig(
initial_request=TwoStepFileUploadRequest(
method="POST",
url=api_base,
headers={
"API_KEY": api_key,
"Content-Type": "application/json",
},
data={"filename": filename},
),
upload_request=TwoStepFileUploadRequest(
method="PUT",
url="", # Will be populated from initial_request response
headers={},
data=content,
),
upload_url_location="body",
upload_url_key="upload_url",
)
def transform_create_file_response(
self,
model: Optional[str],
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""
Transform Manus's file upload response into OpenAI-style FileObject.
For two-step uploads, the handler stores the initial response in litellm_params.
We need to return the file object from the initial POST, not the final PUT.
Manus initial response format:
{
"id": "file-abc123xyz",
"object": "file",
"filename": "document.pdf",
"status": "pending",
"upload_url": "https://...",
"upload_expires_at": "...",
"created_at": "..."
}
"""
try:
# For two-step uploads, get the initial response from litellm_params
initial_response_data = litellm_params.get("initial_file_response")
if initial_response_data:
response_json = initial_response_data
else:
# Log raw response for debugging
verbose_logger.debug(f"Manus raw response text: {raw_response.text}")
response_json = raw_response.json()
verbose_logger.debug(f"Manus file response: {response_json}")
# Parse created_at timestamp
created_at_str = response_json.get("created_at", "")
if created_at_str:
try:
# Try parsing ISO format
created_at = int(
time.mktime(
time.strptime(
created_at_str.replace("Z", "+00:00")[:19],
"%Y-%m-%dT%H:%M:%S",
)
)
)
except (ValueError, TypeError):
created_at = int(time.time())
else:
created_at = int(time.time())
return OpenAIFileObject(
id=response_json.get("id", ""),
bytes=response_json.get("bytes", 0),
created_at=created_at,
filename=response_json.get("filename", ""),
object="file",
purpose=response_json.get("purpose", "assistants"),
status="uploaded", # After successful upload, status is uploaded
status_details=response_json.get("status_details"),
)
except Exception as e:
verbose_logger.exception(f"Error parsing Manus file response: {str(e)}")
raise ValueError(f"Error parsing Manus file response: {str(e)}")
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for retrieving a file."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""Transform retrieve file response."""
return self.transform_create_file_response(
model=None,
raw_response=raw_response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for deleting a file."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
"""Transform delete file response."""
response_json = raw_response.json()
return FileDeleted(**response_json)
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for listing files."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
params = {}
if purpose:
params["purpose"] = purpose
return api_base, params
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
"""Transform list files response."""
response_json = raw_response.json()
files_data = response_json.get("data", [])
return [self._parse_file_dict(f) for f in files_data]
def _parse_file_dict(self, file_dict: Dict[str, Any]) -> OpenAIFileObject:
"""Parse a file dict into OpenAIFileObject."""
created_at_str = file_dict.get("created_at", "")
if created_at_str:
try:
created_at = int(
time.mktime(
time.strptime(
created_at_str.replace("Z", "+00:00")[:19],
"%Y-%m-%dT%H:%M:%S",
)
)
)
except (ValueError, TypeError):
created_at = int(time.time())
else:
created_at = int(time.time())
return OpenAIFileObject(
id=file_dict.get("id", ""),
bytes=file_dict.get("bytes", 0),
created_at=created_at,
filename=file_dict.get("filename", ""),
object="file",
purpose=file_dict.get("purpose", "assistants"),
status=file_dict.get("status", "uploaded"),
status_details=file_dict.get("status_details"),
)
def transform_file_content_request(
self,
file_content_request: FileContentRequest,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for retrieving file content."""
file_id = file_content_request.get("file_id")
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}/content", {}
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
"""Transform file content response."""
return HttpxBinaryResponseContent(response=raw_response)

36
litellm/llms/manus/responses/transformation.py
View file

@ -1,3 +1,4 @@
import uuid
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
import httpx
@ -94,9 +95,11 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
# Manus uses API_KEY header, not Authorization: Bearer
# Content-Type is required for all requests (including GET)
headers.update(
{
"API_KEY": api_key,
"Content-Type": "application/json",
}
)
return headers
@ -164,8 +167,9 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
if extra_body:
base_request.update(extra_body)
# Avoid logging potentially sensitive agent_profile value
verbose_logger.debug("Manus: Using task_mode=agent")
verbose_logger.debug(
f"Manus: Using agent_profile={agent_profile}, task_mode=agent"
)
return base_request
@ -224,6 +228,12 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
total_tokens=0,
)
# Ensure id is present - failed responses may not include it
if "id" not in raw_response_json or raw_response_json.get("id") is None:
# Generate a placeholder id for failed responses
# This allows the response object to be created even when the API doesn't return an id
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
except Exception:
@ -293,6 +303,28 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
raw_response_headers = dict(raw_response.headers)
processed_headers = process_response_headers(raw_response_headers)
# Ensure reasoning, text, output, and usage are present with defaults
if "reasoning" not in raw_response_json or raw_response_json.get("reasoning") is None:
raw_response_json["reasoning"] = {}
if "text" not in raw_response_json or raw_response_json.get("text") is None:
raw_response_json["text"] = {}
if "output" not in raw_response_json or raw_response_json.get("output") is None:
raw_response_json["output"] = []
if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
raw_response_json["usage"] = ResponseAPIUsage(
input_tokens=0,
output_tokens=0,
total_tokens=0,
)
# Ensure id is present - failed responses may not include it
if "id" not in raw_response_json or raw_response_json.get("id") is None:
# Generate a placeholder id for failed responses
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
except Exception:

5
litellm/llms/oci/chat/transformation.py
View file

@ -1124,8 +1124,11 @@ def adapt_messages_to_generic_oci_standard_content_message(
elif type == "image_url":
image_url = content_item.get("image_url")
# Handle both OpenAI format (object with url) and string format
if isinstance(image_url, dict):
image_url = image_url.get("url")
if not isinstance(image_url, str):
raise Exception("Prop `image_url` is not a string")
raise Exception("Prop `image_url` must be a string or an object with a `url` property")
new_content.append(OCIImageContentPart(imageUrl=image_url))
return OCIMessage(

4
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -83,6 +83,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["structured_messages"] = (
messages # pass the openai /chat/completions messages to the guardrail, as-is
)
# Pass tools (function definitions) to the guardrail
tools = data.get("tools")
if tools:
inputs["tools"] = tools
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,

4
litellm/llms/openai_like/providers.json
View file

@ -61,6 +61,10 @@
"max_completion_tokens": "max_tokens"
}
},
"abliteration": {
"base_url": "https://api.abliteration.ai/v1",
"api_key_env": "ABLITERATION_API_KEY"
},
"llamagate": {
"base_url": "https://api.llamagate.dev/v1",
"api_key_env": "LLAMAGATE_API_KEY",

182
litellm/llms/openrouter/embedding/transformation.py Normal file
View file

@ -0,0 +1,182 @@
"""
OpenRouter Embedding API Configuration.
This module provides the configuration for OpenRouter's Embedding API.
OpenRouter is OpenAI-compatible and supports embeddings via the /v1/embeddings endpoint.
Docs: https://openrouter.ai/docs
"""
from typing import TYPE_CHECKING, Any, Optional
import httpx
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.types.llms.openai import AllEmbeddingInputValues
from litellm.types.utils import EmbeddingResponse
from litellm.utils import convert_to_model_response_object
from ..common_utils import OpenRouterException
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OpenrouterEmbeddingConfig(BaseEmbeddingConfig):
"""
Configuration for OpenRouter's Embedding API.
Reference: https://openrouter.ai/docs
"""
def validate_environment(
self,
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for OpenRouter API.
OpenRouter requires:
- Authorization header with Bearer token
- HTTP-Referer header (site URL)
- X-Title header (app name)
"""
from litellm import get_secret
# Get OpenRouter-specific headers
openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_headers = {
"HTTP-Referer": openrouter_site_url,
"X-Title": openrouter_app_name,
"Content-Type": "application/json",
}
# Add Authorization header if api_key is provided
if api_key:
openrouter_headers["Authorization"] = f"Bearer {api_key}"
# Merge with existing headers (user's extra_headers take priority)
merged_headers = {**openrouter_headers, **headers}
return merged_headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for OpenRouter Embedding API endpoint.
"""
# api_base is already set to https://openrouter.ai/api/v1 in main.py
# Remove trailing slashes
if api_base:
api_base = api_base.rstrip("/")
else:
api_base = "https://openrouter.ai/api/v1"
# Return the embeddings endpoint
return f"{api_base}/embeddings"
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
) -> dict:
"""
Transform embedding request to OpenRouter format (OpenAI-compatible).
"""
# Ensure input is a list
if isinstance(input, str):
input = [input]
# OpenRouter expects the full model name (e.g., google/gemini-embedding-001)
# Strip 'openrouter/' prefix if present
if model.startswith("openrouter/"):
model = model.replace("openrouter/", "", 1)
return {
"model": model,
"input": input,
**optional_params,
}
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str],
request_data: dict,
optional_params: dict,
litellm_params: dict,
) -> EmbeddingResponse:
"""
Transform embedding response from OpenRouter format (OpenAI-compatible).
"""
logging_obj.post_call(original_response=raw_response.text)
# OpenRouter returns standard OpenAI-compatible embedding response
response_json = raw_response.json()
return convert_to_model_response_object(
response_object=response_json,
model_response_object=model_response,
response_type="embedding",
)
def get_supported_openai_params(self, model: str) -> list:
"""
Get list of supported OpenAI parameters for OpenRouter embeddings.
"""
return [
"timeout",
"dimensions",
"encoding_format",
"user",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to OpenRouter format.
"""
for param, value in non_default_params.items():
if param in self.get_supported_openai_params(model):
optional_params[param] = value
return optional_params
def get_error_class(
self, error_message: str, status_code: int, headers: Any
) -> Any:
"""
Get the error class for OpenRouter errors.
"""
return OpenRouterException(
message=error_message,
status_code=status_code,
headers=headers,
)

4
litellm/llms/replicate/chat/handler.py
View file

@ -213,7 +213,7 @@ def completion(
response = httpx_client.get(url=prediction_url, headers=headers)
if (
response.status_code == 200
and response.json().get("status") == "processing"
and response.json().get("status") in ["processing", "starting"]
):
continue
return litellm.ReplicateConfig().transform_response(
@ -284,7 +284,7 @@ async def async_completion(
response = await async_handler.get(url=prediction_url, headers=headers)
if (
response.status_code == 200
and response.json().get("status") == "processing"
and response.json().get("status") in ["processing", "starting"]
):
continue
return litellm.ReplicateConfig().transform_response(

68
litellm/llms/vertex_ai/files/transformation.py
View file

@ -1,11 +1,12 @@
import json
import os
import time
from litellm._uuid import uuid
from typing import Any, Dict, List, Optional, Tuple, Union
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from litellm._uuid import uuid
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -24,6 +25,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileTypes,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
@ -333,6 +335,70 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
status_code=status_code, message=error_message, headers=headers
)
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("VertexAIFilesConfig does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file deletion")
def transform_delete_file_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("VertexAIFilesConfig does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file listing")
def transform_list_files_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("VertexAIFilesConfig does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("VertexAIFilesConfig does not support file content retrieval")
class VertexAIJsonlFilesTransformation(VertexGeminiConfig):
"""

212
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -310,9 +310,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"""
return Tools(googleSearch={})
def _transform_computer_use_config(
self, computer_use_config: dict
) -> dict:
def _transform_computer_use_config(self, computer_use_config: dict) -> dict:
"""
Transform Computer Use configuration to Gemini API format.
@ -323,7 +321,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Transformed computer use configuration for Gemini API
"""
transformed_config = {}
# Transform environment values if needed
if "environment" in computer_use_config:
env_value = computer_use_config["environment"]
@ -339,13 +337,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
f"Invalid environment value for computer_use: {env_value}. "
f"Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'"
)
# Transform excluded_predefined_functions to camelCase
if "excluded_predefined_functions" in computer_use_config:
transformed_config["excludedPredefinedFunctions"] = computer_use_config["excluded_predefined_functions"]
transformed_config["excludedPredefinedFunctions"] = computer_use_config[
"excluded_predefined_functions"
]
elif "excludedPredefinedFunctions" in computer_use_config:
transformed_config["excludedPredefinedFunctions"] = computer_use_config["excludedPredefinedFunctions"]
transformed_config["excludedPredefinedFunctions"] = computer_use_config[
"excludedPredefinedFunctions"
]
return transformed_config
def _extract_google_maps_retrieval_config(
@ -446,9 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
value = _remove_strict_from_schema(value)
for tool in value:
openai_function_object: Optional[
ChatCompletionToolParamFunctionChunk
] = None
openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = (
None
)
if "function" in tool: # tools list
_openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore
**tool["function"]
@ -480,20 +482,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
or tool_name == VertexToolName.CODE_EXECUTION.value
): # code_execution maintained for backwards compatibility
code_execution = self.get_tool_value(tool, "codeExecution")
elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value:
googleSearch = self.get_tool_value(
tool, VertexToolName.GOOGLE_SEARCH.value
)
elif (
tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value
elif tool_name and (
tool_name == VertexToolName.GOOGLE_SEARCH.value
or tool_name == "google_search"
):
googleSearchRetrieval = self.get_tool_value(
tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value
)
elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value:
enterpriseWebSearch = self.get_tool_value(
tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value
)
googleSearch = self.get_tool_value(tool, tool_name)
elif tool_name and (
tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value
or tool_name == "google_search_retrieval"
):
googleSearchRetrieval = self.get_tool_value(tool, tool_name)
elif tool_name and (
tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value
or tool_name == "enterprise_web_search"
):
enterpriseWebSearch = self.get_tool_value(tool, tool_name)
elif tool_name and (
tool_name == VertexToolName.URL_CONTEXT.value
or tool_name == "urlContext"
@ -552,7 +555,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request."
)
# Build list of Tool objects - each Tool should contain exactly one type
# Build list of Tool objects - each Tool should contain exactly one type
# per Vertex AI API spec: "A Tool object should contain exactly one type of Tool"
_tools_list: List[Tools] = []
@ -569,11 +572,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
_tools_list.append(search_tool)
if googleSearchRetrieval is not None:
retrieval_tool = Tools()
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = (
googleSearchRetrieval
)
_tools_list.append(retrieval_tool)
if enterpriseWebSearch is not None:
enterprise_tool = Tools()
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = (
enterpriseWebSearch
)
_tools_list.append(enterprise_tool)
if code_execution is not None:
code_tool = Tools()
@ -592,7 +599,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
computer_tool[VertexToolName.COMPUTER_USE.value] = computerUse
_tools_list.append(computer_tool)
# Add retrieval config to toolConfig if googleMaps has location data
if google_maps_retrieval_config is not None:
if "toolConfig" not in optional_params:
@ -709,8 +715,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
GeminiThinkingConfig with thinkingLevel and includeThoughts
"""
# Check if this is gemini-3-flash which supports MINIMAL thinking level
is_gemini3flash= model and (
"gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
is_gemini3flash = model and (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
)
if reasoning_effort == "minimal":
if is_gemini3flash:
@ -798,7 +805,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
thinking_budget = thinking_param.get("budget_tokens")
params: GeminiThinkingConfig = {}
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
if thinking_enabled:
@ -807,11 +814,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
params["includeThoughts"] = True
if thinking_budget >= 10000:
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
params["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
is_gemini3flash = (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
)
params["thinkingLevel"] = (
"minimal" if is_gemini3flash else "low"
)
else:
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
params["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
is_gemini3flash = (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
)
params["thinkingLevel"] = (
"minimal" if is_gemini3flash else "low"
)
else:
# Thinking disabled
params["includeThoughts"] = False
@ -823,7 +840,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = True
if thinking_budget is not None and isinstance(thinking_budget, int):
params["thinkingBudget"] = thinking_budget
return params
def map_response_modalities(self, value: list) -> list:
@ -979,16 +996,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
param_description="thinking_budget",
)
if VertexGeminiConfig._is_gemini_3_or_newer(model):
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
value, model
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
value, model
)
)
else:
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
value, model
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
value, model
)
)
elif param == "thinking":
# Validate no conflict with thinking_level
@ -997,11 +1014,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
param_name="thinking",
param_description="thinking_budget",
)
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value),
model=model,
optional_params["thinkingConfig"] = (
VertexGeminiConfig._map_thinking_param(
cast(AnthropicThinkingParam, value),
model=model,
)
)
elif param == "modalities" and isinstance(value, list):
response_modalities = self.map_response_modalities(value)
@ -1035,8 +1052,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
):
# For gemini-3-flash-preview, default to "minimal" to match Gemini 2.5 Flash behavior
# For other Gemini 3 models, default to "low"
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
thinking_config["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
is_gemini3flash = (
"gemini-3-flash-preview" in model.lower()
or "gemini-3-flash" in model.lower()
)
thinking_config["thinkingLevel"] = (
"minimal" if is_gemini3flash else "low"
)
optional_params["thinkingConfig"] = thinking_config
return optional_params
@ -1225,7 +1247,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
block: ChatCompletionThinkingBlock = {
"type": "thinking",
"thinking": thinking_text,
}
}
signature = part.get("thoughtSignature")
if signature is not None:
block["signature"] = signature
@ -1359,10 +1381,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
_tool_response_chunk["provider_specific_fields"] = { # type: ignore
"thought_signature": thought_signature
}
_tool_response_chunk[
"id"
] = _encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
_tool_response_chunk["id"] = (
_encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
)
_tools.append(_tool_response_chunk)
cumulative_tool_call_idx += 1
@ -1514,8 +1536,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
cached_tokens: Optional[int] = None
audio_tokens: Optional[int] = None
text_tokens: Optional[int] = None
image_tokens: Optional[int] = None
text_tokens: Optional[int] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
reasoning_tokens: Optional[int] = None
response_tokens: Optional[int] = None
@ -1534,6 +1556,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
response_tokens_details.text_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "AUDIO":
response_tokens_details.audio_tokens = detail.get("tokenCount", 0)
#########################################################
## CANDIDATES TOKEN DETAILS (e.g., for image generation models) ##
@ -1550,16 +1573,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif modality == "IMAGE":
response_tokens_details.image_tokens = token_count
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
# candidatesTokenCount includes all modalities, so: text = total - (image + audio)
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
# candidatesTokenCount includes all modalities, so: text = total - (image + audio)
candidates_token_count = usage_metadata.get("candidatesTokenCount", 0)
if candidates_token_count > 0:
if response_tokens_details is None:
response_tokens_details = CompletionTokensDetailsWrapper()
if response_tokens_details.text_tokens is None:
candidates_token_count = usage_metadata.get("candidatesTokenCount", 0)
image_tokens = response_tokens_details.image_tokens or 0
audio_tokens_candidate = response_tokens_details.audio_tokens or 0
calculated_text_tokens = candidates_token_count - image_tokens - audio_tokens_candidate
calculated_text_tokens = (
candidates_token_count - image_tokens - audio_tokens_candidate
)
response_tokens_details.text_tokens = calculated_text_tokens
#########################################################
## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached)
if "promptTokensDetails" in usage_metadata:
for detail in usage_metadata["promptTokensDetails"]:
if detail["modality"] == "AUDIO":
@ -1568,6 +1597,32 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
text_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "IMAGE":
image_tokens = detail.get("tokenCount", 0)
## Parse cacheTokensDetails (breakdown of cached tokens by modality)
## When explicit caching is used, Gemini provides this field to show which modalities were cached
cached_text_tokens: Optional[int] = None
cached_audio_tokens: Optional[int] = None
cached_image_tokens: Optional[int] = None
if "cacheTokensDetails" in usage_metadata:
for detail in usage_metadata["cacheTokensDetails"]:
if detail["modality"] == "AUDIO":
cached_audio_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "TEXT":
cached_text_tokens = detail.get("tokenCount", 0)
elif detail["modality"] == "IMAGE":
cached_image_tokens = detail.get("tokenCount", 0)
## Calculate non-cached tokens by subtracting cached from total (per modality)
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
## See: https://github.com/BerriAI/litellm/issues/18750
if cached_text_tokens is not None and text_tokens is not None:
text_tokens = text_tokens - cached_text_tokens
if cached_audio_tokens is not None and audio_tokens is not None:
audio_tokens = audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and image_tokens is not None:
image_tokens = image_tokens - cached_image_tokens
if "thoughtsTokenCount" in usage_metadata:
reasoning_tokens = usage_metadata["thoughtsTokenCount"]
# Also add reasoning tokens to response_tokens_details
@ -1575,15 +1630,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
response_tokens_details = CompletionTokensDetailsWrapper()
response_tokens_details.reasoning_tokens = reasoning_tokens
## adjust 'text_tokens' to subtract cached tokens
if (
(audio_tokens is None or audio_tokens == 0)
and text_tokens is not None
and text_tokens > 0
and cached_tokens is not None
):
text_tokens = text_tokens - cached_tokens
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
audio_tokens=audio_tokens,
@ -1605,6 +1651,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
completion_tokens=completion_tokens,
total_tokens=usage_metadata.get("totalTokenCount", 0),
prompt_tokens_details=prompt_tokens_details,
cache_read_input_tokens=cached_tokens,
reasoning_tokens=reasoning_tokens,
completion_tokens_details=response_tokens_details,
)
@ -1811,6 +1858,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
functions: Optional[ChatCompletionToolCallFunctionChunk] = None
thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
reasoning_content: Optional[str] = None
thought_signatures: Optional[Any] = None
for idx, candidate in enumerate(_candidates):
if "content" not in candidate:
@ -2074,28 +2122,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
## ADD METADATA TO RESPONSE ##
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
model_response._hidden_params[
"vertex_ai_grounding_metadata"
] = grounding_metadata
model_response._hidden_params["vertex_ai_grounding_metadata"] = (
grounding_metadata
)
setattr(
model_response, "vertex_ai_url_context_metadata", url_context_metadata
)
model_response._hidden_params[
"vertex_ai_url_context_metadata"
] = url_context_metadata
model_response._hidden_params["vertex_ai_url_context_metadata"] = (
url_context_metadata
)
setattr(model_response, "vertex_ai_safety_results", safety_ratings)
model_response._hidden_params[
"vertex_ai_safety_results"
] = safety_ratings # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_safety_results"] = (
safety_ratings # older approach - maintaining to prevent regressions
)
## ADD CITATION METADATA ##
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
model_response._hidden_params[
"vertex_ai_citation_metadata"
] = citation_metadata # older approach - maintaining to prevent regressions
model_response._hidden_params["vertex_ai_citation_metadata"] = (
citation_metadata # older approach - maintaining to prevent regressions
)
except Exception as e:
raise VertexAIError(

48
litellm/llms/watsonx/audio_transcription/transformation.py
View file

@ -7,13 +7,14 @@ WatsonX follows the OpenAI spec for audio transcription.
from typing import Any, Dict, List, Optional
import litellm
from httpx import Response
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.llms.watsonx import WatsonXAudioTranscriptionRequestBody
from litellm.types.utils import FileTypes
from litellm.types.utils import FileTypes, TranscriptionResponse
from ...base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
@ -156,3 +157,48 @@ class IBMWatsonXAudioTranscriptionConfig(
url = f"{url}?version={api_version}"
return url
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
"""
Transform the audio transcription response from WatsonX.
WatsonX may include a 'model' field in the response, which needs to be
removed before creating the TranscriptionResponse object.
"""
try:
raw_response_json = raw_response.json()
except Exception as e:
raise ValueError(
f"Error transforming response to json: {str(e)}\nResponse: {raw_response.text}"
)
# Extract only valid fields for TranscriptionResponse.__init__()
# TranscriptionResponse only accepts 'text' and 'usage' in __init__()
text = raw_response_json.get("text")
usage = raw_response_json.get("usage")
# Create response with only valid fields
response_kwargs = {}
if text is not None:
response_kwargs["text"] = text
if usage is not None:
response_kwargs["usage"] = usage
if not response_kwargs:
raise ValueError(
"Invalid response format. Received response does not match the expected format. Got: ",
raw_response_json,
)
response = TranscriptionResponse(**response_kwargs)
# Add other fields using dictionary-style assignment (like duration, task, etc.)
# Skip fields that TranscriptionResponse doesn't accept in __init__()
for key, value in raw_response_json.items():
if key not in ["text", "usage", "model"]: # text/usage already set, model should be excluded
response[key] = value
return response

51
litellm/main.py
View file

@ -4701,6 +4701,51 @@ def embedding( # noqa: PLR0915
litellm_params=litellm_params_dict,
headers=headers,
)
elif custom_llm_provider == "openrouter":
api_base = (
api_base
or litellm.api_base
or get_secret_str("OPENROUTER_API_BASE")
or "https://openrouter.ai/api/v1"
)
api_key = (
api_key
or litellm.api_key
or litellm.openrouter_key
or get_secret("OPENROUTER_API_KEY")
or get_secret("OR_API_KEY")
)
openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_headers = {
"HTTP-Referer": openrouter_site_url,
"X-Title": openrouter_app_name,
}
_headers = headers or litellm.headers
if _headers:
openrouter_headers.update(_headers)
headers = openrouter_headers
response = base_llm_http_handler.embedding(
model=model,
input=input,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
logging_obj=logging,
timeout=timeout,
model_response=EmbeddingResponse(),
optional_params=optional_params,
client=client,
aembedding=aembedding,
litellm_params=litellm_params_dict,
headers=headers,
)
elif custom_llm_provider == "huggingface":
api_key = (
api_key
@ -5602,11 +5647,9 @@ def text_completion( # noqa: PLR0915
)
and isinstance(prompt, list)
and len(prompt) > 0
and isinstance(prompt[0], list)
and (isinstance(prompt[0], list) or isinstance(prompt[0], int))
):
verbose_logger.warning(
msg="List of lists being passed. If this is for tokens, then it might not work across all models."
)
# Support for token IDs as prompt (list of integers or list of lists of integers)
messages = [{"role": "user", "content": prompt}] # type: ignore
else:
raise Exception(

2445
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

42
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -551,6 +551,7 @@ class MCPServerManager:
allowed_tools=getattr(mcp_server, "allowed_tools", None),
disallowed_tools=getattr(mcp_server, "disallowed_tools", None),
allow_all_keys=mcp_server.allow_all_keys,
updated_at=getattr(mcp_server, "updated_at", None),
)
return new_server
@ -697,9 +698,7 @@ class MCPServerManager:
results = await asyncio.gather(*tasks)
# Flatten results into single list
list_tools_result: List[MCPTool] = [
tool for tools in results for tool in tools
]
list_tools_result: List[MCPTool] = [tool for tools in results for tool in tools]
verbose_logger.info(
f"Successfully fetched {len(list_tools_result)} tools total from all servers"
@ -2059,7 +2058,8 @@ class MCPServerManager:
return None
async def _add_mcp_servers_from_db_to_in_memory_registry(self):
async def reload_servers_from_database(self):
"""Re-synchronize the in-memory MCP server registry with the database."""
from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
get_prisma_client_or_throw,
@ -2074,15 +2074,34 @@ class MCPServerManager:
db_mcp_servers = await get_all_mcp_servers(prisma_client)
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
# ensure the global_mcp_server_manager is up to date with the db
previous_registry = self.registry
new_registry: Dict[str, MCPServer] = {}
for server in db_mcp_servers:
existing_server = previous_registry.get(server.server_id)
if (
existing_server is not None
and existing_server.updated_at is not None
and server.updated_at is not None
and existing_server.updated_at == server.updated_at
):
# Re-use existing server instance to avoid re-running build_mcp_server_from_table()
# which can perform network discovery for OAuth2 servers.
new_registry[server.server_id] = existing_server
continue
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
f"Building server from DB: {server.server_id} ({server.server_name})"
)
await self.add_server(server)
new_registry[server.server_id] = await self.build_mcp_server_from_table(
server
)
self.registry = new_registry
verbose_logger.debug(
f"Registry now contains {len(self.get_registry())} servers"
"MCP registry refreshed (%s servers in registry)", len(new_registry)
)
def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]:
@ -2369,13 +2388,6 @@ class MCPServerManager:
servers.append(self._build_mcp_server_table(server))
return servers
async def reload_servers_from_database(self):
"""
Public method to reload all MCP servers from database into registry.
This can be called from management endpoints to ensure registry is up to date.
"""
await self._add_mcp_servers_from_db_to_in_memory_registry()
async def get_all_mcp_servers_with_health_unfiltered(
self, server_ids: Optional[List[str]] = None
) -> List[LiteLLM_MCPServerTable]:

1
litellm/proxy/_experimental/out/_next/static/chunks/1108-909e2cc499a05c96.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1128-64fa4a41ccaf67ea.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1130-8e58d6f70a0ae076.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1200-cf5c22d7c680d667.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1250-85d99b7c90e56c2a.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1253-154d1dd5b99252f0.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1274-d488be5745db83cd.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/137-cbbf776473e39926.js → litellm/proxy/_experimental/out/_next/static/chunks/137-f4f97d78b223b4c9.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1385-7a20fecf18a7fb6a.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1414-2770d1155b664522.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/143-9c81168540978019.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1602-158ea5a27f7c5d7c.js
View file

File diff suppressed because one or more lines are too long

0
litellm/proxy/_experimental/out/_next/static/chunks/1623-995fddc2b5647961.js → litellm/proxy/_experimental/out/_next/static/chunks/1623-54c56cbe1afc3953.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/1713-b3fdb241d0f3ae7a.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1713-ce16d8a0e658a15d.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/173-d2d392886190d22e.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1739-e00951b4ce375e4e.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1803-798ae903473c4b6c.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1909-3c9de76bfc43e12c.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2004-092c0438baa1cbfc.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2004-db4889006b357dff.js Normal file
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/_next/static/chunks/2068-2c78bfc32dc0de5f.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/2202-f9c2c9a967498fa2.js → litellm/proxy/_experimental/out/_next/static/chunks/2202-8048c018718c1d43.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2249-3d0096e095fc9d09.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2273-23d4f6fdcd9c3a35.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2377-7121736141e67af2.js
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more