scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into fix/tpm-rate-limit

Browse source
This commit is contained in:
Harshit Jain 2026-01-12 19:13:56 +05:30 • committed by GitHub
commit 6db1740d80
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
527 changed files with 24945 additions and 3941 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
.circleci/config.yml
View file

@ -2038,6 +2038,39 @@ jobs:
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
memory_leak_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
steps:
- setup_litellm_test_deps
- run:
name: Install Memory Test Dependencies
command: |
pip install "psutil>=5.9.0"
pip install "fastapi>=0.100.0"
pip install "httpx>=0.24.0"
pip install "uvicorn>=0.23.0"
- run:
name: Run Linear Memory Growth Tests
command: |
echo "Running memory leak tests individually to avoid baseline drift..."
echo "Running test_memory_baseline_1k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_1k -v -s --tb=short
echo "Running test_memory_baseline_2k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_2k -v -s --tb=short
echo "Running test_memory_baseline_4k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_4k -v -s --tb=short
echo "Running test_memory_baseline_10k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_10k -v -s --tb=short
echo "Running test_memory_baseline_30k..."
python -m pytest tests/load_tests/test_linear_memory_growth.py::test_memory_baseline_30k -v -s --tb=short
no_output_timeout: 60m
db_migration_disable_update_check:
machine:
image: ubuntu-2204:2023.10.1
@ -2102,10 +2135,11 @@ jobs:
name: Check container logs for expected message
command: |
echo "=== Printing Full Container Startup Logs ==="
docker logs my-app
LOG_OUTPUT="$(docker logs my-app 2>&1)"
printf '%s\n' "$LOG_OUTPUT"
echo "=== End of Full Container Startup Logs ==="
if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
echo "Expected message found in logs. Test passed."
else
echo "Expected message not found in logs. Test failed."
@ -3557,12 +3591,34 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7} \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7} \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$SMALL_DATABASE_URL \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e UI_USERNAME="admin" \
@ -3765,6 +3821,12 @@ workflows:
only:
- main
- /litellm_.*/
- memory_leak_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
@ -3792,6 +3854,7 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
context: e2e_ui_tests
requires:
- ui_build
- build_docker_database_image

15
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View file

@ -16,6 +16,21 @@ body:
value: "A bug happened!"
validations:
required: true
- type: textarea
id: steps-to-reproduce
attributes:
label: Steps to Reproduce
description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
placeholder: |
1. config.yaml file/ .env file/ etc.
2. Run the following code...
3. Observe the error...
value: |
1.
2.
3.
validations:
required: true
- type: textarea
id: logs
attributes:

29
.github/workflows/ghcr_deploy.yml vendored
View file

@ -5,6 +5,7 @@ on:
inputs:
tag:
description: "The tag version you want to build"
required: true
release_type:
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
type: string
@ -336,9 +337,9 @@ jobs:
run: |
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
if [ -z "${CHART_LIST}" ]; then
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
else
# Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
# Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
fi
@ -350,28 +351,42 @@ jobs:
id: bump_version
uses: christian-draeger/increment-semantic-version@1.1.0
with:
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
version-fragment: 'bug'
# Add suffix for non-stable releases (semantic versioning)
- name: Calculate chart version with prerelease suffix
- name: Calculate chart and app versions
id: chart_version
shell: bash
run: |
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
INPUT_TAG="${{ github.event.inputs.tag }}"
# Chart version (independent Helm chart versioning with release type suffix)
if [ "$RELEASE_TYPE" = "stable" ]; then
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
else
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
fi
# App version (must match Docker tags)
# stable/rc releases: Docker creates main-{tag}, so use the tag
# latest/dev releases: Docker only creates main-{release_type}, so use release_type
if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
APP_VERSION="${INPUT_TAG}"
else
APP_VERSION="${RELEASE_TYPE}"
fi
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
- uses: ./.github/actions/helm-oci-chart-releaser
with:
name: ${{ env.CHART_NAME }}
repository: ${{ env.REPO_OWNER }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
app_version: ${{ steps.chart_version.outputs.app_version }}
path: deploy/charts/${{ env.CHART_NAME }}
registry: ${{ env.REGISTRY }}
registry_username: ${{ github.actor }}

174
.github/workflows/label-component.yml vendored
View file

@ -11,134 +11,72 @@ jobs:
permissions:
issues: write
steps:
- name: Add SDK label
if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nSDK (litellm Python package)')
- name: Add component labels
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'sdk';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
const body = context.payload.issue.body;
if (!body) return;
- name: Add Proxy label
if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nProxy')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'proxy';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
});
} else {
throw error;
// Define component mappings with regex patterns that handle flexible whitespace
const components = [
{
pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
label: 'sdk',
color: '0E7C86',
description: 'Issues related to the litellm Python SDK'
},
{
pattern: /What part of LiteLLM is this about\?\s*Proxy/,
label: 'proxy',
color: '5319E7',
description: 'Issues related to the LiteLLM Proxy'
},
{
pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
label: 'ui-dashboard',
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
},
{
pattern: /What part of LiteLLM is this about\?\s*Docs/,
label: 'docs',
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
];
- name: Add UI Dashboard label
if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nUI Dashboard')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'ui-dashboard';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'D876E3',
description: 'Issues related to the LiteLLM UI Dashboard'
});
} else {
throw error;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});
// Find matching component
for (const component of components) {
if (component.pattern.test(body)) {
// Ensure label exists
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: component.label,
color: component.color,
description: component.description
});
}
}
- name: Add Docs label
if: contains(github.event.issue.body, 'What part of LiteLLM is this about?\n\nDocs')
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'docs';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
// Add label to issue
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'FBCA04',
description: 'Issues related to LiteLLM documentation'
issue_number: context.issue.number,
labels: [component.label]
});
} else {
throw error;
break;
}
}
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [labelName]
});

1
.github/workflows/publish-migrations.yml vendored
View file

@ -13,6 +13,7 @@ on:
jobs:
publish-migrations:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
services:
postgres:

2
README.md
View file

@ -262,6 +262,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ | | | | | | | | | |
| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
@ -455,4 +456,3 @@ All these checks must pass before your PR can be merged.
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
</a>

4
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,13 +18,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.10
version: 1.0.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.50.2
appVersion: v1.80.12
dependencies:
- name: "postgresql"

42
docs/my-website/docs/completion/input.md
View file

@ -142,7 +142,47 @@ def completion(
- `tool_call_id`: *str (optional)* - Tool call that this message is responding to.
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392)
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664)
#### Content Types
`content` can be a string (text only) or a list of content blocks (multimodal):
| Type | Description | Docs |
|------|-------------|------|
| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) |
| `image_url` | Images | [Vision](./vision.md) |
| `input_audio` | Audio input | [Audio](./audio.md) |
| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) |
| `file` | Files | [Document Understanding](./document_understanding.md) |
| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) |
**Examples:**
```python
# Text
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}]
# Image
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}]
# Audio
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "<base64>", "format": "wav"}}]}]
# Video
messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}]
# File
messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}]
# Document
messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": "<base64>"}}]}]
# Combining multiple types (multimodal)
messages=[{"role": "user", "content": [
{"type": "text", "text": "Generate a product description based on this image"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
]}]
```
## Optional Fields

13
docs/my-website/docs/mcp_control.md
View file

@ -649,3 +649,16 @@ general_settings:
```
This is useful when you want discoverability for MCP offerings without granting additional execution privileges.
## Publish MCP Registry
If you want other systems—for example external agent frameworks such as MCP-capable IDEs running outside your network—to automatically discover the MCP servers hosted on LiteLLM, you can expose a Model Context Protocol Registry endpoint. This registry lists the built-in LiteLLM MCP server and every server you have configured, using the [official MCP Registry spec](https://github.com/modelcontextprotocol/registry).
1. Set `enable_mcp_registry: true` under `general_settings` in your proxy config (or DB settings) and restart the proxy.
2. LiteLLM will serve the registry at `GET /v1/mcp/registry.json`.
3. Each entry points to either `/mcp` (built-in server) or `/{mcp_server_name}/mcp` for your custom servers, so clients can connect directly using the advertised Streamable HTTP URL.
:::note Permissions still apply
The registry only advertises server URLs. Actual access control is still enforced by LiteLLM when the client connects to `/mcp` or `/{server}/mcp`, so publishing the registry does not bypass per-key permissions.
:::

93
docs/my-website/docs/observability/focus.md Normal file
View file

@ -0,0 +1,93 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Focus Export (Experimental)
:::caution Experimental feature
Focus Format export is under active development and currently considered experimental.
Interfaces, schema mappings, and configuration options may change as we iterate based on user feedback.
Please treat this integration as a preview and report any issues or suggestions to help us stabilize and improve the workflow.
:::
LiteLLM can emit usage data in the [FinOps FOCUS format](https://focus.finops.org/focus-specification/v1-2/) and push artifacts (for example Parquet files) to destinations such as Amazon S3. This enables downstream cost-analysis tooling to ingest a standardised dataset directly from LiteLLM.
LiteLLM currently conforms to the FinOps FOCUS v1.2 specification when emitting this dataset.
## Overview
| Property | Details |
|----------|---------|
| Destination | Export LiteLLM usage data in FOCUS format to managed storage (currently S3) |
| Callback name | `focus` |
| Supported operations | Automatic scheduled export |
| Data format | FOCUS Normalised Dataset (Parquet) |
## Environment Variables
### Common settings
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_PROVIDER` | No | Destination provider (defaults to `s3`). |
| `FOCUS_FORMAT` | No | Output format (currently only `parquet`). |
| `FOCUS_FREQUENCY` | No | Export cadence. Prefer `hourly` or `daily` for production; `interval` is intended for short test loops. Defaults to `hourly`. |
| `FOCUS_CRON_OFFSET` | No | Minute offset used for hourly/daily cron triggers. Defaults to `5`. |
| `FOCUS_INTERVAL_SECONDS` | No | Interval (seconds) when `FOCUS_FREQUENCY="interval"`. |
| `FOCUS_PREFIX` | No | Object key prefix/folder. Defaults to `focus_exports`. |
### S3 destination
| Variable | Required | Description |
|----------|----------|-------------|
| `FOCUS_S3_BUCKET_NAME` | Yes | Destination bucket for exported files. |
| `FOCUS_S3_REGION_NAME` | No | AWS region for the bucket. |
| `FOCUS_S3_ENDPOINT_URL` | No | Custom endpoint (useful for S3-compatible storage). |
| `FOCUS_S3_ACCESS_KEY` | Yes | AWS access key for uploads. |
| `FOCUS_S3_SECRET_KEY` | Yes | AWS secret key for uploads. |
| `FOCUS_S3_SESSION_TOKEN` | No | AWS session token if using temporary credentials. |
## Setup via Config
### Configure environment variables
```bash
export FOCUS_PROVIDER="s3"
export FOCUS_PREFIX="focus_exports"
# S3 example
export FOCUS_S3_BUCKET_NAME="my-litellm-focus-bucket"
export FOCUS_S3_REGION_NAME="us-east-1"
export FOCUS_S3_ACCESS_KEY="AKIA..."
export FOCUS_S3_SECRET_KEY="..."
```
### Update LiteLLM config
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-your-key
litellm_settings:
callbacks: ["focus"]
```
### Start the proxy
```bash
litellm --config /path/to/config.yaml
```
During boot LiteLLM registers the Focus logger and a background job that runs according to the configured frequency.
## Planned Enhancements
- Add "Setup on UI" flow alongside the current configuration-based setup.
- Add GCS / Azure Blob to the Destination options.
- Support CSV output alongside Parquet.
## Related Links
- [Focus](https://focus.finops.org/)

122
docs/my-website/docs/observability/qualifire_integration.md Normal file
View file

@ -0,0 +1,122 @@
import Image from '@theme/IdealImage';
# Qualifire - LLM Evaluation, Guardrails & Observability
[Qualifire](https://qualifire.ai/) provides real-time Agentic evaluations, guardrails and observability for production AI applications.
**Key Features:**
- **Evaluation** - Systematically assess AI behavior to detect hallucinations, jailbreaks, policy breaches, and other vulnerabilities
- **Guardrails** - Real-time interventions to prevent risks like brand damage, data leaks, and compliance breaches
- **Observability** - Complete tracing and logging for RAG pipelines, chatbots, and AI agents
- **Prompt Management** - Centralized prompt management with versioning and no-code studio
:::tip
Looking for Qualifire Guardrails? Check out the [Qualifire Guardrails Integration](../proxy/guardrails/qualifire.md) for real-time content moderation, prompt injection detection, PII checks, and more.
:::
## Pre-Requisites
1. Create an account on [Qualifire](https://app.qualifire.ai/)
2. Get your API key and webhook URL from the Qualifire dashboard
```bash
pip install litellm
```
## Quick Start
Use just 2 lines of code to instantly log your responses **across all providers** with Qualifire.
```python
litellm.callbacks = ["qualifire_eval"]
```
```python
import litellm
import os
# Set Qualifire credentials
os.environ["QUALIFIRE_API_KEY"] = "your-qualifire-api-key"
os.environ["QUALIFIRE_WEBHOOK_URL"] = "https://your-qualifire-webhook-url"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = "your-openai-api-key"
# Set qualifire_eval as a callback & LiteLLM will send the data to Qualifire
litellm.callbacks = ["qualifire_eval"]
# OpenAI call
response = litellm.completion(
model="gpt-5",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
]
)
```
## Using with LiteLLM Proxy
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["qualifire_eval"]
general_settings:
master_key: "sk-1234"
environment_variables:
QUALIFIRE_API_KEY: "your-qualifire-api-key"
QUALIFIRE_WEBHOOK_URL: "https://app.qualifire.ai/api/v1/webhooks/evaluations"
```
2. Start the proxy
```bash
litellm --config config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
```
## Environment Variables
| Variable | Description |
| ----------------------- | ------------------------------------------------------ |
| `QUALIFIRE_API_KEY` | Your Qualifire API key for authentication |
| `QUALIFIRE_WEBHOOK_URL` | The Qualifire webhook endpoint URL from your dashboard |
## What Gets Logged?
The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your Qualifire endpoint on each successful LLM API call.
This includes:
- Request messages and parameters
- Response content and metadata
- Token usage statistics
- Latency metrics
- Model information
- Cost data
Once data is in Qualifire, you can:
- Run evaluations to detect hallucinations, toxicity, and policy violations
- Set up guardrails to block or modify responses in real-time
- View traces across your entire AI pipeline
- Track performance and quality metrics over time

109
docs/my-website/docs/providers/abliteration.md Normal file
View file

@ -0,0 +1,109 @@
# Abliteration
## Overview
| Property | Details |
|-------|-------|
| Description | Abliteration provides an OpenAI-compatible `/chat/completions` endpoint. |
| Provider Route on LiteLLM | `abliteration/` |
| Link to Provider Doc | [Abliteration](https://abliteration.ai) |
| Base URL | `https://api.abliteration.ai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["ABLITERATION_API_KEY"] = "" # your Abliteration API key
```
## Sample Usage
```python showLineNumbers title="Abliteration Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Hello from LiteLLM"}],
)
print(response)
```
## Sample Usage - Streaming
```python showLineNumbers title="Abliteration Streaming Completion"
import os
from litellm import completion
os.environ["ABLITERATION_API_KEY"] = ""
response = completion(
model="abliteration/abliterated-model",
messages=[{"role": "user", "content": "Stream a short reply"}],
stream=True,
)
for chunk in response:
print(chunk)
```
## Usage with LiteLLM Proxy Server
1. Add the model to your proxy config:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: abliteration-chat
litellm_params:
model: abliteration/abliterated-model
api_key: os.environ/ABLITERATION_API_KEY
```
2. Start the proxy:
```bash
litellm --config /path/to/config.yaml
```
## Direct API Usage (Bearer Token)
Use the environment variable as a Bearer token against the OpenAI-compatible endpoint:
`https://api.abliteration.ai/v1/chat/completions`.
```bash showLineNumbers title="cURL"
export ABLITERATION_API_KEY=""
curl https://api.abliteration.ai/v1/chat/completions \
-H "Authorization: Bearer ${ABLITERATION_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}]
}'
```
```python showLineNumbers title="Python (requests)"
import os
import requests
api_key = os.environ["ABLITERATION_API_KEY"]
response = requests.post(
"https://api.abliteration.ai/v1/chat/completions",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"model": "abliterated-model",
"messages": [{"role": "user", "content": "Hello from Abliteration"}],
},
timeout=60,
)
print(response.json())
```

4
docs/my-website/docs/providers/anthropic.md
View file

@ -1692,9 +1692,9 @@ Assistant:
```
## Usage - PDF
## Usage - PDF
Pass base64 encoded PDF files to Anthropic models using the `image_url` field.
Pass base64 encoded PDF files to Anthropic models using the `file` content type with a `file_data` field.
<Tabs>
<TabItem value="sdk" label="SDK">

27
docs/my-website/docs/providers/bedrock.md
View file

@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc), [`bedrock/moonshot`](./bedrock_imported.md#moonshot-kimi-k2-thinking) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Rerank Endpoint | `/rerank` |
@ -967,6 +967,30 @@ Control the processing tier for your Bedrock requests using `serviceTier`. Valid
[Bedrock ServiceTier API Reference](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ServiceTier.html)
### OpenAI-compatible `service_tier` parameter
LiteLLM also supports the OpenAI-style `service_tier` parameter, which is automatically translated to Bedrock's native `serviceTier` format:
| OpenAI `service_tier` | Bedrock `serviceTier` |
|-----------------------|----------------------|
| `"priority"` | `{"type": "priority"}` |
| `"default"` | `{"type": "default"}` |
| `"flex"` | `{"type": "flex"}` |
| `"auto"` | `{"type": "default"}` |
```python
from litellm import completion
# Using OpenAI-style service_tier parameter
response = completion(
model="bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
service_tier="priority" # Automatically translated to serviceTier={"type": "priority"}
)
```
### Native Bedrock `serviceTier` parameter
<Tabs>
<TabItem value="sdk" label="SDK">
@ -1941,6 +1965,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Moonshot Kimi K2 Thinking | `completion(model='bedrock/moonshot.kimi-k2-thinking', messages=messages)` or `completion(model='bedrock/invoke/moonshot.kimi-k2-thinking', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding

178
docs/my-website/docs/providers/bedrock_imported.md
View file

@ -431,4 +431,180 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"max_tokens": 300,
"temperature": 0.5
}'
```
```
### Moonshot Kimi K2 Thinking
Moonshot AI's Kimi K2 Thinking model is now available on Amazon Bedrock. This model features advanced reasoning capabilities with automatic reasoning content extraction.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/moonshot.kimi-k2-thinking`, `bedrock/invoke/moonshot.kimi-k2-thinking` |
| Provider Documentation | [AWS Bedrock Moonshot Announcement ↗](https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/) |
| Supported Parameters | `temperature`, `max_tokens`, `top_p`, `stream`, `tools`, `tool_choice` |
| Special Features | Reasoning content extraction, Tool calling |
#### Supported Features
- **Reasoning Content Extraction**: Automatically extracts `<reasoning>` tags and returns them as `reasoning_content` (similar to OpenAI's o1 models)
- **Tool Calling**: Full support for function/tool calling with tool responses
- **Streaming**: Both streaming and non-streaming responses
- **System Messages**: System message support
#### Basic Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python title="Moonshot Kimi K2 SDK Usage" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2" # or your preferred region
# Basic completion
response = completion(
model="bedrock/moonshot.kimi-k2-thinking", # or bedrock/invoke/moonshot.kimi-k2-thinking
messages=[
{"role": "user", "content": "What is 2+2? Think step by step."}
],
temperature=0.7,
max_tokens=200
)
print(response.choices[0].message.content)
# Access reasoning content if present
if response.choices[0].message.reasoning_content:
print("Reasoning:", response.choices[0].message.reasoning_content)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: kimi-k2
litellm_params:
model: bedrock/moonshot.kimi-k2-thinking
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
```
**2. Start proxy**
```bash title="Start LiteLLM Proxy" showLineNumbers
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash title="Test Kimi K2 via Proxy" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "kimi-k2",
"messages": [
{
"role": "user",
"content": "What is 2+2? Think step by step."
}
],
"temperature": 0.7,
"max_tokens": 200
}'
```
</TabItem>
</Tabs>
#### Tool Calling Example
```python title="Kimi K2 with Tool Calling" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
# Tool calling example
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "What's the weather in Tokyo?"}
],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name"
}
},
"required": ["location"]
}
}
}
]
)
if response.choices[0].message.tool_calls:
tool_call = response.choices[0].message.tool_calls[0]
print(f"Tool called: {tool_call.function.name}")
print(f"Arguments: {tool_call.function.arguments}")
```
#### Streaming Example
```python title="Kimi K2 Streaming" showLineNumbers
from litellm import completion
import os
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-west-2"
response = completion(
model="bedrock/moonshot.kimi-k2-thinking",
messages=[
{"role": "user", "content": "Explain quantum computing in simple terms."}
],
stream=True,
temperature=0.7
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
# Check for reasoning content in streaming
if hasattr(chunk.choices[0].delta, 'reasoning_content') and chunk.choices[0].delta.reasoning_content:
print(f"\n[Reasoning: {chunk.choices[0].delta.reasoning_content}]")
```
#### Supported Parameters
| Parameter | Type | Description | Supported |
|-----------|------|-------------|-----------|
| `temperature` | float (0-1) | Controls randomness in output | ✅ |
| `max_tokens` | integer | Maximum tokens to generate | ✅ |
| `top_p` | float | Nucleus sampling parameter | ✅ |
| `stream` | boolean | Enable streaming responses | ✅ |
| `tools` | array | Tool/function definitions | ✅ |
| `tool_choice` | string/object | Tool choice specification | ✅ |
| `stop` | array | Stop sequences | ❌ (Not supported on Bedrock) |

369
docs/my-website/docs/providers/manus.md Normal file
View file

@ -0,0 +1,369 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Manus
Use Manus AI agents through LiteLLM's OpenAI-compatible Responses API.
| Property | Details |
|----------|---------|
| Description | Manus is an AI agent platform for complex reasoning tasks, document analysis, and multi-step workflows with asynchronous task execution. |
| Provider Route on LiteLLM | `manus/{agent_profile}` |
| Supported Operations | `/responses` (Responses API), `/files` (Files API) |
| Provider Doc | [Manus API ↗](https://open.manus.im/docs/openai-compatibility) |
## Model Format
```shell
manus/{agent_profile}
```
**Examples:**
- `manus/manus-1.6` - General purpose agent
- `manus/manus-1.6-lite` - Lightweight agent for simple tasks
- `manus/manus-1.6-max` - Advanced agent for complex analysis
## LiteLLM Python SDK
```python showLineNumbers title="Basic Usage"
import litellm
import os
import time
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Create task
response = litellm.responses(
model="manus/manus-1.6",
input="What's the capital of France?",
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = litellm.get_response(
response_id=task_id,
custom_llm_provider="manus",
)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
## LiteLLM AI Gateway
### Setup
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: manus-agent
litellm_params:
model: manus/manus-1.6
api_key: os.environ/MANUS_API_KEY
```
```bash title="Start Proxy"
litellm --config config.yaml
```
### Usage
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Create Task"
# Create task
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": "What is the capital of France?"
}'
# Response
{
"id": "task_abc123",
"status": "running",
"metadata": {
"task_url": "https://manus.im/app/task_abc123"
}
}
```
```bash showLineNumbers title="Poll for Completion"
# Check status (repeat until status is "completed")
curl http://localhost:4000/responses/task_abc123 \
-H "Authorization: Bearer your-proxy-key"
# When completed
{
"id": "task_abc123",
"status": "completed",
"output": [
{
"role": "user",
"content": [{"text": "What is the capital of France?"}]
},
{
"role": "assistant",
"content": [{"text": "The capital of France is Paris."}]
}
]
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Create Task and Poll"
import openai
import time
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Create task
response = client.responses.create(
model="manus-agent",
input="What is the capital of France?"
)
print(f"Task ID: {response.id}")
print(f"Status: {response.status}") # "running"
# Poll until complete
task_id = response.id
while response.status == "running":
time.sleep(5)
response = client.responses.retrieve(response_id=task_id)
print(f"Status: {response.status}")
# Get results
if response.status == "completed":
for message in response.output:
if message.role == "assistant":
print(message.content[0].text)
```
</TabItem>
</Tabs>
## How It Works
Manus operates as an **asynchronous agent API**:
1. **Create Task**: When you call `litellm.responses()`, Manus creates a task and returns immediately with `status: "running"`
2. **Task Executes**: The agent works on your request in the background
3. **Poll for Completion**: You must repeatedly call `litellm.get_response()` or `client.responses.retrieve()` until the status changes to `"completed"`
4. **Get Results**: Once completed, the `output` field contains the full conversation
**Task Statuses:**
- `running` - Agent is actively working
- `pending` - Agent is waiting for input
- `completed` - Task finished successfully
- `error` - Task failed
:::tip Production Usage
For production applications, use [webhooks](https://open.manus.im/docs/webhooks) instead of polling to get notified when tasks complete.
:::
## Supported Parameters
| Parameter | Supported | Notes |
|-----------|-----------|-------|
| `input` | ✅ | Text, images, or structured content |
| `stream` | ✅ | Fake streaming (task runs async) |
| `max_output_tokens` | ✅ | Limits response length |
| `previous_response_id` | ✅ | For multi-turn conversations |
## Files API
Manus supports file uploads for document analysis and processing. Files can be uploaded and then referenced in Responses API calls.
### LiteLLM Python SDK
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import litellm
import os
# Set API key
os.environ["MANUS_API_KEY"] = "your-manus-api-key"
# Upload file
file_content = b"This is a document for analysis."
created_file = await litellm.acreate_file(
file=("document.txt", file_content),
purpose="assistants",
custom_llm_provider="manus",
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = await litellm.aresponses(
model="manus/manus-1.6",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id},
],
},
],
extra_body={"task_mode": "agent", "agent_profile": "manus-1.6-agent"},
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = await litellm.afile_retrieve(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"File details: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = await litellm.afile_delete(
file_id=created_file.id,
custom_llm_provider="manus",
)
print(f"Deleted: {deleted_file.deleted}")
```
### LiteLLM AI Gateway
<Tabs>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Upload File"
# Upload file
curl -X POST http://localhost:4000/v1/files \
-H "Authorization: Bearer your-proxy-key" \
-F "file=@document.txt" \
-F "purpose=assistants" \
-F "custom_llm_provider=manus"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Use File with Responses API"
# Create response with file
curl -X POST http://localhost:4000/responses \
-H "Authorization: Bearer your-proxy-key" \
-H "Content-Type: application/json" \
-d '{
"model": "manus-agent",
"input": [
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": "file_abc123"}
]
}
]
}'
```
```bash showLineNumbers title="Retrieve File"
# Get file details
curl http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"bytes": 1024,
"created_at": 1234567890,
"filename": "document.txt",
"purpose": "assistants",
"status": "uploaded"
}
```
```bash showLineNumbers title="Delete File"
# Delete file
curl -X DELETE http://localhost:4000/v1/files/file_abc123 \
-H "Authorization: Bearer your-proxy-key"
# Response
{
"id": "file_abc123",
"object": "file",
"deleted": true
}
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python showLineNumbers title="Upload, Use, Retrieve, and Delete Files"
import openai
client = openai.OpenAI(
base_url="http://localhost:4000",
api_key="your-proxy-key"
)
# Upload file
with open("document.txt", "rb") as f:
created_file = client.files.create(
file=f,
purpose="assistants",
extra_body={"custom_llm_provider": "manus"}
)
print(f"Uploaded file: {created_file.id}")
# Use file with Responses API
response = client.responses.create(
model="manus-agent",
input=[
{
"role": "user",
"content": [
{"type": "input_text", "text": "Summarize this document."},
{"type": "input_file", "file_id": created_file.id}
]
}
]
)
print(f"Response: {response.id}")
# Retrieve file
retrieved_file = client.files.retrieve(created_file.id)
print(f"File: {retrieved_file.filename}, {retrieved_file.bytes} bytes")
# Delete file
deleted_file = client.files.delete(created_file.id)
print(f"Deleted: {deleted_file.deleted}")
```
</TabItem>
</Tabs>
## Related Documentation
- [LiteLLM Responses API](/docs/response_api)
- [LiteLLM Files API](/docs/proxy/litellm_managed_files)
- [Manus OpenAI Compatibility](https://open.manus.im/docs/openai-compatibility)

17
docs/my-website/docs/providers/openrouter.md
View file

@ -1,5 +1,5 @@
# OpenRouter
LiteLLM supports all the text / chat / vision models from [OpenRouter](https://openrouter.ai/docs)
LiteLLM supports all the text / chat / vision / embedding models from [OpenRouter](https://openrouter.ai/docs)
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_OpenRouter.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
@ -78,3 +78,18 @@ response = completion(
route= ""
)
```
## Embedding
```python
from litellm import embedding
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = embedding(
model="openrouter/openai/text-embedding-3-small",
input=["good morning from litellm", "this is another item"],
)
print(response)
```

15
docs/my-website/docs/proxy/config_settings.md
View file

@ -146,6 +146,7 @@ router_settings:
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@ -293,6 +294,7 @@ router_settings:
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@ -322,6 +324,7 @@ router_settings:
| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
| enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) |
| tag_filtering_match_any | boolean | Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags |
| cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. |
| disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) |
| retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) |
@ -578,6 +581,18 @@ router_settings:
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
| FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80
| FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176
| FOCUS_PROVIDER | Destination provider for Focus exports (e.g., `s3`). Defaults to `s3`.
| FOCUS_FORMAT | Output format for Focus exports. Defaults to `parquet`.
| FOCUS_FREQUENCY | Frequency for scheduled Focus exports (`hourly`, `daily`, or `interval`). Defaults to `hourly`.
| FOCUS_CRON_OFFSET | Minute offset used when scheduling hourly/daily Focus exports. Defaults to `5` minutes.
| FOCUS_INTERVAL_SECONDS | Interval (in seconds) for Focus exports when `frequency` is `interval`.
| FOCUS_PREFIX | Object key prefix (or folder) used when uploading Focus export files. Defaults to `focus_exports`.
| FOCUS_S3_BUCKET_NAME | S3 bucket to upload Focus export files when using the S3 destination.
| FOCUS_S3_REGION_NAME | AWS region for the Focus export S3 bucket.
| FOCUS_S3_ENDPOINT_URL | Custom endpoint for the Focus export S3 client (optional; useful for S3-compatible storage).
| FOCUS_S3_ACCESS_KEY | AWS access key ID used by the Focus export S3 client.
| FOCUS_S3_SECRET_KEY | AWS secret access key used by the Focus export S3 client.
| FOCUS_S3_SESSION_TOKEN | AWS session token used by the Focus export S3 client (optional).
| FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication

6
docs/my-website/docs/proxy/db_deadlocks.md
View file

@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# High Availability Setup (Resolve DB Deadlocks)
:::tip Essential for Production
This configuration is **required** for production deployments handling 1000+ requests per second. Without Redis configured, you may experience PostgreSQL connection exhaustion (`FATAL: sorry, too many clients already`).
:::
Resolve any Database Deadlocks you see in high traffic by using this setup
## What causes the problem?

20
docs/my-website/docs/proxy/deploy.md
View file

@ -359,6 +359,26 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M
### Deploy with Database
##### Docker, Kubernetes, Helm Chart
:::warning High Traffic Deployments (1000+ RPS)
If you expect high traffic (1000+ requests per second), **Redis is required** to prevent database connection exhaustion and deadlocks.
Add this to your config:
```yaml
general_settings:
use_redis_transaction_buffer: true
litellm_settings:
cache: true
cache_params:
type: redis
host: your-redis-host
```
See [Resolve DB Deadlocks](/docs/proxy/db_deadlocks) for details.
:::
Requirements:
- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
- Set a `LITELLM_MASTER_KEY`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`)

117
docs/my-website/docs/proxy/endpoint_activity.md Normal file
View file

@ -0,0 +1,117 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Endpoint Activity
Track and visualize API endpoint usage directly in the dashboard. Monitor endpoint-level activity analytics, spend breakdowns, and performance metrics to understand which endpoints are receiving the most traffic and how they're performing.
## Overview
Endpoint Activity enables you to track spend and usage for individual API endpoints automatically. Every time you call an endpoint through the LiteLLM proxy, activity is automatically tracked and aggregated. This allows you to:
- Track spend per endpoint automatically
- View endpoint-level usage analytics in the Admin UI
- Monitor token consumption by endpoint
- Analyze success and failure rates per endpoint
- Identify which endpoints are getting the most activity
- View trend data showing endpoint usage over time
<Image img={require('../../img/ui_endpoint_activity.png')} />
## How Endpoint Activity Works
Endpoint activity is **automatically tracked** whenever you make API calls through the LiteLLM proxy. No additional configuration is required - simply call your endpoints as usual and activity will be tracked.
### Example API Call
When you make a request to any endpoint, activity is automatically recorded:
```bash showLineNumbers title="Endpoint activity is automatically tracked"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \ # 👈 ENDPOINT AUTOMATICALLY TRACKED
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
The endpoint (`/chat/completions`) will be automatically tracked with:
- Token counts (prompt tokens, completion tokens, total tokens)
- Spend for the request
- Request status (success or failure)
- Timestamp and other metadata
## How to View Endpoint Activity
### View Activity in Admin UI
Navigate to the Endpoint Activity tab in the Admin UI to view endpoint-level analytics:
#### 1. Access Endpoint Activity
Go to the Usage page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=new_usage`) and click on the **Endpoint Activity** tab.
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/67601fc0-8415-49b4-8e55-0673d37540c2/ascreenshot_f609a506dfe745c5aadccd332681c32d_text_export.jpeg)
#### 2. View Endpoint Analytics
The Endpoint Activity dashboard provides:
- **Endpoint usage table**: View all endpoints with aggregated metrics including:
- Total requests (successful and failed)
- Success rate percentage
- Total tokens consumed
- Total spend per endpoint
- **Success vs Failed requests chart**: Visualize request success and failure rates by endpoint
- **Usage trends**: See how endpoint activity changes over time with daily trend data
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/41b2b158-3ab3-4154-a0d0-7233451d3f2b/ascreenshot_ff46db6e09b54ea9bf34ae9028aff58a_text_export.jpeg)
![](https://colony-recorder.s3.amazonaws.com/files/2026-01-10/bce32f99-f0ba-4502-8a3a-76257ff5e47a/ascreenshot_2273d3a94acd42e983ad7d6436722c2a_text_export.jpeg)
#### 3. Understand Endpoint Metrics
Each endpoint displays the following metrics:
- **Successful Requests**: Number of requests that completed successfully
- **Failed Requests**: Number of requests that encountered errors
- **Total Requests**: Sum of successful and failed requests
- **Success Rate**: Percentage of successful requests
- **Total Tokens**: Sum of prompt and completion tokens
- **Spend**: Total cost for all requests to that endpoint
## Use Cases
### Performance Monitoring
Monitor endpoint health and performance:
- Identify endpoints with high failure rates
- Track which endpoints are receiving the most traffic
- Monitor token consumption patterns by endpoint
- Detect anomalies in endpoint usage
### Cost Optimization
Understand spend distribution across endpoints:
- Identify high-cost endpoints
- Optimize expensive endpoints
- Allocate budget based on endpoint usage
- Track cost trends over time
---
## Related Features
- [Customer Usage](./customer_usage.md) - Track spend and usage for individual customers
- [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics
- [Spend Logs](./spend_logs.md) - Detailed request-level spend logs

41
docs/my-website/docs/proxy/guardrails/qualifire.md
View file

@ -8,13 +8,7 @@ Use [Qualifire](https://qualifire.ai) to evaluate LLM outputs for quality, safet
## Quick Start
### 1. Install the Qualifire SDK
```bash
pip install qualifire
```
### 2. Define Guardrails on your LiteLLM config.yaml
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
@ -61,13 +55,13 @@ guardrails:
- `post_call` Run **after** LLM call, on **input & output**
- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel as LLM call. Response not returned until guardrail check completes
### 3. Start LiteLLM Gateway
### 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 4. Test request
### 3. Test request
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
@ -142,7 +136,7 @@ guardrails:
evaluation_id: eval_abc123 # Your evaluation ID from Qualifire dashboard
```
When `evaluation_id` is provided, LiteLLM will use `invoke_evaluation()` instead of `evaluate()`, running the pre-configured evaluation from your dashboard.
When `evaluation_id` is provided, LiteLLM will use the invoke evaluation API endpoint instead of the evaluate endpoint, running the pre-configured evaluation from your dashboard.
## Available Checks
@ -213,19 +207,19 @@ guardrails:
### Parameter Reference
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | --------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `None` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
| Parameter | Type | Default | Description |
| ------------------------------ | ----------- | ---------------------------- | -------------------------------------------------------- |
| `api_key` | `str` | `QUALIFIRE_API_KEY` env var | Your Qualifire API key |
| `api_base` | `str` | `https://proxy.qualifire.ai` | Custom API base URL (optional) |
| `evaluation_id` | `str` | `None` | Pre-configured evaluation ID from Qualifire dashboard |
| `prompt_injections` | `bool` | `true` (if no other checks) | Enable prompt injection detection |
| `hallucinations_check` | `bool` | `None` | Enable hallucination detection |
| `grounding_check` | `bool` | `None` | Enable grounding verification |
| `pii_check` | `bool` | `None` | Enable PII detection |
| `content_moderation_check` | `bool` | `None` | Enable content moderation |
| `tool_selection_quality_check` | `bool` | `None` | Enable tool selection quality check |
| `assertions` | `List[str]` | `None` | Custom assertions to validate |
| `on_flagged` | `str` | `"block"` | Action when content is flagged: `"block"` or `"monitor"` |
### Default Behavior
@ -261,4 +255,3 @@ This evaluates whether the LLM selected the appropriate tools and provided corre
- [Qualifire Documentation](https://docs.qualifire.ai)
- [Qualifire Dashboard](https://app.qualifire.ai)
- [Qualifire Python SDK](https://github.com/qualifire-dev/qualifire-python-sdk)

7
docs/my-website/docs/proxy/load_balancing.md
View file

@ -264,8 +264,15 @@ model_list:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
```
:::important
The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`.
:::
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
### When You'll See Load Balancing in Action

2
docs/my-website/docs/proxy/logging.md
View file

@ -67,7 +67,7 @@ Set `litellm.turn_off_message_logging=True` This will prevent the messages and r
<TabItem value="global" label="Global">
**1. Setup config.yaml **
**1. Setup config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo

18
docs/my-website/docs/proxy/pass_through.md
View file

@ -165,6 +165,7 @@ general_settings:
target: string # Target URL for forwarding
auth: boolean # Enable LiteLLM authentication (Enterprise)
forward_headers: boolean # Forward all incoming headers
include_subpath: boolean # If true, forwards requests to sub-paths (default: false)
headers: # Custom headers to add
Authorization: string # Auth header for target API
content-type: string # Request content type
@ -181,6 +182,23 @@ general_settings:
- **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration
- **Custom headers**: Any additional key-value pairs
### Sub-path Routing
By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`:
```yaml
general_settings:
pass_through_endpoints:
- path: "/custom-api" # Any path prefix you choose
target: "https://api.example.com"
include_subpath: true # Forward /custom-api/*, not just /custom-api
```
| Setting | Behavior |
|---------|----------|
| `include_subpath: false` (default) | Only `/custom-api` is forwarded |
| `include_subpath: true` | `/custom-api`, `/custom-api/v1/chat`, `/custom-api/anything` are all forwarded |
---
## Advanced: Custom Adapters

9
docs/my-website/docs/routing.md
View file

@ -861,9 +861,13 @@ model_list = [
},
]
router = Router(model_list=model_list)
router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work
```
:::important
The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router.
:::
</TabItem>
<TabItem value="proxy" label="PROXY">
@ -880,6 +884,9 @@ model_list:
model: azure/gpt-4-fallback
api_key: os.environ/AZURE_API_KEY_2
order: 2 # 👈 Used when order=1 is unavailable
router_settings:
enable_pre_call_checks: true # 👈 Required for 'order' to work
```
</TabItem>

BIN
docs/my-website/img/ui_endpoint_activity.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 503 KiB

4
docs/my-website/release_notes/v1.80.11-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.80.11 - Google Interactions API"
title: "v1.80.11 - Google Interactions API"
slug: "v1-80-11"
date: 2025-12-20T10:00:00
authors:
@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.80.11.rc.1
docker.litellm.ai/berriai/litellm:v1.80.11-stable
```
</TabItem>

643
docs/my-website/release_notes/v1.80.15/index.md Normal file
View file

@ -0,0 +1,643 @@
---
title: "v1.80.15 - Manus API Support"
slug: "v1-80-15"
date: 2026-01-10T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
docker.litellm.ai/berriai/litellm:v1.80.15.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.15
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Manus API Support** - [New provider support for Manus API on /responses and GET /responses endpoints](../../docs/providers/manus)
- **MiniMax Provider** - [Full support for MiniMax chat completions, TTS, and Anthropic native endpoint](../../docs/providers/minimax)
- **AWS Polly TTS** - [New TTS provider using AWS Polly API](../../docs/providers/aws_polly)
- **SSO Role Mapping** - Configure role mappings for SSO providers directly in the UI
- **Cost Estimator** - New UI tool for estimating costs across multiple models and requests
- **MCP Global Mode** - [Configure MCP servers globally with visibility controls](../../docs/mcp)
- **Interactions API Bridge** - [Use all LiteLLM providers with the Interactions API](../../docs/interactions)
- **RAG Query Endpoint** - [New RAG Search/Query endpoint for retrieval-augmented generation](../../docs/search/index)
- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity.md)
- **50% Overhead Reduction** - LiteLLM now sends 2.5× more requests to LLM providers
---
## Performance - 50% Overhead Reduction
LiteLLM now sends 2.5× more requests to LLM providers by replacing sequential if/elif chains with O(1) dictionary lookups for provider configuration resolution (92.7% faster). This optimization has a high impact because it runs inside the client decorator, which is invoked on every HTTP request made to the proxy server.
### Before
> **Note:** Worse-looking provider metrics are a good sign here—they indicate requests spend less time inside LiteLLM.
```
============================================================
Fake LLM Provider Stats (When called by LiteLLM)
============================================================
Total Time: 0.56s
Requests/Second: 10746.68
Latency Statistics (seconds):
Mean: 0.2039s
Median (p50): 0.2310s
Min: 0.0323s
Max: 0.3928s
Std Dev: 0.1166s
p95: 0.3574s
p99: 0.3748s
Status Codes:
200: 6000
```
### After
```
============================================================
Fake LLM Provider Stats (When called by LiteLLM)
============================================================
Total Time: 1.42s
Requests/Second: 4224.49
Latency Statistics (seconds):
Mean: 0.5300s
Median (p50): 0.5871s
Min: 0.0885s
Max: 1.0482s
Std Dev: 0.3065s
p95: 0.9750s
p99: 1.0444s
Status Codes:
200: 6000
```
> The benchmarks run LiteLLM locally with a lightweight LLM provider to eliminate network latency, isolating internal overhead and bottlenecks so we can focus on reducing pure LiteLLM overhead on a single instance.
---
### UI Usage - Endpoint Activity
<Image
img={require('../../img/ui_endpoint_activity.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
Users can now see Endpoint Activity Metrics in the UI.
---
## New Providers and Endpoints
### New Providers (11 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | ------------------- | ----------- |
| [Manus](../../docs/providers/manus) | `/responses` | Manus API for agentic workflows |
| [Manus](../../docs/providers/manus) | `GET /responses` | Manus API for retrieving responses |
| [Manus](../../docs/providers/manus) | `/files` | Manus API for file management |
| [MiniMax](../../docs/providers/minimax) | `/chat/completions` | MiniMax chat completions |
| [MiniMax](../../docs/providers/minimax) | `/audio/speech` | MiniMax text-to-speech |
| [AWS Polly](../../docs/providers/aws_polly) | `/audio/speech` | AWS Polly text-to-speech API |
| [GigaChat](../../docs/providers/gigachat) | `/chat/completions` | GigaChat provider for Russian language AI |
| [LlamaGate](../../docs/providers/llamagate) | `/chat/completions` | LlamaGate chat completions |
| [LlamaGate](../../docs/providers/llamagate) | `/embeddings` | LlamaGate embeddings |
| [Abliteration AI](../../docs/providers/abliteration) | `/chat/completions` | Abliteration.ai provider support |
| [Bedrock](../../docs/providers/bedrock) | `/v1/messages/count_tokens` | Bedrock as new provider for token counting |
### New LLM API Endpoints (3 new endpoints)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
| `/responses/compact` | POST | Compact responses API endpoint | [Docs](../../docs/response_api) |
| `/rag/query` | POST | RAG Search/Query endpoint | [Docs](../../docs/search/index) |
| `/containers/{id}/files` | POST | Upload files to containers | [Docs](../../docs/container_files) |
---
## New Models / Updated Models
#### New Model Support (100+ new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Azure | `azure/gpt-5.2` | 400K | $1.75 | $14.00 | Reasoning, vision, caching |
| Azure | `azure/gpt-5.2-chat` | 128K | $1.75 | $14.00 | Reasoning, vision |
| Azure | `azure/gpt-5.2-pro` | 400K | $21.00 | $168.00 | Reasoning, vision, web search |
| Azure | `azure/gpt-image-1.5` | - | Token-based | Token-based | Image generation/editing |
| Azure AI | `azure_ai/gpt-oss-120b` | 131K | $0.15 | $0.60 | Function calling |
| Azure AI | `azure_ai/flux.2-pro` | - | - | $0.04/image | Image generation |
| Azure AI | `azure_ai/deepseek-v3.2` | 164K | $0.58 | $1.68 | Reasoning, function calling |
| Bedrock | `amazon.nova-2-multimodal-embeddings-v1:0` | 8K | $0.135 | - | Multimodal embeddings |
| Bedrock | `writer.palmyra-x4-v1:0` | 128K | $2.50 | $10.00 | Function calling, PDF |
| Bedrock | `writer.palmyra-x5-v1:0` | 1M | $0.60 | $6.00 | Function calling, PDF |
| Bedrock | `moonshot.kimi-k2-v1:0` | - | - | - | Kimi K2 model |
| Cerebras | `cerebras/zai-glm-4.6` | 128K | $2.25 | $2.75 | Reasoning, function calling |
| GigaChat | `gigachat/GigaChat-2-Lite` | - | - | - | Chat completions |
| GigaChat | `gigachat/GigaChat-2-Max` | - | - | - | Chat completions |
| GigaChat | `gigachat/GigaChat-2-Pro` | - | - | - | Chat completions |
| Gemini | `gemini/veo-3.1-generate-001` | - | - | - | Video generation |
| Gemini | `gemini/veo-3.1-fast-generate-001` | - | - | - | Video generation |
| GitHub Copilot | 25+ models | Various | - | - | Chat completions |
| LlamaGate | 15+ models | Various | - | - | Chat, vision, embeddings |
| MiniMax | `minimax/abab7-chat-preview` | - | - | - | Chat completions |
| Novita | 80+ models | Various | Various | Various | Chat, vision, embeddings |
| OpenRouter | `openrouter/google/gemini-3-flash-preview` | - | - | - | Chat completions |
| Together AI | Multiple models | Various | Various | Various | Response schema support |
| Vertex AI | `vertex_ai/zai-glm-4.7` | - | - | - | GLM 4.7 support |
#### Features
- **[Gemini](../../docs/providers/gemini)**
- Add image tokens in chat completion - [PR #18327](https://github.com/BerriAI/litellm/pull/18327)
- Add usage object in image generation - [PR #18328](https://github.com/BerriAI/litellm/pull/18328)
- Add thought signature support via tool call id - [PR #18374](https://github.com/BerriAI/litellm/pull/18374)
- Add thought signature for non tool call requests - [PR #18581](https://github.com/BerriAI/litellm/pull/18581)
- Preserve system instructions - [PR #18585](https://github.com/BerriAI/litellm/pull/18585)
- Fix Gemini 3 images in tool response - [PR #18190](https://github.com/BerriAI/litellm/pull/18190)
- Support snake_case for google_search tool parameters - [PR #18451](https://github.com/BerriAI/litellm/pull/18451)
- Google GenAI adapter inline data support - [PR #18477](https://github.com/BerriAI/litellm/pull/18477)
- Add deprecation_date for discontinued Google models - [PR #18550](https://github.com/BerriAI/litellm/pull/18550)
- **[Vertex AI](../../docs/providers/vertex)**
- Add centralized get_vertex_base_url() helper for global location support - [PR #18410](https://github.com/BerriAI/litellm/pull/18410)
- Convert image URLs to base64 for Vertex AI Anthropic - [PR #18497](https://github.com/BerriAI/litellm/pull/18497)
- Separate Tool objects for each tool type per API spec - [PR #18514](https://github.com/BerriAI/litellm/pull/18514)
- Add thought_signatures to VertexGeminiConfig - [PR #18853](https://github.com/BerriAI/litellm/pull/18853)
- Add support for Vertex AI API keys - [PR #18806](https://github.com/BerriAI/litellm/pull/18806)
- Add zai glm-4.7 model support - [PR #18782](https://github.com/BerriAI/litellm/pull/18782)
- **[Azure](../../docs/providers/azure/azure)**
- Add Azure gpt-image-1.5 pricing to cost map - [PR #18347](https://github.com/BerriAI/litellm/pull/18347)
- Add azure/gpt-5.2-chat model - [PR #18361](https://github.com/BerriAI/litellm/pull/18361)
- Add support for image generation via Azure AD token - [PR #18413](https://github.com/BerriAI/litellm/pull/18413)
- Add logprobs support for Azure OpenAI GPT-5.2 model - [PR #18856](https://github.com/BerriAI/litellm/pull/18856)
- Add Azure BFL Flux 2 models for image generation and editing - [PR #18764](https://github.com/BerriAI/litellm/pull/18764), [PR #18766](https://github.com/BerriAI/litellm/pull/18766)
- **[Bedrock](../../docs/providers/bedrock)**
- Add Bedrock Kimi K2 model support - [PR #18797](https://github.com/BerriAI/litellm/pull/18797)
- Add support for model id in bedrock passthrough - [PR #18800](https://github.com/BerriAI/litellm/pull/18800)
- Fix Nova model detection for Bedrock provider - [PR #18250](https://github.com/BerriAI/litellm/pull/18250)
- Ensure toolUse.input is always a dict when converting from OpenAI format - [PR #18414](https://github.com/BerriAI/litellm/pull/18414)
- **[Databricks](../../docs/providers/databricks)**
- Add enhanced authentication, security features, and custom user-agent support - [PR #18349](https://github.com/BerriAI/litellm/pull/18349)
- **[MiniMax](../../docs/providers/minimax)**
- Add MiniMax chat completion support - [PR #18380](https://github.com/BerriAI/litellm/pull/18380)
- Add Anthropic native endpoint support for MiniMax - [PR #18377](https://github.com/BerriAI/litellm/pull/18377)
- Add support for MiniMax TTS - [PR #18334](https://github.com/BerriAI/litellm/pull/18334)
- Add MiniMax provider support to UI dashboard - [PR #18496](https://github.com/BerriAI/litellm/pull/18496)
- **[Together AI](../../docs/providers/togetherai)**
- Add supports_response_schema to all supported Together AI models - [PR #18368](https://github.com/BerriAI/litellm/pull/18368)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add OpenRouter embeddings API support - [PR #18391](https://github.com/BerriAI/litellm/pull/18391)
- **[Anthropic](../../docs/providers/anthropic)**
- Pass server_tool_use and tool_search_tool_result blocks - [PR #18770](https://github.com/BerriAI/litellm/pull/18770)
- Add Anthropic cache control option to image tool call results - [PR #18674](https://github.com/BerriAI/litellm/pull/18674)
- **[Ollama](../../docs/providers/ollama)**
- Add dimensions for ollama embedding - [PR #18536](https://github.com/BerriAI/litellm/pull/18536)
- Extract pure base64 data from data URLs for Ollama - [PR #18465](https://github.com/BerriAI/litellm/pull/18465)
- **[Watsonx](../../docs/providers/watsonx/index)**
- Add Watsonx fields support - [PR #18569](https://github.com/BerriAI/litellm/pull/18569)
- Fix Watsonx Audio Transcription - filter model field - [PR #18810](https://github.com/BerriAI/litellm/pull/18810)
- **[SAP](../../docs/providers/sap)**
- Add SAP creds for list in proxy UI - [PR #18375](https://github.com/BerriAI/litellm/pull/18375)
- Pass through extra params from allowed_openai_params - [PR #18432](https://github.com/BerriAI/litellm/pull/18432)
- Add client header for SAP AI Core Tracking - [PR #18714](https://github.com/BerriAI/litellm/pull/18714)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Correct deepseek-v3p2 pricing - [PR #18483](https://github.com/BerriAI/litellm/pull/18483)
- **[ZAI](../../docs/providers/zai)**
- Add GLM-4.7 model with reasoning support - [PR #18476](https://github.com/BerriAI/litellm/pull/18476)
- **[Codestral](../../docs/providers/codestral)**
- Correctly route codestral chat and FIM endpoints - [PR #18467](https://github.com/BerriAI/litellm/pull/18467)
- **[Azure AI](../../docs/providers/azure_ai)**
- Fix authentication errors at messages API via azure_ai - [PR #18500](https://github.com/BerriAI/litellm/pull/18500)
#### New Provider Support
- **[AWS Polly](../../docs/providers/aws_polly)** - Add AWS Polly API for TTS - [PR #18326](https://github.com/BerriAI/litellm/pull/18326)
- **[GigaChat](../../docs/providers/gigachat)** - Add GigaChat provider support - [PR #18564](https://github.com/BerriAI/litellm/pull/18564)
- **[LlamaGate](../../docs/providers/llamagate)** - Add LlamaGate as a new provider - [PR #18673](https://github.com/BerriAI/litellm/pull/18673)
- **[Abliteration AI](../../docs/providers/abliteration)** - Add abliteration.ai provider - [PR #18678](https://github.com/BerriAI/litellm/pull/18678)
- **[Manus](../../docs/providers/manus)** - Add Manus API support on /responses, GET /responses - [PR #18804](https://github.com/BerriAI/litellm/pull/18804)
- **5 AI Providers via openai_like** - Add 5 AI providers using openai_like - [PR #18362](https://github.com/BerriAI/litellm/pull/18362)
### Bug Fixes
- **[Gemini](../../docs/providers/gemini)**
- Properly catch context window exceeded errors - [PR #18283](https://github.com/BerriAI/litellm/pull/18283)
- Remove prompt caching headers as support has been removed - [PR #18579](https://github.com/BerriAI/litellm/pull/18579)
- Fix generate content request with audio file id - [PR #18745](https://github.com/BerriAI/litellm/pull/18745)
- Fix google_genai streaming adapter provider handling - [PR #18845](https://github.com/BerriAI/litellm/pull/18845)
- **[Groq](../../docs/providers/groq)**
- Remove deprecated Groq models and update model registry - [PR #18062](https://github.com/BerriAI/litellm/pull/18062)
- **[Vertex AI](../../docs/providers/vertex)**
- Handle unsupported region for Vertex AI count tokens endpoint - [PR #18665](https://github.com/BerriAI/litellm/pull/18665)
- **General**
- Fix request body for image embedding request - [PR #18336](https://github.com/BerriAI/litellm/pull/18336)
- Fix lost tool_calls when streaming has both text and tool_calls - [PR #18316](https://github.com/BerriAI/litellm/pull/18316)
- Add all resolution for gpt-image-1.5 - [PR #18586](https://github.com/BerriAI/litellm/pull/18586)
- Fix gpt-image-1 cost calculation using token-based pricing - [PR #17906](https://github.com/BerriAI/litellm/pull/17906)
- Fix response_format leaking into extra_body - [PR #18859](https://github.com/BerriAI/litellm/pull/18859)
- Align max_tokens with max_output_tokens for consistency - [PR #18820](https://github.com/BerriAI/litellm/pull/18820)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Add new compact endpoint (v1/responses/compact) - [PR #18697](https://github.com/BerriAI/litellm/pull/18697)
- Support more streaming callback hooks - [PR #18513](https://github.com/BerriAI/litellm/pull/18513)
- Add mapping for reasoning effort to summary param - [PR #18635](https://github.com/BerriAI/litellm/pull/18635)
- Add output_text property to ResponsesAPIResponse - [PR #18491](https://github.com/BerriAI/litellm/pull/18491)
- Add annotations to completions responses API bridge - [PR #18754](https://github.com/BerriAI/litellm/pull/18754)
- **[Interactions API](../../docs/interactions)**
- Allow using all LiteLLM providers (interactions -> responses API bridge) - [PR #18373](https://github.com/BerriAI/litellm/pull/18373)
- **[RAG Search API](../../docs/search/index)**
- Add RAG Search/Query endpoint - [PR #18376](https://github.com/BerriAI/litellm/pull/18376)
- **[CountTokens API](../../docs/anthropic_count_tokens)**
- Add Bedrock as a new provider for `/v1/messages/count_tokens` - [PR #18858](https://github.com/BerriAI/litellm/pull/18858)
- **[Generate Content](../../docs/providers/gemini)**
- Add generate content in LLM route - [PR #18405](https://github.com/BerriAI/litellm/pull/18405)
- **General**
- Enable async_post_call_failure_hook to transform error responses - [PR #18348](https://github.com/BerriAI/litellm/pull/18348)
- Calculate total_tokens manually if missing and can be calculated - [PR #18445](https://github.com/BerriAI/litellm/pull/18445)
- Add custom llm provider to get_llm_provider when sent via UI - [PR #18638](https://github.com/BerriAI/litellm/pull/18638)
#### Bugs
- **General**
- Handle empty error objects in response conversion - [PR #18493](https://github.com/BerriAI/litellm/pull/18493)
- Preserve client error status codes in streaming mode - [PR #18698](https://github.com/BerriAI/litellm/pull/18698)
- Return json error response instead of SSE format for initial streaming errors - [PR #18757](https://github.com/BerriAI/litellm/pull/18757)
- Fix auth header for custom api base in generateContent request - [PR #18637](https://github.com/BerriAI/litellm/pull/18637)
- Tool content should be string for Deepinfra - [PR #18739](https://github.com/BerriAI/litellm/pull/18739)
- Fix incomplete usage in response object passed - [PR #18799](https://github.com/BerriAI/litellm/pull/18799)
- Unify model names to provider-defined names - [PR #18573](https://github.com/BerriAI/litellm/pull/18573)
---
## Management Endpoints / UI
#### Features
- **SSO Configuration**
- Add SSO Role Mapping feature - [PR #18090](https://github.com/BerriAI/litellm/pull/18090)
- Add SSO Settings Page - [PR #18600](https://github.com/BerriAI/litellm/pull/18600)
- Allow adding role mappings for SSO - [PR #18593](https://github.com/BerriAI/litellm/pull/18593)
- SSO Settings Page Add Role Mappings - [PR #18677](https://github.com/BerriAI/litellm/pull/18677)
- SSO Settings Loading State + Deprecate Previous SSO Flow - [PR #18617](https://github.com/BerriAI/litellm/pull/18617)
- **Virtual Keys**
- Allow deleting key expiry - [PR #18278](https://github.com/BerriAI/litellm/pull/18278)
- Add optional query param "expand" to /key/list - [PR #18502](https://github.com/BerriAI/litellm/pull/18502)
- Key Table Loading Skeleton - [PR #18527](https://github.com/BerriAI/litellm/pull/18527)
- Allow column resizing on Keys Table - [PR #18424](https://github.com/BerriAI/litellm/pull/18424)
- Virtual Keys Table Loading State Between Pages - [PR #18619](https://github.com/BerriAI/litellm/pull/18619)
- Key and Team Router Setting - [PR #18790](https://github.com/BerriAI/litellm/pull/18790)
- Allow router_settings on Keys and Teams - [PR #18675](https://github.com/BerriAI/litellm/pull/18675)
- Use timedelta to calculate key expiry on generate - [PR #18666](https://github.com/BerriAI/litellm/pull/18666)
- **Models + Endpoints**
- Add Model Clearer Flow For Team Admins - [PR #18532](https://github.com/BerriAI/litellm/pull/18532)
- Model Page Loading State - [PR #18574](https://github.com/BerriAI/litellm/pull/18574)
- Model Page Model Provider Select Performance - [PR #18425](https://github.com/BerriAI/litellm/pull/18425)
- Model Page Sorting Sorts Entire Set - [PR #18420](https://github.com/BerriAI/litellm/pull/18420)
- Refactor Model Hub Page - [PR #18568](https://github.com/BerriAI/litellm/pull/18568)
- Add request provider form on UI - [PR #18704](https://github.com/BerriAI/litellm/pull/18704)
- **Organizations & Teams**
- Allow Organization Admins to See Organization Tab - [PR #18400](https://github.com/BerriAI/litellm/pull/18400)
- Resolve Organization Alias on Team Table - [PR #18401](https://github.com/BerriAI/litellm/pull/18401)
- Resolve Team Alias in Organization Info View - [PR #18404](https://github.com/BerriAI/litellm/pull/18404)
- Allow Organization Admins to View Their Organization Info - [PR #18417](https://github.com/BerriAI/litellm/pull/18417)
- Allow editing team_member_budget_duration in /team/update - [PR #18735](https://github.com/BerriAI/litellm/pull/18735)
- Reusable Duration Select + Team Update Member Budget Duration - [PR #18736](https://github.com/BerriAI/litellm/pull/18736)
- **Usage & Spend**
- Add Error Code Filtering on Spend Logs - [PR #18359](https://github.com/BerriAI/litellm/pull/18359)
- Add Error Code Filtering on UI - [PR #18366](https://github.com/BerriAI/litellm/pull/18366)
- Usage Page User Max Budget fix - [PR #18555](https://github.com/BerriAI/litellm/pull/18555)
- Add endpoint to Daily Activity Tables - [PR #18729](https://github.com/BerriAI/litellm/pull/18729)
- Endpoint Activity in Usage - [PR #18798](https://github.com/BerriAI/litellm/pull/18798)
- **Cost Estimator**
- Add Cost Estimator for AI Gateway - [PR #18643](https://github.com/BerriAI/litellm/pull/18643)
- Add view for estimating costs across requests - [PR #18645](https://github.com/BerriAI/litellm/pull/18645)
- Allow selecting many models for cost estimator - [PR #18653](https://github.com/BerriAI/litellm/pull/18653)
- **CloudZero**
- Improve Create and Delete Path for CloudZero - [PR #18263](https://github.com/BerriAI/litellm/pull/18263)
- Add CloudZero UI Docs - [PR #18350](https://github.com/BerriAI/litellm/pull/18350)
- **Playground**
- Add MCP test support to completions on Playground - [PR #18440](https://github.com/BerriAI/litellm/pull/18440)
- Add selectable MCP servers to the playground - [PR #18578](https://github.com/BerriAI/litellm/pull/18578)
- Add custom proxy base URL support to Playground - [PR #18661](https://github.com/BerriAI/litellm/pull/18661)
- **General UI**
- UI styling improvements and fixes - [PR #18310](https://github.com/BerriAI/litellm/pull/18310)
- Add reusable "New" badge component for feature highlights - [PR #18537](https://github.com/BerriAI/litellm/pull/18537)
- Hide New Badges - [PR #18547](https://github.com/BerriAI/litellm/pull/18547)
- Change Budget page to Have Tabs - [PR #18576](https://github.com/BerriAI/litellm/pull/18576)
- Clicking on Logo Directs to Correct URL - [PR #18575](https://github.com/BerriAI/litellm/pull/18575)
- Add UI support for configuring meta URLs - [PR #18580](https://github.com/BerriAI/litellm/pull/18580)
- Expire Previous UI Session Tokens on Login - [PR #18557](https://github.com/BerriAI/litellm/pull/18557)
- Add license endpoint - [PR #18311](https://github.com/BerriAI/litellm/pull/18311)
- Router Fields Endpoint + React Query for Router Fields - [PR #18880](https://github.com/BerriAI/litellm/pull/18880)
#### Bugs
- **UI Fixes**
- Fix Key Creation MCP Settings Submit Form Unintentionally - [PR #18355](https://github.com/BerriAI/litellm/pull/18355)
- Fix UI Disappears in Development Environments - [PR #18399](https://github.com/BerriAI/litellm/pull/18399)
- Fix Disable Admin UI Flag - [PR #18397](https://github.com/BerriAI/litellm/pull/18397)
- Remove Model Analytics From Model Page - [PR #18552](https://github.com/BerriAI/litellm/pull/18552)
- Useful Links Remove Modal on Adding Links - [PR #18602](https://github.com/BerriAI/litellm/pull/18602)
- SSO Edit Modal Clear Role Mapping Values on Provider Change - [PR #18680](https://github.com/BerriAI/litellm/pull/18680)
- UI Login Case Sensitivity fix - [PR #18877](https://github.com/BerriAI/litellm/pull/18877)
- **API Fixes**
- Fix User Invite & Key Generation Email Notification Logic - [PR #18524](https://github.com/BerriAI/litellm/pull/18524)
- Normalize Proxy Config Callback - [PR #18775](https://github.com/BerriAI/litellm/pull/18775)
- Return empty data array instead of 500 when no models configured - [PR #18556](https://github.com/BerriAI/litellm/pull/18556)
- Enforce org level max budget - [PR #18813](https://github.com/BerriAI/litellm/pull/18813)
---
## AI Integrations
### New Integrations (4 new integrations)
| Integration | Type | Description |
| ----------- | ---- | ----------- |
| [Focus](../../docs/observability/focus) | Logging | Focus export support for observability - [PR #18802](https://github.com/BerriAI/litellm/pull/18802) |
| [SigNoz](../../docs/observability/signoz) | Logging | SigNoz integration for observability - [PR #18726](https://github.com/BerriAI/litellm/pull/18726) |
| [Qualifire](../../docs/proxy/guardrails/qualifire) | Guardrails | Qualifire guardrails and eval webhook - [PR #18594](https://github.com/BerriAI/litellm/pull/18594) |
| [Levo AI](../../docs/observability/levo_integration) | Guardrails | Levo AI integration for security - [PR #18529](https://github.com/BerriAI/litellm/pull/18529) |
### Logging
- **[DataDog](../../docs/proxy/logging#datadog)**
- Fix span kind fallback when parent_id missing - [PR #18418](https://github.com/BerriAI/litellm/pull/18418)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Map Gemini cached_tokens to Langfuse cache_read_input_tokens - [PR #18614](https://github.com/BerriAI/litellm/pull/18614)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Align prometheus metric names with DEFINED_PROMETHEUS_METRICS - [PR #18463](https://github.com/BerriAI/litellm/pull/18463)
- Add Prometheus metrics for request queue time and guardrails - [PR #17973](https://github.com/BerriAI/litellm/pull/17973)
- Add caching metrics for cache hits, misses, and tokens - [PR #18755](https://github.com/BerriAI/litellm/pull/18755)
- Skip metrics for invalid API key requests - [PR #18788](https://github.com/BerriAI/litellm/pull/18788)
- **[Braintrust](../../docs/proxy/logging#braintrust)**
- Pass span_attributes in async logging and skip tags on non-root spans - [PR #18409](https://github.com/BerriAI/litellm/pull/18409)
- **[CloudZero](../../docs/proxy/logging#cloudzero)**
- Add user email to CloudZero - [PR #18584](https://github.com/BerriAI/litellm/pull/18584)
- **[OpenTelemetry](../../docs/proxy/logging#opentelemetry)**
- Use already configured opentelemetry providers - [PR #18279](https://github.com/BerriAI/litellm/pull/18279)
- Prevent LiteLLM from closing external OTEL spans - [PR #18553](https://github.com/BerriAI/litellm/pull/18553)
- Allow configuring arize project name for OpenTelemetry service name - [PR #18738](https://github.com/BerriAI/litellm/pull/18738)
- **[LangSmith](../../docs/proxy/logging#langsmith)**
- Add support for LangSmith organization-scoped API keys with tenant ID - [PR #18623](https://github.com/BerriAI/litellm/pull/18623)
- **[Generic API Logger](../../docs/proxy/logging#generic-api-logger)**
- Add log_format option to GenericAPILogger - [PR #18587](https://github.com/BerriAI/litellm/pull/18587)
### Guardrails
- **[Content Filter](../../docs/proxy/guardrails/litellm_content_filter)**
- Add content filter logs page - [PR #18335](https://github.com/BerriAI/litellm/pull/18335)
- Log actual event type for guardrails - [PR #18489](https://github.com/BerriAI/litellm/pull/18489)
- **[Qualifire](../../docs/proxy/guardrails/qualifire)**
- Add Qualifire eval webhook - [PR #18836](https://github.com/BerriAI/litellm/pull/18836)
- **[Lasso Security](../../docs/proxy/guardrails/lasso_security)**
- Add Lasso guardrail API docs - [PR #18652](https://github.com/BerriAI/litellm/pull/18652)
- **[Noma Security](../../docs/proxy/guardrails/noma_security)**
- Add MCP guardrail support for Noma - [PR #18668](https://github.com/BerriAI/litellm/pull/18668)
- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)**
- Remove redundant Bedrock guardrail block handling - [PR #18634](https://github.com/BerriAI/litellm/pull/18634)
- **General**
- Generic guardrail API update - [PR #18647](https://github.com/BerriAI/litellm/pull/18647)
- Prevent proxy startup failures from case-sensitive tool permission guardrail validation - [PR #18662](https://github.com/BerriAI/litellm/pull/18662)
- Extend case normalization to ALL guardrail types - [PR #18664](https://github.com/BerriAI/litellm/pull/18664)
- Fix MCP handling in unified guardrail - [PR #18630](https://github.com/BerriAI/litellm/pull/18630)
- Fix embeddings calltype for guardrail precallhook - [PR #18740](https://github.com/BerriAI/litellm/pull/18740)
---
## Spend Tracking, Budgets and Rate Limiting
- **Platform Fee / Margins** - Add support for Platform Fee / Margins - [PR #18427](https://github.com/BerriAI/litellm/pull/18427)
- **Negative Budget Validation** - Add validation for negative budget - [PR #18583](https://github.com/BerriAI/litellm/pull/18583)
- **Cost Calculation Fixes**
- Correct cost calculation when reasoning_tokens are without text_tokens - [PR #18607](https://github.com/BerriAI/litellm/pull/18607)
- Fix background cost tracking tests - [PR #18588](https://github.com/BerriAI/litellm/pull/18588)
- **Tag Routing** - Support toggling tag matching between ANY and ALL - [PR #18776](https://github.com/BerriAI/litellm/pull/18776)
---
## MCP Gateway
- **MCP Global Mode** - Add MCP global mode - [PR #18639](https://github.com/BerriAI/litellm/pull/18639)
- **MCP Server Visibility** - Add configurable MCP server visibility - [PR #18681](https://github.com/BerriAI/litellm/pull/18681)
- **MCP Registry** - Add MCP registry - [PR #18850](https://github.com/BerriAI/litellm/pull/18850)
- **MCP Stdio Header** - Support MCP stdio header env overrides - [PR #18324](https://github.com/BerriAI/litellm/pull/18324)
- **Parallel Tool Fetching** - Parallelize tool fetching from multiple MCP servers - [PR #18627](https://github.com/BerriAI/litellm/pull/18627)
- **Optimize MCP Server Listing** - Separate health checks for optimized listing - [PR #18530](https://github.com/BerriAI/litellm/pull/18530)
- **Auth Improvements**
- Require auth for MCP connection test endpoint - [PR #18290](https://github.com/BerriAI/litellm/pull/18290)
- Fix MCP gateway OAuth2 auth issues and ClosedResourceError - [PR #18281](https://github.com/BerriAI/litellm/pull/18281)
- **Bug Fixes**
- Fix MCP server health status reporting - [PR #18443](https://github.com/BerriAI/litellm/pull/18443)
- Fix OpenAPI to MCP tool conversion - [PR #18597](https://github.com/BerriAI/litellm/pull/18597)
- Remove exec() usage and handle invalid OpenAPI parameter names for security - [PR #18480](https://github.com/BerriAI/litellm/pull/18480)
- Fix MCP error when using multiple servers simultaneously - [PR #18855](https://github.com/BerriAI/litellm/pull/18855)
- **Migrate MCP Fetching Logic to React Query** - [PR #18352](https://github.com/BerriAI/litellm/pull/18352)
---
## Performance / Loadbalancing / Reliability improvements
- **92.7% Faster Provider Config Lookup** - LiteLLM now stresses LLM providers 2.5x more - [PR #18867](https://github.com/BerriAI/litellm/pull/18867)
- **Lazy Loading Improvements**
- Consolidate lazy import handlers with registry pattern - [PR #18389](https://github.com/BerriAI/litellm/pull/18389)
- Complete lazy loading migration for all 180+ LLM config classes - [PR #18392](https://github.com/BerriAI/litellm/pull/18392)
- Lazy load additional components (types, callbacks, utilities) - [PR #18396](https://github.com/BerriAI/litellm/pull/18396)
- Add lazy loading for get_llm_provider - [PR #18591](https://github.com/BerriAI/litellm/pull/18591)
- Lazy-load heavy audio library and loggers - [PR #18592](https://github.com/BerriAI/litellm/pull/18592)
- Lazy load 9 heavy imports in litellm/utils.py - [PR #18595](https://github.com/BerriAI/litellm/pull/18595)
- Lazy load heavy imports to improve import time and memory usage - [PR #18610](https://github.com/BerriAI/litellm/pull/18610)
- Implement lazy loading for provider configs, model info classes, streaming handlers - [PR #18611](https://github.com/BerriAI/litellm/pull/18611)
- Lazy load 15 additional imports - [PR #18613](https://github.com/BerriAI/litellm/pull/18613)
- Lazy load 15+ unused imports - [PR #18616](https://github.com/BerriAI/litellm/pull/18616)
- Lazy load DatadogLLMObsInitParams - [PR #18658](https://github.com/BerriAI/litellm/pull/18658)
- Migrate utils.py lazy imports to registry pattern - [PR #18657](https://github.com/BerriAI/litellm/pull/18657)
- Lazy load get_llm_provider and remove_index_from_tool_calls - [PR #18608](https://github.com/BerriAI/litellm/pull/18608)
- **Router Improvements**
- Validate routing_strategy at startup to fail fast with helpful error - [PR #18624](https://github.com/BerriAI/litellm/pull/18624)
- Correct num_retries tracking in retry logic - [PR #18712](https://github.com/BerriAI/litellm/pull/18712)
- Improve error messages and validation for wildcard routing with multiple credentials - [PR #18629](https://github.com/BerriAI/litellm/pull/18629)
- **Memory Improvements**
- Add memory pattern detection test and fix bad memory patterns - [PR #18589](https://github.com/BerriAI/litellm/pull/18589)
- Add unbounded data structure detection to memory test - [PR #18590](https://github.com/BerriAI/litellm/pull/18590)
- Add memory leak detection tests with CI integration - [PR #18881](https://github.com/BerriAI/litellm/pull/18881)
- **Database**
- Add idx on LOWER(user_email) for faster duplicate email checks - [PR #18828](https://github.com/BerriAI/litellm/pull/18828)
- Proactive RDS IAM token refresh to prevent 15-min connection failed - [PR #18795](https://github.com/BerriAI/litellm/pull/18795)
- Clarify database_connection_pool_limit applies per worker - [PR #18780](https://github.com/BerriAI/litellm/pull/18780)
- Make base_connection_pool_limit default value the same - [PR #18721](https://github.com/BerriAI/litellm/pull/18721)
- **Docker**
- Add libsndfile to database Docker image for audio processing - [PR #18612](https://github.com/BerriAI/litellm/pull/18612)
- Add line_profiler support for performance analysis and fix Windows CRLF issues - [PR #18773](https://github.com/BerriAI/litellm/pull/18773)
- **Helm**
- Add lifecycle support to Helm charts - [PR #18517](https://github.com/BerriAI/litellm/pull/18517)
- **Authentication**
- Add Kubernetes ServiceAccount JWT authentication support - [PR #18055](https://github.com/BerriAI/litellm/pull/18055)
- Use async anthropic client to prevent event loop blocking - [PR #18435](https://github.com/BerriAI/litellm/pull/18435)
- **Logging Worker**
- Handle event loop changes in multiprocessing - [PR #18423](https://github.com/BerriAI/litellm/pull/18423)
- **Security**
- Prevent expired key plaintext leak in error response - [PR #18860](https://github.com/BerriAI/litellm/pull/18860)
- Mask extra header secrets in model info - [PR #18822](https://github.com/BerriAI/litellm/pull/18822)
- Prevent duplicate User-Agent tags in request_tags - [PR #18723](https://github.com/BerriAI/litellm/pull/18723)
- Properly use litellm api keys - [PR #18832](https://github.com/BerriAI/litellm/pull/18832)
- **Misc**
- Remove double imports in main.py - [PR #18406](https://github.com/BerriAI/litellm/pull/18406)
- Add LITELLM_DISABLE_LAZY_LOADING env var to fix VCR cassette creation issue - [PR #18725](https://github.com/BerriAI/litellm/pull/18725)
- Add xiaomi_mimo to LlmProviders enum to fix router support - [PR #18819](https://github.com/BerriAI/litellm/pull/18819)
- Allow installation with current grpcio on old Python - [PR #18473](https://github.com/BerriAI/litellm/pull/18473)
- Add Custom CA certificates to boto3 clients - [PR #18852](https://github.com/BerriAI/litellm/pull/18852)
- Fix bedrock_cache, metadata and max_model_budget - [PR #18872](https://github.com/BerriAI/litellm/pull/18872)
- Fix LiteLLM SDK embedding headers missing field - [PR #18844](https://github.com/BerriAI/litellm/pull/18844)
- Put automatic reasoning summary inclusion behind feat flag - [PR #18688](https://github.com/BerriAI/litellm/pull/18688)
- turn_off_message_logging Does Not Redact Request Messages in proxy_server_request Field - [PR #18897](https://github.com/BerriAI/litellm/pull/18897)
---
## Documentation Updates
- **Provider Documentation**
- Update MiniMax docs to be in proper format - [PR #18403](https://github.com/BerriAI/litellm/pull/18403)
- Add docs for 5 AI providers - [PR #18388](https://github.com/BerriAI/litellm/pull/18388)
- Fix gpt-5-mini reasoning_effort supported values - [PR #18346](https://github.com/BerriAI/litellm/pull/18346)
- Fix PDF documentation inconsistency in Anthropic page - [PR #18816](https://github.com/BerriAI/litellm/pull/18816)
- Update OpenRouter docs to include embedding support - [PR #18874](https://github.com/BerriAI/litellm/pull/18874)
- Add LITELLM_REASONING_AUTO_SUMMARY in doc - [PR #18705](https://github.com/BerriAI/litellm/pull/18705)
- **MCP Documentation**
- Agentcore MCP server docs - [PR #18603](https://github.com/BerriAI/litellm/pull/18603)
- Mention MCP prompt/resources types in overview - [PR #18669](https://github.com/BerriAI/litellm/pull/18669)
- Add Focus docs - [PR #18837](https://github.com/BerriAI/litellm/pull/18837)
- **Guardrails Documentation**
- Qualifire docs hotfix - [PR #18724](https://github.com/BerriAI/litellm/pull/18724)
- **Infrastructure Documentation**
- IAM Roles Anywhere docs - [PR #18559](https://github.com/BerriAI/litellm/pull/18559)
- Fix formatting in proxy configs documentation - [PR #18498](https://github.com/BerriAI/litellm/pull/18498)
- Fix GCS cache docs missing for proxy mode - [PR #13328](https://github.com/BerriAI/litellm/pull/13328)
- Fix how to execute cloudzero sql - [PR #18841](https://github.com/BerriAI/litellm/pull/18841)
- **General**
- LiteLLM adopters section - [PR #18605](https://github.com/BerriAI/litellm/pull/18605)
- Remove redundant comments about setting litellm.callbacks - [PR #18711](https://github.com/BerriAI/litellm/pull/18711)
- Update header to be markdown bold by removing space - [PR #18846](https://github.com/BerriAI/litellm/pull/18846)
- Manus docs - new provider - [PR #18817](https://github.com/BerriAI/litellm/pull/18817)
---
## New Contributors
* @prasadkona made their first contribution in [PR #18349](https://github.com/BerriAI/litellm/pull/18349)
* @lucasrothman made their first contribution in [PR #18283](https://github.com/BerriAI/litellm/pull/18283)
* @aggeentik made their first contribution in [PR #18317](https://github.com/BerriAI/litellm/pull/18317)
* @mihidumh made their first contribution in [PR #18361](https://github.com/BerriAI/litellm/pull/18361)
* @Prazeina made their first contribution in [PR #18498](https://github.com/BerriAI/litellm/pull/18498)
* @systec-dk made their first contribution in [PR #18500](https://github.com/BerriAI/litellm/pull/18500)
* @xuan07t2 made their first contribution in [PR #18514](https://github.com/BerriAI/litellm/pull/18514)
* @RensDimmendaal made their first contribution in [PR #18190](https://github.com/BerriAI/litellm/pull/18190)
* @yurekami made their first contribution in [PR #18483](https://github.com/BerriAI/litellm/pull/18483)
* @agertz7 made their first contribution in [PR #18556](https://github.com/BerriAI/litellm/pull/18556)
* @yudelevi made their first contribution in [PR #18550](https://github.com/BerriAI/litellm/pull/18550)
* @smallp made their first contribution in [PR #18536](https://github.com/BerriAI/litellm/pull/18536)
* @kevinpauer made their first contribution in [PR #18569](https://github.com/BerriAI/litellm/pull/18569)
* @cansakiroglu made their first contribution in [PR #18517](https://github.com/BerriAI/litellm/pull/18517)
* @dee-walia20 made their first contribution in [PR #18432](https://github.com/BerriAI/litellm/pull/18432)
* @luxinfeng made their first contribution in [PR #18477](https://github.com/BerriAI/litellm/pull/18477)
* @cantalupo555 made their first contribution in [PR #18476](https://github.com/BerriAI/litellm/pull/18476)
* @andersk made their first contribution in [PR #18473](https://github.com/BerriAI/litellm/pull/18473)
* @majiayu000 made their first contribution in [PR #18467](https://github.com/BerriAI/litellm/pull/18467)
* @amangupta-20 made their first contribution in [PR #18529](https://github.com/BerriAI/litellm/pull/18529)
* @hamzaq453 made their first contribution in [PR #18480](https://github.com/BerriAI/litellm/pull/18480)
* @ktsaou made their first contribution in [PR #18627](https://github.com/BerriAI/litellm/pull/18627)
* @FlibbertyGibbitz made their first contribution in [PR #18624](https://github.com/BerriAI/litellm/pull/18624)
* @drorIvry made their first contribution in [PR #18594](https://github.com/BerriAI/litellm/pull/18594)
* @urainshah made their first contribution in [PR #18524](https://github.com/BerriAI/litellm/pull/18524)
* @mangabits made their first contribution in [PR #18279](https://github.com/BerriAI/litellm/pull/18279)
* @0717376 made their first contribution in [PR #18564](https://github.com/BerriAI/litellm/pull/18564)
* @nmgarza5 made their first contribution in [PR #17330](https://github.com/BerriAI/litellm/pull/17330)
* @wileykestner made their first contribution in [PR #18445](https://github.com/BerriAI/litellm/pull/18445)
* @minijeong-log made their first contribution in [PR #14440](https://github.com/BerriAI/litellm/pull/14440)
* @Isaac4real made their first contribution in [PR #18710](https://github.com/BerriAI/litellm/pull/18710)
* @marukaz made their first contribution in [PR #18711](https://github.com/BerriAI/litellm/pull/18711)
* @rohitravirane made their first contribution in [PR #18712](https://github.com/BerriAI/litellm/pull/18712)
* @lizzzcai made their first contribution in [PR #18714](https://github.com/BerriAI/litellm/pull/18714)
* @hkd987 made their first contribution in [PR #18673](https://github.com/BerriAI/litellm/pull/18673)
* @Mr-Pepe made their first contribution in [PR #18674](https://github.com/BerriAI/litellm/pull/18674)
* @gkarthi-signoz made their first contribution in [PR #18726](https://github.com/BerriAI/litellm/pull/18726)
* @Tianduo16 made their first contribution in [PR #18723](https://github.com/BerriAI/litellm/pull/18723)
* @wilsonjr made their first contribution in [PR #18721](https://github.com/BerriAI/litellm/pull/18721)
* @abliteration-ai made their first contribution in [PR #18678](https://github.com/BerriAI/litellm/pull/18678)
* @danialkhan02 made their first contribution in [PR #18770](https://github.com/BerriAI/litellm/pull/18770)
* @ihower made their first contribution in [PR #18409](https://github.com/BerriAI/litellm/pull/18409)
* @elkkhan made their first contribution in [PR #18391](https://github.com/BerriAI/litellm/pull/18391)
* @runixer made their first contribution in [PR #18435](https://github.com/BerriAI/litellm/pull/18435)
* @choby-shun made their first contribution in [PR #18776](https://github.com/BerriAI/litellm/pull/18776)
* @jutaz made their first contribution in [PR #18853](https://github.com/BerriAI/litellm/pull/18853)
* @sjmatta made their first contribution in [PR #18250](https://github.com/BerriAI/litellm/pull/18250)
* @andres-ortizl made their first contribution in [PR #18856](https://github.com/BerriAI/litellm/pull/18856)
* @gauthiermartin made their first contribution in [PR #18844](https://github.com/BerriAI/litellm/pull/18844)
* @mel2oo made their first contribution in [PR #18845](https://github.com/BerriAI/litellm/pull/18845)
* @DominikHallab made their first contribution in [PR #18846](https://github.com/BerriAI/litellm/pull/18846)
* @ji-chuan-che made their first contribution in [PR #18540](https://github.com/BerriAI/litellm/pull/18540)
* @raghav-stripe made their first contribution in [PR #18858](https://github.com/BerriAI/litellm/pull/18858)
* @akraines made their first contribution in [PR #18629](https://github.com/BerriAI/litellm/pull/18629)
* @otaviofbrito made their first contribution in [PR #18665](https://github.com/BerriAI/litellm/pull/18665)
* @chetanchoudhary-sumo made their first contribution in [PR #18587](https://github.com/BerriAI/litellm/pull/18587)
* @pascalwhoop made their first contribution in [PR #13328](https://github.com/BerriAI/litellm/pull/13328)
* @orgersh92 made their first contribution in [PR #18652](https://github.com/BerriAI/litellm/pull/18652)
* @DevajMody made their first contribution in [PR #18497](https://github.com/BerriAI/litellm/pull/18497)
* @matt-greathouse made their first contribution in [PR #18247](https://github.com/BerriAI/litellm/pull/18247)
* @emerzon made their first contribution in [PR #18290](https://github.com/BerriAI/litellm/pull/18290)
* @Eric84626 made their first contribution in [PR #18281](https://github.com/BerriAI/litellm/pull/18281)
* @LukasdeBoer made their first contribution in [PR #18055](https://github.com/BerriAI/litellm/pull/18055)
* @LingXuanYin made their first contribution in [PR #18513](https://github.com/BerriAI/litellm/pull/18513)
* @krisxia0506 made their first contribution in [PR #18698](https://github.com/BerriAI/litellm/pull/18698)
* @LouisShark made their first contribution in [PR #18414](https://github.com/BerriAI/litellm/pull/18414)
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.14.rc.1)**

15
docs/my-website/sidebars.js
View file

@ -55,6 +55,7 @@ const sidebars = {
"proxy/guardrails/test_playground",
"proxy/guardrails/litellm_content_filter",
...[
"proxy/guardrails/qualifire",
"proxy/guardrails/aim_security",
"proxy/guardrails/onyx_security",
"proxy/guardrails/aporia_api",
@ -653,12 +654,13 @@ const sidebars = {
"providers/bedrock_writer",
"providers/bedrock_batches",
"providers/aws_polly",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/ai21",
"providers/aiml",
"providers/bedrock_vector_store",
]
},
"providers/litellm_proxy",
"providers/abliteration",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
@ -710,6 +712,7 @@ const sidebars = {
"providers/llamafile",
"providers/llamagate",
"providers/lm_studio",
"providers/manus",
"providers/meta_llama",
"providers/milvus_vector_stores",
"providers/mistral",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.21-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.21.tar.gz vendored Normal file
View file

Binary file not shown.

9
litellm-proxy-extras/litellm_proxy_extras/migrations/20260108_add_user_email_lower_idx/migration.sql Normal file
View file

@ -0,0 +1,9 @@
-- CreateIndex
-- Fixes performance issue in _check_duplicate_user_email function
-- by enabling fast case-insensitive email lookups.
--
-- Without this index, queries with mode: "insensitive" cause full table scans.
-- With this index, PostgreSQL can use an Index Scan for O(log n) performance.
--
-- Related: GitHub Issue #18411
CREATE INDEX "LiteLLM_UserTable_user_email_lower_idx" ON "LiteLLM_UserTable"(LOWER("user_email"));

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.20"
version = "0.4.21"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.20"
version = "0.4.21"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

12
litellm/__init__.py
View file

@ -9,7 +9,7 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
warnings.filterwarnings(
"ignore", message=".*Accessing the.*attribute on the instance is deprecated.*"
)
### INIT VARIABLES #######################
### INIT VARIABLES ########################
import threading
import os
from typing import (
@ -134,6 +134,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"bitbucket",
"gitlab",
"cloudzero",
"focus",
"posthog",
"levo",
]
@ -486,6 +487,7 @@ vertex_mistral_models: Set = set()
vertex_openai_models: Set = set()
vertex_minimax_models: Set = set()
vertex_moonshot_models: Set = set()
vertex_zai_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
@ -664,6 +666,9 @@ def add_known_models():
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
key = key.replace("vertex_ai/", "")
vertex_moonshot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-zai_models":
key = key.replace("vertex_ai/", "")
vertex_zai_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.add(key)
@ -950,7 +955,8 @@ models_by_provider: dict = {
| vertex_language_models
| vertex_deepseek_models
| vertex_minimax_models
| vertex_moonshot_models,
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
@ -1338,6 +1344,7 @@ if TYPE_CHECKING:
from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig
from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config
from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig
from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig
from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig
from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig
@ -1367,6 +1374,7 @@ if TYPE_CHECKING:
from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig
from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig

4
litellm/_lazy_imports_registry.py
View file

@ -165,6 +165,7 @@ LLM_CONFIG_NAMES = (
"AmazonLlamaConfig",
"AmazonDeepSeekR1Config",
"AmazonMistralConfig",
"AmazonMoonshotConfig",
"AmazonTitanConfig",
"AmazonTwelveLabsPegasusConfig",
"AmazonInvokeConfig",
@ -252,6 +253,7 @@ LLM_CONFIG_NAMES = (
"IBMWatsonXAudioTranscriptionConfig",
"GithubCopilotConfig",
"GithubCopilotResponsesAPIConfig",
"ManusResponsesAPIConfig",
"GithubCopilotEmbeddingConfig",
"NebiusConfig",
"WandbConfig",
@ -556,6 +558,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"AmazonLlamaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", "AmazonLlamaConfig"),
"AmazonDeepSeekR1Config": (".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", "AmazonDeepSeekR1Config"),
"AmazonMistralConfig": (".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", "AmazonMistralConfig"),
"AmazonMoonshotConfig": (".llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation", "AmazonMoonshotConfig"),
"AmazonTitanConfig": (".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", "AmazonTitanConfig"),
"AmazonTwelveLabsPegasusConfig": (".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", "AmazonTwelveLabsPegasusConfig"),
"AmazonInvokeConfig": (".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", "AmazonInvokeConfig"),
@ -588,6 +591,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
"AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"),
"XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"),
"LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"),
"ManusResponsesAPIConfig": (".llms.manus.responses.transformation", "ManusResponsesAPIConfig"),
"GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"),
"OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"),
"AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"),

48
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -31,6 +31,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionToolParamFunctionChunk,
Reasoning,
ResponsesAPIOptionalRequestParams,
@ -90,9 +91,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
content_type = content_item.get("type")
if content_type == "output_text":
response_text = content_item.get("text", "")
# Extract annotations from content if present
annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format(
content_item.get("annotations", None)
)
msg = Message(
role=item.get("role", "assistant"),
content=response_text if response_text else "",
annotations=annotations,
)
choice = Choices(message=msg, finish_reason="stop", index=index)
return choice, index + 1
@ -364,10 +370,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
elif isinstance(item, ResponseOutputMessage):
for content in item.content:
response_text = getattr(content, "text", "")
# Extract annotations from content if present
raw_annotations = getattr(content, "annotations", None)
annotations = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format(
raw_annotations
)
msg = Message(
role=item.role,
content=response_text if response_text else "",
reasoning_content=reasoning_content,
annotations=annotations,
)
choices.append(
@ -763,6 +775,42 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return {"format": {"type": "text"}}
return None
@staticmethod
def _convert_annotations_to_chat_format(
annotations: Optional[List[Any]],
) -> Optional[List["ChatCompletionAnnotation"]]:
"""
Convert annotations from Responses API to Chat Completions format.
Annotations are already in compatible format between both APIs,
so we just need to convert Pydantic models to dicts.
"""
if not annotations:
return None
result: List[ChatCompletionAnnotation] = []
for annotation in annotations:
try:
# Convert Pydantic models to dicts (handles both v1 and v2)
if hasattr(annotation, "model_dump"):
annotation_dict = annotation.model_dump()
elif hasattr(annotation, "dict"):
annotation_dict = annotation.dict()
elif isinstance(annotation, dict):
annotation_dict = annotation
else:
# Skip unsupported annotation types
verbose_logger.debug(f"Skipping unsupported annotation type: {type(annotation)}")
continue
result.append(annotation_dict) # type: ignore
except Exception as e:
# Skip malformed annotations
verbose_logger.debug(f"Skipping malformed annotation: {annotation}, error: {e}")
continue
return result if result else None
def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str:
"""Map responses API status to chat completion finish_reason"""

1
litellm/constants.py
View file

@ -909,6 +909,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"twelvelabs",
"openai",
"stability",
"moonshot",
]
BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[

198
litellm/files/main.py
View file

@ -8,6 +8,7 @@ https://platform.openai.com/docs/api-reference/files
import asyncio
import contextvars
import os
import time
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
@ -60,7 +61,7 @@ async def acreate_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -105,7 +106,7 @@ def create_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
expires_after: Optional[FileExpiresAfter] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None,
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "manus"]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -274,7 +275,7 @@ def create_file(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai'] are supported.".format(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai', 'manus'] are supported.".format(
custom_llm_provider
),
model="n/a",
@ -293,7 +294,7 @@ def create_file(
@client
async def afile_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -334,7 +335,7 @@ async def afile_retrieve(
@client
def file_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -428,18 +429,60 @@ def file_retrieve(
file_id=file_id,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_retrieve'. Only 'openai' and 'azure' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
# Try using provider config pattern (for Manus, Bedrock, etc.)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict = get_litellm_params(**kwargs)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_retrieve" if _is_async else "file_retrieve",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id") or ""),
)
client = kwargs.get("client")
response = base_llm_http_handler.retrieve_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_retrieve'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
return cast(FileObject, response)
except Exception as e:
@ -450,7 +493,7 @@ def file_retrieve(
@client
async def afile_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -494,7 +537,7 @@ async def afile_delete(
def file_delete(
file_id: str,
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
custom_llm_provider: Union[Literal["openai", "azure", "manus"], str] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -596,18 +639,58 @@ def file_delete(
litellm_params=litellm_params_dict,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
# Try using provider config pattern (for Manus, Bedrock, etc.)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_delete" if _is_async else "file_delete",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id") or ""),
)
response = base_llm_http_handler.delete_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_delete'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
return cast(FileDeleted, response)
except Exception as e:
raise e
@ -616,7 +699,7 @@ def file_delete(
# List files
@client
async def afile_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
purpose: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -657,7 +740,7 @@ async def afile_list(
@client
def file_list(
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "manus"] = "openai",
purpose: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -687,7 +770,50 @@ def file_list(
timeout = 600.0
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# Check if provider has a custom files config (e.g., Manus, Bedrock, Vertex AI)
provider_config = ProviderConfigManager.get_provider_files_config(
model="",
provider=LlmProviders(custom_llm_provider),
)
if provider_config is not None:
litellm_params_dict = get_litellm_params(**kwargs)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
logging_obj = kwargs.get("litellm_logging_obj")
if logging_obj is None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
logging_obj = LiteLLMLoggingObj(
model="",
messages=[],
stream=False,
call_type="afile_list" if _is_async else "file_list",
start_time=time.time(),
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
function_id=str(kwargs.get("id", "")),
)
client = kwargs.get("client")
response = base_llm_http_handler.list_files(
purpose=purpose,
provider_config=provider_config,
litellm_params=litellm_params_dict,
headers=extra_headers or {},
logging_obj=logging_obj,
_is_async=_is_async,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
return response
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
@ -752,7 +878,7 @@ def file_list(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'file_list'. Only 'openai' and 'azure' are supported.".format(
message="LiteLLM doesn't support {} for 'file_list'. Only 'openai', 'azure', and 'manus' are supported.".format(
custom_llm_provider
),
model="n/a",
@ -771,7 +897,7 @@ def file_list(
@client
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -816,7 +942,7 @@ def file_content(
file_id: str,
model: Optional[str] = None,
custom_llm_provider: Optional[
Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"], str]
Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"], str]
] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -977,7 +1103,7 @@ def file_content(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai', 'bedrock'.".format(
message="LiteLLM doesn't support {} for 'file_content'. Supported providers are 'openai', 'azure', 'vertex_ai', 'bedrock', 'manus'.".format(
custom_llm_provider
),
model="n/a",

11
litellm/google_genai/adapters/handler.py
View file

@ -37,9 +37,14 @@ class GenerateContentToCompletionHandler:
completion_kwargs: Dict[str, Any] = dict(completion_request)
# feed metadata for custom callback
if extra_kwargs is not None and "metadata" in extra_kwargs:
completion_kwargs["metadata"] = extra_kwargs["metadata"]
# Forward extra_kwargs that should be passed to completion call
if extra_kwargs is not None:
# Forward metadata for custom callback
if "metadata" in extra_kwargs:
completion_kwargs["metadata"] = extra_kwargs["metadata"]
# Forward extra_headers for providers that require custom headers (e.g., github_copilot)
if "extra_headers" in extra_kwargs:
completion_kwargs["extra_headers"] = extra_kwargs["extra_headers"]
if stream:
completion_kwargs["stream"] = stream

12
litellm/google_genai/main.py
View file

@ -130,6 +130,9 @@ class GenerateContentHelper:
api_key=litellm_params.api_key,
)
if litellm_params.custom_llm_provider is None:
litellm_params.custom_llm_provider = custom_llm_provider
# get provider config
generate_content_provider_config: Optional[
BaseGoogleGenAIGenerateContentConfig
@ -327,6 +330,7 @@ def generate_content(
tools=tools,
_is_async=_is_async,
litellm_params=setup_result.litellm_params,
extra_headers=extra_headers,
**kwargs,
)
@ -407,6 +411,9 @@ async def agenerate_content_stream(
# Check if we should use the adapter (when provider config is None)
if setup_result.generate_content_provider_config is None:
if "stream" in kwargs:
kwargs.pop("stream", None)
# Use the adapter to convert to completion format
return (
await GenerateContentToCompletionHandler.async_generate_content_handler(
@ -416,6 +423,7 @@ async def agenerate_content_stream(
litellm_params=setup_result.litellm_params,
tools=tools,
stream=True,
extra_headers=extra_headers,
**kwargs,
)
)
@ -490,6 +498,9 @@ def generate_content_stream(
# Check if we should use the adapter (when provider config is None)
if setup_result.generate_content_provider_config is None:
if "stream" in kwargs:
kwargs.pop("stream", None)
# Use the adapter to convert to completion format
return GenerateContentToCompletionHandler.generate_content_handler(
model=model,
@ -498,6 +509,7 @@ def generate_content_stream(
_is_async=_is_async,
litellm_params=setup_result.litellm_params,
stream=True,
extra_headers=extra_headers,
**kwargs,
)

35
litellm/integrations/braintrust_logging.py
View file

@ -225,10 +225,13 @@ class BraintrustLogger(CustomLogger):
"id": litellm_call_id,
"input": prompt["messages"],
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
@ -351,14 +354,37 @@ class BraintrustLogger(CustomLogger):
# Allow metadata override for span name
span_name = dynamic_metadata.get("span_name", "Chat Completion")
# Span parents is a special case
span_parents = dynamic_metadata.get("span_parents")
# Convert comma-separated string to list if present
if span_parents:
span_parents = [s.strip() for s in span_parents.split(",") if s.strip()]
# Add optional span attributes only if present
span_attributes = {
"span_id": dynamic_metadata.get("span_id"),
"root_span_id": dynamic_metadata.get("root_span_id"),
"span_parents": span_parents,
}
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"output": output,
"metadata": standard_logging_object,
"tags": tags,
"span_attributes": {"name": span_name, "type": "llm"},
}
# Braintrust cannot specify 'tags' for non-root spans
if dynamic_metadata.get("root_span_id") is None:
request_data["tags"] = tags
# Only add those that are not None (or falsy)
for key, value in span_attributes.items():
if value:
request_data[key] = value
if choices is not None:
request_data["output"] = [choice.dict() for choice in choices]
else:
@ -367,9 +393,6 @@ class BraintrustLogger(CustomLogger):
if metrics is not None:
request_data["metrics"] = metrics
if metrics is not None:
request_data["metrics"] = metrics
try:
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",

182
litellm/integrations/cloudzero/database.py
View file

@ -19,7 +19,7 @@
"""Database connection and data extraction for LiteLLM."""
from datetime import datetime
from typing import Any, Dict, Optional
from typing import Any, Optional, List
import polars as pl
@ -46,19 +46,9 @@ class LiteLLMDatabase:
"""Retrieve usage data from LiteLLM daily user spend table."""
client = self._ensure_prisma_client()
# Build WHERE clause for time filtering
where_conditions = []
if start_time_utc:
where_conditions.append(f"dus.updated_at >= '{start_time_utc.isoformat()}'")
if end_time_utc:
where_conditions.append(f"dus.updated_at <= '{end_time_utc.isoformat()}'")
where_clause = ""
if where_conditions:
where_clause = "WHERE " + " AND ".join(where_conditions)
# Query to get user spend data with team information
query = f"""
# Query to get user spend data with team information. Use parameter binding to
# avoid SQL injection from user-supplied timestamps or limits.
query = """
SELECT
dus.id,
dus.date,
@ -85,163 +75,27 @@ class LiteLLMDatabase:
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
{where_clause}
WHERE ($1::timestamptz IS NULL OR dus.updated_at >= $1::timestamptz)
AND ($2::timestamptz IS NULL OR dus.updated_at <= $2::timestamptz)
ORDER BY dus.date DESC, dus.created_at DESC
"""
if limit:
query += f" LIMIT {limit}"
params: List[Any] = [
start_time_utc,
end_time_utc,
]
if limit is not None:
try:
params.append(int(limit))
except (TypeError, ValueError):
raise ValueError("limit must be an integer")
query += " LIMIT $3"
try:
db_response = await client.db.query_raw(query)
db_response = await client.db.query_raw(query, *params)
# Convert the response to polars DataFrame with full schema inference
# This prevents schema mismatch errors when data types vary across rows
return pl.DataFrame(db_response, infer_schema_length=None)
except Exception as e:
raise Exception(f"Error retrieving usage data: {str(e)}")
async def get_table_info(self) -> Dict[str, Any]:
"""Get information about the daily user spend table."""
client = self._ensure_prisma_client()
try:
# Get row count from user spend table
user_count = await self._get_table_row_count("LiteLLM_DailyUserSpend")
# Get column structure from user spend table
query = """
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = 'LiteLLM_DailyUserSpend'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(query)
return {
"columns": columns_response,
"row_count": user_count,
"table_name": "LiteLLM_DailyUserSpend",
}
except Exception as e:
raise Exception(f"Error getting table info: {str(e)}")
async def _get_table_row_count(self, table_name: str) -> int:
"""Get row count from specified table."""
client = self._ensure_prisma_client()
try:
query = f'SELECT COUNT(*) as count FROM "{table_name}"'
response = await client.db.query_raw(query)
if response and len(response) > 0:
return response[0].get("count", 0)
return 0
except Exception:
return 0
async def discover_all_tables(self) -> Dict[str, Any]:
"""Discover all tables in the LiteLLM database and their schemas."""
client = self._ensure_prisma_client()
try:
# Get all LiteLLM tables
litellm_tables_query = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name LIKE 'LiteLLM_%'
ORDER BY table_name;
"""
tables_response = await client.db.query_raw(litellm_tables_query)
table_names = [row["table_name"] for row in tables_response]
# Get detailed schema for each table
tables_info = {}
for table_name in table_names:
# Get column information
columns_query = """
SELECT
column_name,
data_type,
is_nullable,
column_default,
character_maximum_length,
numeric_precision,
numeric_scale,
ordinal_position
FROM information_schema.columns
WHERE table_name = $1
AND table_schema = 'public'
ORDER BY ordinal_position;
"""
columns_response = await client.db.query_raw(columns_query, table_name)
# Get primary key information
pk_query = """
SELECT a.attname
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = $1::regclass AND i.indisprimary;
"""
pk_response = await client.db.query_raw(pk_query, f'"{table_name}"')
primary_keys = (
[row["attname"] for row in pk_response] if pk_response else []
)
# Get foreign key information
fk_query = """
SELECT
tc.constraint_name,
kcu.column_name,
ccu.table_name AS foreign_table_name,
ccu.column_name AS foreign_column_name
FROM information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name
WHERE tc.constraint_type = 'FOREIGN KEY'
AND tc.table_name = $1;
"""
fk_response = await client.db.query_raw(fk_query, table_name)
foreign_keys = fk_response if fk_response else []
# Get indexes
indexes_query = """
SELECT
i.relname AS index_name,
array_agg(a.attname ORDER BY a.attnum) AS column_names,
ix.indisunique AS is_unique
FROM pg_class t
JOIN pg_index ix ON t.oid = ix.indrelid
JOIN pg_class i ON i.oid = ix.indexrelid
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey)
WHERE t.relname = $1
AND t.relkind = 'r'
GROUP BY i.relname, ix.indisunique
ORDER BY i.relname;
"""
indexes_response = await client.db.query_raw(indexes_query, table_name)
indexes = indexes_response if indexes_response else []
# Get row count
try:
row_count = await self._get_table_row_count(table_name)
except Exception:
row_count = 0
tables_info[table_name] = {
"columns": columns_response,
"primary_keys": primary_keys,
"foreign_keys": foreign_keys,
"indexes": indexes,
"row_count": row_count,
}
return {
"tables": tables_info,
"table_count": len(table_names),
"table_names": table_names,
}
except Exception as e:
raise Exception(f"Error discovering tables: {str(e)}")

0
litellm/integrations/focus/__init__.py Normal file
View file

113
litellm/integrations/focus/database.py Normal file
View file

@ -0,0 +1,113 @@
"""Database access helpers for Focus export."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, Optional
import polars as pl
class FocusLiteLLMDatabase:
"""Retrieves LiteLLM usage data for Focus export workflows."""
def _ensure_prisma_client(self):
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise RuntimeError(
"Database not connected. Connect a database to your proxy - "
"https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
)
return prisma_client
async def get_usage_data(
self,
*,
limit: Optional[int] = None,
start_time_utc: Optional[datetime] = None,
end_time_utc: Optional[datetime] = None,
) -> pl.DataFrame:
"""Return usage data for the requested window."""
client = self._ensure_prisma_client()
where_clauses: list[str] = []
query_params: list[Any] = []
placeholder_index = 1
if start_time_utc:
where_clauses.append(f"dus.updated_at >= ${placeholder_index}::timestamptz")
query_params.append(start_time_utc)
placeholder_index += 1
if end_time_utc:
where_clauses.append(f"dus.updated_at <= ${placeholder_index}::timestamptz")
query_params.append(end_time_utc)
placeholder_index += 1
where_clause = ""
if where_clauses:
where_clause = "WHERE " + " AND ".join(where_clauses)
limit_clause = ""
if limit is not None:
try:
limit_value = int(limit)
except (TypeError, ValueError) as exc: # pragma: no cover - defensive guard
raise ValueError("limit must be an integer") from exc
if limit_value < 0:
raise ValueError("limit must be non-negative")
limit_clause = f" LIMIT ${placeholder_index}"
query_params.append(limit_value)
query = f"""
SELECT
dus.id,
dus.date,
dus.user_id,
dus.api_key,
dus.model,
dus.model_group,
dus.custom_llm_provider,
dus.prompt_tokens,
dus.completion_tokens,
dus.spend,
dus.api_requests,
dus.successful_requests,
dus.failed_requests,
dus.cache_creation_input_tokens,
dus.cache_read_input_tokens,
dus.created_at,
dus.updated_at,
vt.team_id,
vt.key_alias as api_key_alias,
tt.team_alias,
ut.user_email as user_email
FROM "LiteLLM_DailyUserSpend" dus
LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
LEFT JOIN "LiteLLM_UserTable" ut ON dus.user_id = ut.user_id
{where_clause}
ORDER BY dus.date DESC, dus.created_at DESC
{limit_clause}
"""
try:
db_response = await client.db.query_raw(query, *query_params)
return pl.DataFrame(db_response, infer_schema_length=None)
except Exception as exc:
raise RuntimeError(f"Error retrieving usage data: {exc}") from exc
async def get_table_info(self) -> Dict[str, Any]:
"""Return metadata about the spend table for diagnostics."""
client = self._ensure_prisma_client()
info_query = """
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_name = 'LiteLLM_DailyUserSpend'
ORDER BY ordinal_position;
"""
try:
columns_response = await client.db.query_raw(info_query)
return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"}
except Exception as exc:
raise RuntimeError(f"Error getting table info: {exc}") from exc

12
litellm/integrations/focus/destinations/__init__.py Normal file
View file

@ -0,0 +1,12 @@
"""Destination implementations for Focus export."""
from .base import FocusDestination, FocusTimeWindow
from .factory import FocusDestinationFactory
from .s3_destination import FocusS3Destination
__all__ = [
"FocusDestination",
"FocusDestinationFactory",
"FocusTimeWindow",
"FocusS3Destination",
]

30
litellm/integrations/focus/destinations/base.py Normal file
View file

@ -0,0 +1,30 @@
"""Abstract destination interfaces for Focus export."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
@dataclass(frozen=True)
class FocusTimeWindow:
"""Represents the span of data exported in a single batch."""
start_time: datetime
end_time: datetime
frequency: str
class FocusDestination(Protocol):
"""Protocol for anything that can receive Focus export files."""
async def deliver(
self,
*,
content: bytes,
time_window: FocusTimeWindow,
filename: str,
) -> None:
"""Persist the serialized export for the provided time window."""
...

59
litellm/integrations/focus/destinations/factory.py Normal file
View file

@ -0,0 +1,59 @@
"""Factory helpers for Focus export destinations."""
from __future__ import annotations
import os
from typing import Any, Dict, Optional
from .base import FocusDestination
from .s3_destination import FocusS3Destination
class FocusDestinationFactory:
"""Builds destination instances based on provider/config settings."""
@staticmethod
def create(
*,
provider: str,
prefix: str,
config: Optional[Dict[str, Any]] = None,
) -> FocusDestination:
"""Return a destination implementation for the requested provider."""
provider_lower = provider.lower()
normalized_config = FocusDestinationFactory._resolve_config(
provider=provider_lower, overrides=config or {}
)
if provider_lower == "s3":
return FocusS3Destination(prefix=prefix, config=normalized_config)
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export"
)
@staticmethod
def _resolve_config(
*,
provider: str,
overrides: Dict[str, Any],
) -> Dict[str, Any]:
if provider == "s3":
resolved = {
"bucket_name": overrides.get("bucket_name")
or os.getenv("FOCUS_S3_BUCKET_NAME"),
"region_name": overrides.get("region_name")
or os.getenv("FOCUS_S3_REGION_NAME"),
"endpoint_url": overrides.get("endpoint_url")
or os.getenv("FOCUS_S3_ENDPOINT_URL"),
"aws_access_key_id": overrides.get("aws_access_key_id")
or os.getenv("FOCUS_S3_ACCESS_KEY"),
"aws_secret_access_key": overrides.get("aws_secret_access_key")
or os.getenv("FOCUS_S3_SECRET_KEY"),
"aws_session_token": overrides.get("aws_session_token")
or os.getenv("FOCUS_S3_SESSION_TOKEN"),
}
if not resolved.get("bucket_name"):
raise ValueError("FOCUS_S3_BUCKET_NAME must be provided for S3 exports")
return {k: v for k, v in resolved.items() if v is not None}
raise NotImplementedError(
f"Provider '{provider}' not supported for Focus export configuration"
)

74
litellm/integrations/focus/destinations/s3_destination.py Normal file
View file

@ -0,0 +1,74 @@
"""S3 destination implementation for Focus export."""
from __future__ import annotations
import asyncio
from datetime import timezone
from typing import Any, Optional
import boto3
from .base import FocusDestination, FocusTimeWindow
class FocusS3Destination(FocusDestination):
"""Handles uploading serialized exports to S3 buckets."""
def __init__(
self,
*,
prefix: str,
config: Optional[dict[str, Any]] = None,
) -> None:
config = config or {}
bucket_name = config.get("bucket_name")
if not bucket_name:
raise ValueError("bucket_name must be provided for S3 destination")
self.bucket_name = bucket_name
self.prefix = prefix.rstrip("/")
self.config = config
async def deliver(
self,
*,
content: bytes,
time_window: FocusTimeWindow,
filename: str,
) -> None:
object_key = self._build_object_key(time_window=time_window, filename=filename)
await asyncio.to_thread(self._upload, content, object_key)
def _build_object_key(self, *, time_window: FocusTimeWindow, filename: str) -> str:
start_utc = time_window.start_time.astimezone(timezone.utc)
date_component = f"date={start_utc.strftime('%Y-%m-%d')}"
parts = [self.prefix, date_component]
if time_window.frequency == "hourly":
parts.append(f"hour={start_utc.strftime('%H')}")
key_prefix = "/".join(filter(None, parts))
return f"{key_prefix}/{filename}" if key_prefix else filename
def _upload(self, content: bytes, object_key: str) -> None:
client_kwargs: dict[str, Any] = {}
region_name = self.config.get("region_name")
if region_name:
client_kwargs["region_name"] = region_name
endpoint_url = self.config.get("endpoint_url")
if endpoint_url:
client_kwargs["endpoint_url"] = endpoint_url
session_kwargs: dict[str, Any] = {}
for key in (
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
):
if self.config.get(key):
session_kwargs[key] = self.config[key]
s3_client = boto3.client("s3", **client_kwargs, **session_kwargs)
s3_client.put_object(
Bucket=self.bucket_name,
Key=object_key,
Body=content,
ContentType="application/octet-stream",
)

124
litellm/integrations/focus/export_engine.py Normal file
View file

@ -0,0 +1,124 @@
"""Core export engine for Focus integrations (heavy dependencies)."""
from __future__ import annotations
from typing import Any, Dict, Optional
import polars as pl
from litellm._logging import verbose_logger
from .database import FocusLiteLLMDatabase
from .destinations import FocusDestinationFactory, FocusTimeWindow
from .serializers import FocusParquetSerializer, FocusSerializer
from .transformer import FocusTransformer
class FocusExportEngine:
"""Engine that fetches, normalizes, and uploads Focus exports."""
def __init__(
self,
*,
provider: str,
export_format: str,
prefix: str,
destination_config: Optional[dict[str, Any]] = None,
) -> None:
self.provider = provider
self.export_format = export_format
self.prefix = prefix
self._destination = FocusDestinationFactory.create(
provider=self.provider,
prefix=self.prefix,
config=destination_config,
)
self._serializer = self._init_serializer()
self._transformer = FocusTransformer()
self._database = FocusLiteLLMDatabase()
def _init_serializer(self) -> FocusSerializer:
if self.export_format != "parquet":
raise NotImplementedError("Only parquet export supported currently")
return FocusParquetSerializer()
async def dry_run_export_usage_data(self, limit: Optional[int]) -> Dict[str, Any]:
data = await self._database.get_usage_data(limit=limit)
normalized = self._transformer.transform(data)
usage_sample = data.head(min(50, len(data))).to_dicts()
normalized_sample = normalized.head(min(50, len(normalized))).to_dicts()
summary = {
"total_records": len(normalized),
"total_spend": self._sum_column(normalized, "spend"),
"total_tokens": self._sum_column(normalized, "total_tokens"),
"unique_teams": self._count_unique(normalized, "team_id"),
"unique_models": self._count_unique(normalized, "model"),
}
return {
"usage_data": usage_sample,
"normalized_data": normalized_sample,
"summary": summary,
}
async def export_window(
self,
*,
window: FocusTimeWindow,
limit: Optional[int],
) -> None:
data = await self._database.get_usage_data(
limit=limit,
start_time_utc=window.start_time,
end_time_utc=window.end_time,
)
if data.is_empty():
verbose_logger.debug("Focus export: no usage data for window %s", window)
return
normalized = self._transformer.transform(data)
if normalized.is_empty():
verbose_logger.debug(
"Focus export: normalized data empty for window %s", window
)
return
await self._serialize_and_upload(normalized, window)
async def _serialize_and_upload(
self, frame: pl.DataFrame, window: FocusTimeWindow
) -> None:
payload = self._serializer.serialize(frame)
if not payload:
verbose_logger.debug("Focus export: serializer returned empty payload")
return
await self._destination.deliver(
content=payload,
time_window=window,
filename=self._build_filename(),
)
def _build_filename(self) -> str:
if not self._serializer.extension:
raise ValueError("Serializer must declare a file extension")
return f"usage.{self._serializer.extension}"
@staticmethod
def _sum_column(frame: pl.DataFrame, column: str) -> float:
if frame.is_empty() or column not in frame.columns:
return 0.0
value = frame.select(pl.col(column).sum().alias("sum")).row(0)[0]
if value is None:
return 0.0
return float(value)
@staticmethod
def _count_unique(frame: pl.DataFrame, column: str) -> int:
if frame.is_empty() or column not in frame.columns:
return 0
value = frame.select(pl.col(column).n_unique().alias("unique")).row(0)[0]
if value is None:
return 0
return int(value)

211
litellm/integrations/focus/focus_logger.py Normal file
View file

@ -0,0 +1,211 @@
"""Focus export logger orchestrating DB pull/transform/upload."""
from __future__ import annotations
import os
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from .destinations import FocusTimeWindow
if TYPE_CHECKING:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from .export_engine import FocusExportEngine
else:
AsyncIOScheduler = Any
FOCUS_USAGE_DATA_JOB_NAME = "focus_export_usage_data"
DEFAULT_DRY_RUN_LIMIT = 500
class FocusLogger(CustomLogger):
"""Coordinates Focus export jobs across transformer/serializer/destination layers."""
def __init__(
self,
*,
provider: Optional[str] = None,
export_format: Optional[str] = None,
frequency: Optional[str] = None,
cron_offset_minute: Optional[int] = None,
interval_seconds: Optional[int] = None,
prefix: Optional[str] = None,
destination_config: Optional[dict[str, Any]] = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self.provider = (provider or os.getenv("FOCUS_PROVIDER") or "s3").lower()
self.export_format = (
export_format or os.getenv("FOCUS_FORMAT") or "parquet"
).lower()
self.frequency = (frequency or os.getenv("FOCUS_FREQUENCY") or "hourly").lower()
self.cron_offset_minute = (
cron_offset_minute
if cron_offset_minute is not None
else int(os.getenv("FOCUS_CRON_OFFSET", "5"))
)
raw_interval = (
interval_seconds
if interval_seconds is not None
else os.getenv("FOCUS_INTERVAL_SECONDS")
)
self.interval_seconds = int(raw_interval) if raw_interval is not None else None
env_prefix = os.getenv("FOCUS_PREFIX")
self.prefix: str = (
prefix if prefix is not None else (env_prefix if env_prefix else "focus_exports")
)
self._destination_config = destination_config
self._engine: Optional["FocusExportEngine"] = None
def _ensure_engine(self) -> "FocusExportEngine":
"""Instantiate the heavy export engine lazily."""
if self._engine is None:
from .export_engine import FocusExportEngine
self._engine = FocusExportEngine(
provider=self.provider,
export_format=self.export_format,
prefix=self.prefix,
destination_config=self._destination_config,
)
return self._engine
async def export_usage_data(
self,
*,
limit: Optional[int] = None,
start_time_utc: Optional[datetime] = None,
end_time_utc: Optional[datetime] = None,
) -> None:
"""Public hook to trigger export immediately."""
if bool(start_time_utc) ^ bool(end_time_utc):
raise ValueError(
"start_time_utc and end_time_utc must be provided together"
)
if start_time_utc and end_time_utc:
window = FocusTimeWindow(
start_time=start_time_utc,
end_time=end_time_utc,
frequency=self.frequency,
)
else:
window = self._compute_time_window(datetime.now(timezone.utc))
await self._export_window(window=window, limit=limit)
async def dry_run_export_usage_data(
self, limit: Optional[int] = DEFAULT_DRY_RUN_LIMIT
) -> dict[str, Any]:
"""Return transformed data without uploading."""
engine = self._ensure_engine()
return await engine.dry_run_export_usage_data(limit=limit)
async def initialize_focus_export_job(self) -> None:
"""Entry point for scheduler jobs to run export cycle with locking."""
from litellm.proxy.proxy_server import proxy_logging_obj
pod_lock_manager = None
if proxy_logging_obj is not None:
writer = getattr(proxy_logging_obj, "db_spend_update_writer", None)
if writer is not None:
pod_lock_manager = getattr(writer, "pod_lock_manager", None)
if pod_lock_manager and pod_lock_manager.redis_cache:
acquired = await pod_lock_manager.acquire_lock(
cronjob_id=FOCUS_USAGE_DATA_JOB_NAME
)
if not acquired:
verbose_logger.debug("Focus export: unable to acquire pod lock")
return
try:
await self._run_scheduled_export()
finally:
await pod_lock_manager.release_lock(
cronjob_id=FOCUS_USAGE_DATA_JOB_NAME
)
else:
await self._run_scheduled_export()
@staticmethod
async def init_focus_export_background_job(
scheduler: AsyncIOScheduler,
) -> None:
"""Register the export cron/interval job with the provided scheduler."""
focus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=FocusLogger
)
if not focus_loggers:
verbose_logger.debug(
"No Focus export logger registered; skipping scheduler"
)
return
focus_logger = cast(FocusLogger, focus_loggers[0])
trigger_kwargs = focus_logger._build_scheduler_trigger()
scheduler.add_job(
focus_logger.initialize_focus_export_job,
**trigger_kwargs,
)
def _build_scheduler_trigger(self) -> Dict[str, Any]:
"""Return scheduler configuration for the selected frequency."""
if self.frequency == "interval":
seconds = self.interval_seconds or 60
return {"trigger": "interval", "seconds": seconds}
if self.frequency == "hourly":
minute = max(0, min(59, self.cron_offset_minute))
return {"trigger": "cron", "minute": minute, "second": 0}
if self.frequency == "daily":
total_minutes = max(0, self.cron_offset_minute)
hour = min(23, total_minutes // 60)
minute = min(59, total_minutes % 60)
return {"trigger": "cron", "hour": hour, "minute": minute, "second": 0}
raise ValueError(f"Unsupported frequency: {self.frequency}")
async def _run_scheduled_export(self) -> None:
"""Execute the scheduled export for the configured window."""
window = self._compute_time_window(datetime.now(timezone.utc))
await self._export_window(window=window, limit=None)
async def _export_window(
self,
*,
window: FocusTimeWindow,
limit: Optional[int],
) -> None:
engine = self._ensure_engine()
await engine.export_window(window=window, limit=limit)
def _compute_time_window(self, now: datetime) -> FocusTimeWindow:
"""Derive the time window to export based on configured frequency."""
now_utc = now.astimezone(timezone.utc)
if self.frequency == "hourly":
end_time = now_utc.replace(minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(hours=1)
elif self.frequency == "daily":
end_time = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
start_time = end_time - timedelta(days=1)
elif self.frequency == "interval":
interval = timedelta(seconds=self.interval_seconds or 60)
end_time = now_utc
start_time = end_time - interval
else:
raise ValueError(f"Unsupported frequency: {self.frequency}")
return FocusTimeWindow(
start_time=start_time,
end_time=end_time,
frequency=self.frequency,
)
__all__ = ["FocusLogger"]

50
litellm/integrations/focus/schema.py Normal file
View file

@ -0,0 +1,50 @@
"""Schema definitions for Focus export data."""
from __future__ import annotations
import polars as pl
# see: https://focus.finops.org/focus-specification/v1-2/
FOCUS_NORMALIZED_SCHEMA = pl.Schema(
[
("BilledCost", pl.Decimal(18, 6)),
("BillingAccountId", pl.String),
("BillingAccountName", pl.String),
("BillingCurrency", pl.String),
("BillingPeriodStart", pl.Datetime(time_unit="us")),
("BillingPeriodEnd", pl.Datetime(time_unit="us")),
("ChargeCategory", pl.String),
("ChargeClass", pl.String),
("ChargeDescription", pl.String),
("ChargeFrequency", pl.String),
("ChargePeriodStart", pl.Datetime(time_unit="us")),
("ChargePeriodEnd", pl.Datetime(time_unit="us")),
("ConsumedQuantity", pl.Decimal(18, 6)),
("ConsumedUnit", pl.String),
("ContractedCost", pl.Decimal(18, 6)),
("ContractedUnitPrice", pl.Decimal(18, 6)),
("EffectiveCost", pl.Decimal(18, 6)),
("InvoiceIssuerName", pl.String),
("ListCost", pl.Decimal(18, 6)),
("ListUnitPrice", pl.Decimal(18, 6)),
("PricingCategory", pl.String),
("PricingQuantity", pl.Decimal(18, 6)),
("PricingUnit", pl.String),
("ProviderName", pl.String),
("PublisherName", pl.String),
("RegionId", pl.String),
("RegionName", pl.String),
("ResourceId", pl.String),
("ResourceName", pl.String),
("ResourceType", pl.String),
("ServiceCategory", pl.String),
("ServiceSubcategory", pl.String),
("ServiceName", pl.String),
("SubAccountId", pl.String),
("SubAccountName", pl.String),
("SubAccountType", pl.String),
("Tags", pl.Object),
]
)
__all__ = ["FOCUS_NORMALIZED_SCHEMA"]

6
litellm/integrations/focus/serializers/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""Serializer package exports for Focus integration."""
from .base import FocusSerializer
from .parquet import FocusParquetSerializer
__all__ = ["FocusSerializer", "FocusParquetSerializer"]

18
litellm/integrations/focus/serializers/base.py Normal file
View file

@ -0,0 +1,18 @@
"""Serializer abstractions for Focus export."""
from __future__ import annotations
from abc import ABC, abstractmethod
import polars as pl
class FocusSerializer(ABC):
"""Base serializer turning Focus frames into bytes."""
extension: str = ""
@abstractmethod
def serialize(self, frame: pl.DataFrame) -> bytes:
"""Convert the normalized Focus frame into the chosen format."""
raise NotImplementedError

22
litellm/integrations/focus/serializers/parquet.py Normal file
View file

@ -0,0 +1,22 @@
"""Parquet serializer for Focus export."""
from __future__ import annotations
import io
import polars as pl
from .base import FocusSerializer
class FocusParquetSerializer(FocusSerializer):
"""Serialize normalized Focus frames to Parquet bytes."""
extension = "parquet"
def serialize(self, frame: pl.DataFrame) -> bytes:
"""Encode the provided frame as a parquet payload."""
target = frame if not frame.is_empty() else pl.DataFrame(schema=frame.schema)
buffer = io.BytesIO()
target.write_parquet(buffer, compression="snappy")
return buffer.getvalue()

90
litellm/integrations/focus/transformer.py Normal file
View file

@ -0,0 +1,90 @@
"""Focus export data transformer."""
from __future__ import annotations
from datetime import timedelta
import polars as pl
from .schema import FOCUS_NORMALIZED_SCHEMA
class FocusTransformer:
"""Transforms LiteLLM DB rows into Focus-compatible schema."""
schema = FOCUS_NORMALIZED_SCHEMA
def transform(self, frame: pl.DataFrame) -> pl.DataFrame:
"""Return a normalized frame expected by downstream serializers."""
if frame.is_empty():
return pl.DataFrame(schema=self.schema)
# derive period start/end from usage date
frame = frame.with_columns(
pl.col("date")
.cast(pl.Utf8)
.str.strptime(pl.Datetime(time_unit="us"), format="%Y-%m-%d", strict=False)
.alias("usage_date"),
)
frame = frame.with_columns(
pl.col("usage_date").alias("ChargePeriodStart"),
(pl.col("usage_date") + timedelta(days=1)).alias("ChargePeriodEnd"),
)
def fmt(col):
return col.dt.strftime("%Y-%m-%dT%H:%M:%SZ")
DEC = pl.Decimal(18, 6)
def dec(col):
return col.cast(DEC)
none_str = pl.lit(None, dtype=pl.Utf8)
none_dec = pl.lit(None, dtype=pl.Decimal(18, 6))
return frame.select(
dec(pl.col("spend").fill_null(0.0)).alias("BilledCost"),
pl.col("api_key").cast(pl.String).alias("BillingAccountId"),
pl.col("api_key_alias").cast(pl.String).alias("BillingAccountName"),
pl.lit("API Key").alias("BillingAccountType"),
pl.lit("USD").alias("BillingCurrency"),
fmt(pl.col("ChargePeriodEnd")).alias("BillingPeriodEnd"),
fmt(pl.col("ChargePeriodStart")).alias("BillingPeriodStart"),
pl.lit("Usage").alias("ChargeCategory"),
none_str.alias("ChargeClass"),
pl.col("model").cast(pl.String).alias("ChargeDescription"),
pl.lit("Usage-Based").alias("ChargeFrequency"),
fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"),
fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"),
dec(pl.lit(1.0)).alias("ConsumedQuantity"),
pl.lit("Requests").alias("ConsumedUnit"),
dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"),
none_str.alias("ContractedUnitPrice"),
dec(pl.col("spend").fill_null(0.0)).alias("EffectiveCost"),
pl.col("custom_llm_provider").cast(pl.String).alias("InvoiceIssuerName"),
none_str.alias("InvoiceId"),
dec(pl.col("spend").fill_null(0.0)).alias("ListCost"),
none_dec.alias("ListUnitPrice"),
none_str.alias("AvailabilityZone"),
pl.lit("USD").alias("PricingCurrency"),
none_str.alias("PricingCategory"),
dec(pl.lit(1.0)).alias("PricingQuantity"),
none_dec.alias("PricingCurrencyContractedUnitPrice"),
dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"),
none_dec.alias("PricingCurrencyListUnitPrice"),
pl.lit("Requests").alias("PricingUnit"),
pl.col("custom_llm_provider").cast(pl.String).alias("ProviderName"),
pl.col("custom_llm_provider").cast(pl.String).alias("PublisherName"),
none_str.alias("RegionId"),
none_str.alias("RegionName"),
pl.col("model").cast(pl.String).alias("ResourceId"),
pl.col("model").cast(pl.String).alias("ResourceName"),
pl.col("model").cast(pl.String).alias("ResourceType"),
pl.lit("AI and Machine Learning").alias("ServiceCategory"),
pl.lit("Generative AI").alias("ServiceSubcategory"),
pl.col("model_group").cast(pl.String).alias("ServiceName"),
pl.col("team_id").cast(pl.String).alias("SubAccountId"),
pl.col("team_alias").cast(pl.String).alias("SubAccountName"),
none_str.alias("SubAccountType"),
none_str.alias("Tags"),
)

59
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -1,28 +1,37 @@
{
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
"sample_callback": {
"event_types": ["llm_api_success", "llm_api_failure"],
"endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
"environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
},
"rubrik": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
}
}
"environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
},
"sumologic": {
"endpoint": "{{environment_variables.SUMOLOGIC_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
},
"qualifire_eval": {
"event_types": ["llm_api_success"],
"endpoint": "{{environment_variables.QUALIFIRE_WEBHOOK_URL}}",
"headers": {
"Content-Type": "application/json",
"X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}"
},
"environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"]
}
}

388
litellm/integrations/prometheus.py
View file

@ -14,6 +14,7 @@ from typing import (
Literal,
Optional,
Tuple,
Union,
cast,
)
@ -44,6 +45,7 @@ def _get_cached_end_user_id_for_cost_tracking():
global _get_end_user_id_for_cost_tracking
if _get_end_user_id_for_cost_tracking is None:
from litellm.utils import get_end_user_id_for_cost_tracking
_get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking
return _get_end_user_id_for_cost_tracking
@ -237,6 +239,36 @@ class PrometheusLogger(CustomLogger):
),
buckets=LATENCY_BUCKETS,
)
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time spent in request queue before processing starts (seconds)",
labelnames=self.get_labels_for_metric(
"litellm_request_queue_time_seconds"
),
buckets=LATENCY_BUCKETS,
)
# Guardrail metrics
self.litellm_guardrail_latency_metric = self._histogram_factory(
"litellm_guardrail_latency_seconds",
"Latency (seconds) for guardrail execution",
labelnames=["guardrail_name", "status", "error_type", "hook_type"],
buckets=LATENCY_BUCKETS,
)
self.litellm_guardrail_errors_total = self._counter_factory(
"litellm_guardrail_errors_total",
"Total number of errors encountered during guardrail execution",
labelnames=["guardrail_name", "error_type", "hook_type"],
)
self.litellm_guardrail_requests_total = self._counter_factory(
"litellm_guardrail_requests_total",
"Total number of guardrail invocations",
labelnames=["guardrail_name", "status", "hook_type"],
)
# llm api provider budget metrics
self.litellm_provider_remaining_budget_metric = self._gauge_factory(
"litellm_provider_remaining_budget_metric",
@ -329,6 +361,25 @@ class PrometheusLogger(CustomLogger):
labelnames=self.get_labels_for_metric("litellm_requests_metric"),
)
# Cache metrics
self.litellm_cache_hits_metric = self._counter_factory(
name="litellm_cache_hits_metric",
documentation="Total number of LiteLLM cache hits",
labelnames=self.get_labels_for_metric("litellm_cache_hits_metric"),
)
self.litellm_cache_misses_metric = self._counter_factory(
name="litellm_cache_misses_metric",
documentation="Total number of LiteLLM cache misses",
labelnames=self.get_labels_for_metric("litellm_cache_misses_metric"),
)
self.litellm_cached_tokens_metric = self._counter_factory(
name="litellm_cached_tokens_metric",
documentation="Total tokens served from LiteLLM cache",
labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"),
)
except Exception as e:
print_verbose(f"Got exception on init prometheus client {str(e)}")
raise e
@ -791,11 +842,16 @@ class PrometheusLogger(CustomLogger):
f"standard_logging_object is required, got={standard_logging_payload}"
)
if self._should_skip_metrics_for_invalid_key(
kwargs=kwargs, standard_logging_payload=standard_logging_payload
):
return
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {})
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
end_user_id = get_end_user_id_for_cost_tracking(
litellm_params, service_type="prometheus"
)
@ -815,20 +871,8 @@ class PrometheusLogger(CustomLogger):
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
# Include top-level metadata fields (excluding nested dictionaries)
# This allows accessing fields like requester_ip_address from top-level metadata
top_level_metadata = standard_logging_payload.get("metadata", {})
top_level_fields: Dict[str, Any] = {}
if isinstance(top_level_metadata, dict):
top_level_fields = {
k: v
for k, v in top_level_metadata.items()
if not isinstance(v, dict) # Exclude nested dicts to avoid conflicts
}
combined_metadata: Dict[str, Any] = {
**top_level_fields, # Include top-level fields first
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
}
@ -945,6 +989,12 @@ class PrometheusLogger(CustomLogger):
kwargs, start_time, end_time, enum_values, output_tokens
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
)
if (
standard_logging_payload["stream"] is True
): # log successful streaming requests from logging event hook.
@ -1014,6 +1064,54 @@ class PrometheusLogger(CustomLogger):
standard_logging_payload["completion_tokens"]
)
def _increment_cache_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
):
"""
Increment cache-related Prometheus metrics based on cache hit/miss status.
Args:
standard_logging_payload: Contains cache_hit field (True/False/None)
enum_values: Label values for Prometheus metrics
"""
cache_hit = standard_logging_payload.get("cache_hit")
# Only track if cache_hit has a definite value (True or False)
if cache_hit is None:
return
if cache_hit is True:
# Increment cache hits counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_hits_metric"
),
enum_values=enum_values,
)
self.litellm_cache_hits_metric.labels(**_labels).inc()
# Increment cached tokens counter
total_tokens = standard_logging_payload.get("total_tokens", 0)
if total_tokens > 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cached_tokens_metric"
),
enum_values=enum_values,
)
self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens)
else:
# cache_hit is False - increment cache misses counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_misses_metric"
),
enum_values=enum_values,
)
self.litellm_cache_misses_metric.labels(**_labels).inc()
async def _increment_remaining_budget_metrics(
self,
user_api_team: Optional[str],
@ -1182,6 +1280,22 @@ class PrometheusLogger(CustomLogger):
total_time_seconds
)
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
queue_time_seconds = _litellm_params.get("metadata", {}).get(
"queue_time_seconds"
)
if queue_time_seconds is not None and queue_time_seconds >= 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_request_queue_time_seconds"
),
enum_values=enum_values,
)
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
from litellm.types.utils import StandardLoggingPayload
@ -1189,14 +1303,20 @@ class PrometheusLogger(CustomLogger):
f"prometheus Logging - Enters failure logging function for kwargs {kwargs}"
)
# unpack kwargs
model = kwargs.get("model", "")
standard_logging_payload: StandardLoggingPayload = kwargs.get(
"standard_logging_object", {}
)
if self._should_skip_metrics_for_invalid_key(
kwargs=kwargs, standard_logging_payload=standard_logging_payload
):
return
model = kwargs.get("model", "")
litellm_params = kwargs.get("litellm_params", {}) or {}
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
end_user_id = get_end_user_id_for_cost_tracking(
litellm_params, service_type="prometheus"
)
@ -1207,7 +1327,6 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias = standard_logging_payload["metadata"][
"user_api_key_team_alias"
]
kwargs.get("exception", None)
try:
self.litellm_llm_api_failed_requests_metric.labels(
@ -1227,6 +1346,139 @@ class PrometheusLogger(CustomLogger):
pass
pass
def _extract_status_code(
self,
kwargs: Optional[dict] = None,
enum_values: Optional[Any] = None,
exception: Optional[Exception] = None,
) -> Optional[int]:
"""
Extract HTTP status code from various input formats for validation.
This is a centralized helper to extract status code from different
callback function signatures. Handles both ProxyException (uses 'code')
and standard exceptions (uses 'status_code').
Args:
kwargs: Dictionary potentially containing 'exception' key
enum_values: Object with 'status_code' attribute
exception: Exception object to extract status code from directly
Returns:
Status code as integer if found, None otherwise
"""
status_code = None
# Try from enum_values first (most common in our callbacks)
if enum_values and hasattr(enum_values, "status_code") and enum_values.status_code:
try:
status_code = int(enum_values.status_code)
except (ValueError, TypeError):
pass
if not status_code and exception:
# ProxyException uses 'code' attribute, other exceptions may use 'status_code'
status_code = getattr(exception, "status_code", None) or getattr(exception, "code", None)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
if not status_code and kwargs:
exception_in_kwargs = kwargs.get("exception")
if exception_in_kwargs:
status_code = getattr(exception_in_kwargs, "status_code", None) or getattr(exception_in_kwargs, "code", None)
if status_code is not None:
try:
status_code = int(status_code)
except (ValueError, TypeError):
status_code = None
return status_code
def _is_invalid_api_key_request(
self,
status_code: Optional[int],
exception: Optional[Exception] = None,
) -> bool:
"""
Determine if a request has an invalid API key based on status code and exception.
This method prevents invalid authentication attempts from being recorded in
Prometheus metrics. A 401 status code is the definitive indicator of authentication
failure. Additionally, we check exception messages for authentication error patterns
to catch cases where the exception hasn't been converted to a ProxyException yet.
Args:
status_code: HTTP status code (401 indicates authentication error)
exception: Exception object to check for auth-related error messages
Returns:
True if the request has an invalid API key and metrics should be skipped,
False otherwise
"""
if status_code == 401:
return True
# Handle cases where AssertionError is raised before conversion to ProxyException
if exception is not None:
exception_str = str(exception).lower()
auth_error_patterns = [
"virtual key expected",
"expected to start with 'sk-'",
"authentication error",
"invalid api key",
"api key not valid",
]
if any(pattern in exception_str for pattern in auth_error_patterns):
return True
return False
def _should_skip_metrics_for_invalid_key(
self,
kwargs: Optional[dict] = None,
user_api_key_dict: Optional[Any] = None,
enum_values: Optional[Any] = None,
standard_logging_payload: Optional[Union[dict, StandardLoggingPayload]] = None,
exception: Optional[Exception] = None,
) -> bool:
"""
Determine if Prometheus metrics should be skipped for invalid API key requests.
This is a centralized validation method that extracts status code and exception
information from various callback function signatures and determines if the request
represents an invalid API key attempt that should be filtered from metrics.
Args:
kwargs: Dictionary potentially containing exception and other data
user_api_key_dict: User API key authentication object (currently unused)
enum_values: Object with status_code attribute
standard_logging_payload: Standard logging payload dictionary
exception: Exception object to check directly
Returns:
True if metrics should be skipped (invalid key detected), False otherwise
"""
status_code = self._extract_status_code(
kwargs=kwargs,
enum_values=enum_values,
exception=exception,
)
if exception is None and kwargs:
exception = kwargs.get("exception")
if self._is_invalid_api_key_request(status_code, exception=exception):
verbose_logger.debug(
"Skipping Prometheus metrics for invalid API key request: "
f"status_code={status_code}, exception={type(exception).__name__ if exception else None}"
)
return True
return False
async def async_post_call_failure_hook(
self,
request_data: dict,
@ -1252,6 +1504,14 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict,
exception=original_exception,
):
return
status_code = self._extract_status_code(exception=original_exception)
try:
_tags = StandardLoggingPayloadSetup._get_request_tags(
litellm_params=request_data,
@ -1266,8 +1526,8 @@ class PrometheusLogger(CustomLogger):
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,
requested_model=request_data.get("model", ""),
status_code=str(getattr(original_exception, "status_code", None)),
exception_status=str(getattr(original_exception, "status_code", None)),
status_code=str(status_code),
exception_status=str(status_code),
exception_class=self._get_exception_class_name(original_exception),
tags=_tags,
route=user_api_key_dict.request_route,
@ -1305,6 +1565,11 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict
):
return
enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id,
hashed_api_key=user_api_key_dict.api_key,
@ -1360,6 +1625,15 @@ class PrometheusLogger(CustomLogger):
exception = request_kwargs.get("exception", None)
llm_provider = _litellm_params.get("custom_llm_provider", None)
if self._should_skip_metrics_for_invalid_key(
kwargs=request_kwargs,
standard_logging_payload=standard_logging_payload,
):
return
hashed_api_key = standard_logging_payload.get("metadata", {}).get(
"user_api_key_hash"
)
# Create enum_values for the label factory (always create for use in different metrics)
enum_values = UserAPIKeyLabelValues(
@ -1374,9 +1648,7 @@ class PrometheusLogger(CustomLogger):
self._get_exception_class_name(exception) if exception else None
),
requested_model=model_group,
hashed_api_key=standard_logging_payload["metadata"][
"user_api_key_hash"
],
hashed_api_key=hashed_api_key,
api_key_alias=standard_logging_payload["metadata"][
"user_api_key_alias"
],
@ -1398,7 +1670,6 @@ class PrometheusLogger(CustomLogger):
api_provider=llm_provider or "",
)
if exception is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_failure_responses"
@ -1431,16 +1702,23 @@ class PrometheusLogger(CustomLogger):
enum_values: UserAPIKeyLabelValues,
output_tokens: float = 1.0,
):
try:
verbose_logger.debug("setting remaining tokens requests metric")
standard_logging_payload: Optional[StandardLoggingPayload] = (
request_kwargs.get("standard_logging_object")
)
standard_logging_payload: Optional[
StandardLoggingPayload
] = request_kwargs.get("standard_logging_object")
if standard_logging_payload is None:
return
# Skip recording metrics for invalid API key requests
if self._should_skip_metrics_for_invalid_key(
kwargs=request_kwargs,
enum_values=enum_values,
standard_logging_payload=standard_logging_payload,
):
return
api_base = standard_logging_payload["api_base"]
_litellm_params = request_kwargs.get("litellm_params", {}) or {}
_metadata = _litellm_params.get("metadata", {})
@ -1571,6 +1849,50 @@ class PrometheusLogger(CustomLogger):
)
return
def _record_guardrail_metrics(
self,
guardrail_name: str,
latency_seconds: float,
status: str,
error_type: Optional[str],
hook_type: str,
):
"""
Record guardrail metrics for prometheus.
Args:
guardrail_name: Name of the guardrail
latency_seconds: Execution latency in seconds
status: "success" or "error"
error_type: Type of error if any, None otherwise
hook_type: "pre_call", "during_call", or "post_call"
"""
try:
# Record latency
self.litellm_guardrail_latency_metric.labels(
guardrail_name=guardrail_name,
status=status,
error_type=error_type or "none",
hook_type=hook_type,
).observe(latency_seconds)
# Record request count
self.litellm_guardrail_requests_total.labels(
guardrail_name=guardrail_name,
status=status,
hook_type=hook_type,
).inc()
# Record error count if there was an error
if status == "error" and error_type:
self.litellm_guardrail_errors_total.labels(
guardrail_name=guardrail_name,
error_type=error_type,
hook_type=hook_type,
).inc()
except Exception as e:
verbose_logger.debug(f"Error recording guardrail metrics: {str(e)}")
@staticmethod
def _get_exception_class_name(exception: Exception) -> str:
exception_class_name = ""
@ -2208,10 +2530,10 @@ class PrometheusLogger(CustomLogger):
from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
prometheus_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
)
prometheus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
)
# we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers))
@ -2283,7 +2605,7 @@ def prometheus_label_factory(
if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
get_end_user_id_for_cost_tracking = _get_cached_end_user_id_for_cost_tracking()
filtered_labels["end_user"] = get_end_user_id_for_cost_tracking(
litellm_params={"user_api_key_end_user_id": enum_values.end_user},
service_type="prometheus",

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -18,6 +18,7 @@ from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLog
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.focus.focus_logger import FocusLogger
from litellm.integrations.datadog.datadog import DataDogLogger
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from litellm.integrations.deepeval import DeepEvalLogger
@ -93,6 +94,7 @@ class CustomLoggerRegistry:
"bitbucket": BitBucketPromptManager,
"gitlab": GitLabPromptManager,
"cloudzero": CloudZeroLogger,
"focus": FocusLogger,
"posthog": PostHogLogger,
}

8
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -913,6 +913,14 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "http://localhost:2024"
)
dynamic_api_key = api_key or get_secret_str("LANGGRAPH_API_KEY")
elif custom_llm_provider == "manus":
# Manus is OpenAI compatible for responses API
api_base = (
api_base
or get_secret_str("MANUS_API_BASE")
or "https://api.manus.im"
)
dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))

21
litellm/litellm_core_utils/litellm_logging.py
View file

@ -3756,6 +3756,15 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
cloudzero_logger = CloudZeroLogger()
_in_memory_loggers.append(cloudzero_logger)
return cloudzero_logger # type: ignore
elif logging_integration == "focus":
from litellm.integrations.focus.focus_logger import FocusLogger
for callback in _in_memory_loggers:
if isinstance(callback, FocusLogger):
return callback # type: ignore
focus_logger = FocusLogger()
_in_memory_loggers.append(focus_logger)
return focus_logger # type: ignore
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
@ -4076,6 +4085,12 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, CloudZeroLogger):
return callback
elif logging_integration == "focus":
from litellm.integrations.focus.focus_logger import FocusLogger
for callback in _in_memory_loggers:
if isinstance(callback, FocusLogger):
return callback
elif logging_integration == "deepeval":
for callback in _in_memory_loggers:
if isinstance(callback, DeepEvalLogger):
@ -4800,7 +4815,7 @@ class StandardLoggingPayloadSetup:
"""
Extract additional header tags for spend tracking based on config.
"""
extra_headers: List[str] = litellm.extra_spend_tag_headers or []
extra_headers: List[str] = getattr(litellm, "extra_spend_tag_headers", None) or []
if not extra_headers:
return None
@ -4824,9 +4839,9 @@ class StandardLoggingPayloadSetup:
metadata = litellm_params.get("metadata") or {}
litellm_metadata = litellm_params.get("litellm_metadata") or {}
if metadata.get("tags", []):
request_tags = metadata.get("tags", [])
request_tags = metadata.get("tags", []).copy()
elif litellm_metadata.get("tags", []):
request_tags = litellm_metadata.get("tags", [])
request_tags = litellm_metadata.get("tags", []).copy()
else:
request_tags = []
user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags(

11
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -485,9 +485,14 @@ def _calculate_input_cost(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### IMAGE TOKEN COST (for gpt-image-1 and similar models)
### IMAGE TOKEN COST
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
if model_info.get(image_token_cost_key) is None:
image_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"]
model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
@ -521,7 +526,7 @@ def _calculate_input_cost(
return prompt_cost
def generic_cost_per_token(
def generic_cost_per_token( # noqa: PLR0915
model: str,
usage: Usage,
custom_llm_provider: str,

119
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -6,6 +6,7 @@ import io
import mimetypes
import re
from os import PathLike
from pathlib import Path
from typing import (
TYPE_CHECKING,
Any,
@ -94,7 +95,9 @@ def handle_messages_with_content_list_to_str_conversion(
return messages
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[str] = ["user"]) -> AllMessageValues:
def strip_name_from_message(
message: AllMessageValues, allowed_name_roles: List[str] = ["user"]
) -> AllMessageValues:
"""
Removes 'name' from message
"""
@ -103,6 +106,7 @@ def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[
msg_copy.pop("name", None) # type: ignore
return msg_copy
def strip_name_from_messages(
messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"]
) -> List[AllMessageValues]:
@ -443,7 +447,7 @@ def update_responses_input_with_model_file_ids(
"""
Updates responses API input with provider-specific file IDs.
File IDs are always inside the content array, not as direct input_file items.
For managed files (unified file IDs), decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly.
"""
@ -451,25 +455,28 @@ def update_responses_input_with_model_file_ids(
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
if isinstance(input, str):
return input
if not isinstance(input, list):
return input
updated_input = []
for item in input:
if not isinstance(item, dict):
updated_input.append(item)
continue
updated_item = item.copy()
content = item.get("content")
if isinstance(content, list):
updated_content = []
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
if (
isinstance(content_item, dict)
and content_item.get("type") == "input_file"
):
file_id = content_item.get("file_id")
if file_id:
# Check if this is a managed file ID (base64-encoded unified file ID)
@ -477,7 +484,9 @@ def update_responses_input_with_model_file_ids(
if is_unified_file_id:
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
else:
# Fallback: keep original if we can't extract
provider_file_id = file_id
@ -491,9 +500,9 @@ def update_responses_input_with_model_file_ids(
else:
updated_content.append(content_item)
updated_item["content"] = updated_content
updated_input.append(updated_item)
return updated_input
@ -533,6 +542,12 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
# Convert content to bytes
if isinstance(file_content, (str, PathLike)):
# If it's a path, open and read the file
# Extract filename from path if not already set
if filename is None:
if isinstance(file_content, PathLike):
filename = Path(file_content).name
else:
filename = Path(str(file_content)).name
with open(file_content, "rb") as f:
content = f.read()
elif isinstance(file_content, io.IOBase):
@ -550,11 +565,11 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
# Use provided content type or guess based on filename
if not content_type:
content_type = (
mimetypes.guess_type(filename)[0]
if filename
else "application/octet-stream"
)
if filename:
guessed_type = mimetypes.guess_type(filename)[0]
content_type = guessed_type if guessed_type else "application/octet-stream"
else:
content_type = "application/octet-stream"
return ExtractedFileData(
filename=filename,
@ -690,9 +705,9 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]:
video/flv
"""
from urllib.parse import urlparse
url = url.lower()
# Parse URL to extract path without query parameters
# This handles URLs like: https://example.com/image.jpg?signature=...
parsed = urlparse(url)
@ -737,28 +752,28 @@ def infer_content_type_from_url_and_content(
) -> str:
"""
Infer content type from URL extension and binary content when content-type header is missing or generic.
This helper implements a fallback strategy for determining MIME types when HTTP headers
are missing or provide generic values (like binary/octet-stream). It's commonly used
when processing images and documents from various sources (S3, URLs, etc.).
Fallback Strategy:
1. If current_content_type is valid (not None and not generic octet-stream), return it
2. Try to infer from URL extension (handles query parameters)
3. Try to detect from binary content signature (magic bytes)
4. Raise ValueError if all methods fail
Args:
url: The URL of the content (used to extract file extension)
content: The binary content (first ~100 bytes are sufficient for detection)
current_content_type: The current content-type from headers (may be None or generic)
Returns:
str: The inferred MIME type (e.g., "image/png", "application/pdf")
Raises:
ValueError: If content type cannot be determined by any method
Example:
>>> content_type = infer_content_type_from_url_and_content(
... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123",
@ -769,14 +784,14 @@ def infer_content_type_from_url_and_content(
"image/png"
"""
from litellm.litellm_core_utils.token_counter import get_image_type
# If we have a valid content type that's not generic, use it
if current_content_type and current_content_type not in [
"binary/octet-stream",
"application/octet-stream",
]:
return current_content_type
# Extension to MIME type mapping
# Supports images, documents, and other common file types
extension_to_mime = {
@ -797,14 +812,14 @@ def infer_content_type_from_url_and_content(
"txt": "text/plain",
"md": "text/markdown",
}
# Try to infer from URL extension
if url:
extension = url.split(".")[-1].lower().split("?")[0] # Remove query params
inferred_type = extension_to_mime.get(extension)
if inferred_type:
return inferred_type
# Try to detect from binary content signature (magic bytes)
if content:
detected_type = get_image_type(content[:100])
@ -818,7 +833,7 @@ def infer_content_type_from_url_and_content(
}
if detected_type in type_to_mime:
return type_to_mime[detected_type]
# If all fallbacks failed, raise error
raise ValueError(
f"Unable to determine content type from URL: {url}. "
@ -1078,7 +1093,9 @@ def _parse_content_for_reasoning(
return None, message_text
reasoning_match = re.match(
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)", message_text, re.DOTALL
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)",
message_text,
re.DOTALL,
)
if reasoning_match:
@ -1128,3 +1145,47 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
elif isinstance(image_url, dict) and "url" in image_url:
images.append(_extract_base64_data(image_url["url"]))
return images
def parse_tool_call_arguments(
arguments: Optional[str],
tool_name: Optional[str] = None,
context: Optional[str] = None,
) -> Dict[str, Any]:
"""
Parse tool call arguments from a JSON string.
This function handles malformed JSON gracefully by raising a ValueError
with context about what failed and what the problematic input was.
Args:
arguments: The JSON string containing tool arguments, or None.
tool_name: Optional name of the tool (for error messages).
context: Optional context string (e.g., "Anthropic Messages API").
Returns:
Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty.
Raises:
ValueError: If the arguments string is not valid JSON.
"""
import json
if not arguments:
return {}
try:
return json.loads(arguments)
except json.JSONDecodeError as e:
error_parts = ["Failed to parse tool call arguments"]
if tool_name:
error_parts.append(f"for tool '{tool_name}'")
if context:
error_parts.append(f"({context})")
error_message = (
" ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}"
)
raise ValueError(error_message) from e

171
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -44,6 +44,7 @@ from .common_utils import (
convert_content_list_to_str,
infer_content_type_from_url_and_content,
is_non_content_values_set,
parse_tool_call_arguments,
)
from .image_handling import convert_url_to_base64
@ -911,13 +912,13 @@ def convert_to_anthropic_image_obj(
def create_anthropic_image_param(
image_url_input: Union[str, dict],
image_url_input: Union[str, dict],
format: Optional[str] = None,
is_bedrock_invoke: bool = False
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
# Extract URL and format from input
@ -927,7 +928,7 @@ def create_anthropic_image_param(
image_url = image_url_input.get("url", "")
if format is None:
format = image_url_input.get("format")
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
@ -1031,9 +1032,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
tool_function = get_attribute_or_key(tool, "function")
tool_name = get_attribute_or_key(tool_function, "name")
tool_arguments = get_attribute_or_key(tool_function, "arguments")
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
)
parameters = "".join(
f"<{param}>{val}</{param}>\n"
for param, val in json.loads(tool_arguments).items()
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
)
invokes += (
"<invoke>\n"
@ -1071,8 +1074,14 @@ def anthropic_messages_pt_xml(messages: list):
if isinstance(messages[msg_i]["content"], list):
for m in messages[msg_i]["content"]:
if m.get("type", "") == "image_url":
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
image_param = create_anthropic_image_param(m["image_url"], format=format)
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
image_param = create_anthropic_image_param(
m["image_url"], format=format
)
# Convert to dict format for XML version
source = image_param["source"]
if isinstance(source, dict) and source.get("type") == "url":
@ -1381,10 +1390,10 @@ def convert_to_gemini_tool_call_invoke(
if tool_calls is not None:
for idx, tool in enumerate(tool_calls):
if "function" in tool:
gemini_function_call: Optional[
VertexFunctionCall
] = _gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
gemini_function_call: Optional[VertexFunctionCall] = (
_gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
)
)
if gemini_function_call is not None:
part_dict: VertexPartType = {
@ -1484,10 +1493,10 @@ def convert_to_gemini_tool_call_result(
}
"""
from litellm.types.llms.vertex_ai import BlobType
content_str: str = ""
inline_data: Optional[BlobType] = None
if "content" in message:
if isinstance(message["content"], str):
content_str = message["content"]
@ -1500,15 +1509,21 @@ def convert_to_gemini_tool_call_result(
elif content_type in ("input_image", "image_url"):
# Extract image for inline_data (for Computer Use screenshots and tool results)
image_url_data = content.get("image_url", "")
image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data
image_url = (
image_url_data.get("url", "")
if isinstance(image_url_data, dict)
else image_url_data
)
if image_url:
# Convert image to base64 blob format for Gemini
try:
image_obj = convert_to_anthropic_image_obj(image_url, format=None)
image_obj = convert_to_anthropic_image_obj(
image_url, format=None
)
inline_data = BlobType(
data=image_obj["data"],
mime_type=image_obj["media_type"]
mime_type=image_obj["media_type"],
)
except Exception as e:
verbose_logger.warning(
@ -1541,6 +1556,7 @@ def convert_to_gemini_tool_call_result(
response_data: dict
try:
import json
if content_str.strip().startswith("{") or content_str.strip().startswith("["):
# Try to parse as JSON (for Computer Use structured responses)
parsed = json.loads(content_str)
@ -1553,7 +1569,7 @@ def convert_to_gemini_tool_call_result(
except (json.JSONDecodeError, ValueError):
# Not valid JSON, wrap in content field
response_data = {"content": content_str}
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
_function_response = VertexFunctionResponse(
@ -1562,7 +1578,7 @@ def convert_to_gemini_tool_call_result(
# Create part with function_response, and optionally inline_data for images (Computer Use)
_part: VertexPartType = {"function_response": _function_response}
# For Computer Use, if we have an image, we need separate parts:
# - One part with function_response
# - One part with inline_data
@ -1570,19 +1586,19 @@ def convert_to_gemini_tool_call_result(
if inline_data:
image_part: VertexPartType = {"inline_data": inline_data}
return [_part, image_part]
return _part
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
This function replaces any invalid characters with underscores.
"""
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
sanitized = re.sub(r'[^a-zA-Z0-9_-]', '_', tool_use_id)
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
# Ensure it's not empty (fallback to a default if needed)
if not sanitized:
sanitized = "tool_use_id"
@ -1644,10 +1660,19 @@ def convert_to_anthropic_tool_result(
)
)
elif content["type"] == "image_url":
format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None
anthropic_content_list.append(
create_anthropic_image_param(content["image_url"], format=format)
format = (
content["image_url"].get("format")
if isinstance(content["image_url"], dict)
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(_anthropic_image_param)
anthropic_content = anthropic_content_list
anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@ -1662,7 +1687,9 @@ def convert_to_anthropic_tool_result(
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if message["role"] == "function":
@ -1671,7 +1698,9 @@ def convert_to_anthropic_tool_result(
# Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if anthropic_tool_result is None:
@ -1687,12 +1716,17 @@ def convert_function_to_anthropic_tool_invoke(
try:
_name = get_attribute_or_key(function_call, "name") or ""
_arguments = get_attribute_or_key(function_call, "arguments")
tool_input = parse_tool_call_arguments(
_arguments, tool_name=_name, context="Anthropic function to tool invoke"
)
anthropic_tool_invoke = [
AnthropicMessagesToolUseParam(
type="tool_use",
id=str(uuid.uuid4()),
name=_name,
input=json.loads(_arguments) if _arguments else {},
input=tool_input,
)
]
return anthropic_tool_invoke
@ -1746,7 +1780,9 @@ def convert_to_anthropic_tool_invoke(
Fixes: https://github.com/BerriAI/litellm/issues/17737
"""
anthropic_tool_invoke: List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]] = []
anthropic_tool_invoke: List[
Union[AnthropicMessagesToolUseParam, Dict[str, Any]]
] = []
for tool in tool_calls:
if not get_attribute_or_key(tool, "type") == "function":
@ -1757,10 +1793,10 @@ def convert_to_anthropic_tool_invoke(
str,
get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
)
tool_input = json.loads(
get_attribute_or_key(
get_attribute_or_key(tool, "function"), "arguments"
)
tool_input = parse_tool_call_arguments(
get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"),
tool_name=tool_name,
context="Anthropic tool invoke",
)
# Check if this is a server-side tool (web_search, tool_search, etc.)
@ -2012,11 +2048,17 @@ def anthropic_messages_pt( # noqa: PLR0915
for m in user_message_types_block["content"]:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
# Convert ChatCompletionImageUrlObject to dict if needed
image_url_value = m["image_url"]
if isinstance(image_url_value, str):
image_url_input: Union[str, dict[str, Any]] = image_url_value
image_url_input: Union[str, dict[str, Any]] = (
image_url_value
)
else:
# ChatCompletionImageUrlObject or dict case - convert to dict
image_url_input = {
@ -2026,20 +2068,26 @@ def anthropic_messages_pt( # noqa: PLR0915
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
is_vertex_ai = (
llm_provider.startswith("vertex_ai")
if llm_provider
else False
)
force_base64 = is_bedrock_invoke or is_vertex_ai
_anthropic_content_element = create_anthropic_image_param(
image_url_input, format=format, is_bedrock_invoke=force_base64
)
image_url_input,
format=format,
is_bedrock_invoke=force_base64,
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_element,
original_content_element=dict(m),
)
if "cache_control" in _content_element:
_anthropic_content_element[
"cache_control"
] = _content_element["cache_control"]
_anthropic_content_element["cache_control"] = (
_content_element["cache_control"]
)
user_content.append(_anthropic_content_element)
elif m.get("type", "") == "text":
m = cast(ChatCompletionTextObject, m)
@ -2077,9 +2125,9 @@ def anthropic_messages_pt( # noqa: PLR0915
)
if "cache_control" in _content_element:
_anthropic_content_text_element[
"cache_control"
] = _content_element["cache_control"]
_anthropic_content_text_element["cache_control"] = (
_content_element["cache_control"]
)
user_content.append(_anthropic_content_text_element)
@ -2175,18 +2223,27 @@ def anthropic_messages_pt( # noqa: PLR0915
): # support assistant tool invoke conversion
# Get web_search_results from provider_specific_fields for server_tool_use reconstruction
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields_raw = assistant_content_block.get(
"provider_specific_fields"
)
_provider_specific_fields: Dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(Dict[str, Any], _provider_specific_fields_raw)
_web_search_results = _provider_specific_fields.get("web_search_results")
_provider_specific_fields = cast(
Dict[str, Any], _provider_specific_fields_raw
)
_web_search_results = _provider_specific_fields.get(
"web_search_results"
)
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
)
# AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
assistant_content.extend(
cast(List[AnthropicMessagesAssistantMessageValues], tool_invoke_results)
cast(
List[AnthropicMessagesAssistantMessageValues],
tool_invoke_results,
)
)
assistant_function_call = assistant_content_block.get("function_call")
@ -3249,14 +3306,18 @@ def _convert_to_bedrock_tool_call_result(
"""
-
"""
tool_result_content_blocks:List[BedrockToolResultContentBlock] = []
tool_result_content_blocks: List[BedrockToolResultContentBlock] = []
if isinstance(message["content"], str):
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=message["content"])
)
elif isinstance(message["content"], List):
content_list = message["content"]
for content in content_list:
if content["type"] == "text":
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=content["text"])
)
elif content["type"] == "image_url":
format: Optional[str] = None
if isinstance(content["image_url"], dict):
@ -3264,12 +3325,14 @@ def _convert_to_bedrock_tool_call_result(
format = content["image_url"].get("format")
else:
image_url = content["image_url"]
_block:BedrockContentBlock = BedrockImageProcessor.process_image_sync(
_block: BedrockContentBlock = BedrockImageProcessor.process_image_sync(
image_url=image_url,
format=format,
)
if "image" in _block:
tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"]))
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_block["image"])
)
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))

67
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -1,4 +1,5 @@
from typing import Any, Dict, Optional, Set
from collections.abc import Mapping
from typing import Any, Dict, List, Optional, Set
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
@ -17,6 +18,7 @@ class SensitiveDataMasker:
"key",
"token",
"auth",
"authorization",
"credential",
"access",
"private",
@ -42,22 +44,52 @@ class SensitiveDataMasker:
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str, excluded_keys: Optional[Set[str]] = None) -> bool:
def is_sensitive_key(
self, key: str, excluded_keys: Optional[Set[str]] = None
) -> bool:
# Check if key is in excluded_keys first (exact match)
if excluded_keys and key in excluded_keys:
return False
key_lower = str(key).lower()
# Split on underscores and check if any segment matches the pattern
# Split on underscores/hyphens and check if any segment matches the pattern
# This avoids false positives like "max_tokens" matching "token"
# but still catches "api_key", "access_token", etc.
key_segments = key_lower.replace('-', '_').split('_')
result = any(
pattern in key_segments
for pattern in self.sensitive_patterns
)
key_segments = key_lower.replace("-", "_").split("_")
result = any(pattern in key_segments for pattern in self.sensitive_patterns)
return result
def _mask_sequence(
self,
values: List[Any],
depth: int,
max_depth: int,
excluded_keys: Optional[Set[str]],
key_is_sensitive: bool,
) -> List[Any]:
masked_items: List[Any] = []
if depth >= max_depth:
return values
for item in values:
if isinstance(item, Mapping):
masked_items.append(
self.mask_dict(dict(item), depth + 1, max_depth, excluded_keys)
)
elif isinstance(item, list):
masked_items.append(
self._mask_sequence(
item, depth + 1, max_depth, excluded_keys, key_is_sensitive
)
)
elif key_is_sensitive and isinstance(item, str):
masked_items.append(self._mask_value(item))
else:
masked_items.append(
item if isinstance(item, (int, float, bool, str, list)) else str(item)
)
return masked_items
def mask_dict(
self,
data: Dict[str, Any],
@ -71,11 +103,20 @@ class SensitiveDataMasker:
masked_data: Dict[str, Any] = {}
for k, v in data.items():
try:
if isinstance(v, dict):
masked_data[k] = self.mask_dict(v, depth + 1, max_depth, excluded_keys)
key_is_sensitive = self.is_sensitive_key(k, excluded_keys)
if isinstance(v, Mapping):
masked_data[k] = self.mask_dict(
dict(v), depth + 1, max_depth, excluded_keys
)
elif isinstance(v, list):
masked_data[k] = self._mask_sequence(
v, depth + 1, max_depth, excluded_keys, key_is_sensitive
)
elif hasattr(v, "__dict__") and not isinstance(v, type):
masked_data[k] = self.mask_dict(vars(v), depth + 1, max_depth, excluded_keys)
elif self.is_sensitive_key(k, excluded_keys):
masked_data[k] = self.mask_dict(
vars(v), depth + 1, max_depth, excluded_keys
)
elif key_is_sensitive:
str_value = str(v) if v is not None else ""
masked_data[k] = self._mask_value(str_value)
else:

13
litellm/llms/anthropic/chat/handler.py
View file

@ -732,6 +732,19 @@ class ModelResponseIterator:
provider_specific_fields["web_search_results"] = (
self.web_search_results
)
elif (
content_block_start["content_block"]["type"]
== "web_fetch_tool_result"
):
# Capture web_fetch_tool_result for multi-turn reconstruction
# The full content comes in content_block_start, not in deltas
# Fixes: https://github.com/BerriAI/litellm/issues/18137
self.web_search_results.append(
content_block_start["content_block"]
)
provider_specific_fields["web_search_results"] = (
self.web_search_results
)
elif type_chunk == "content_block_stop":
ContentBlockStop(**chunk) # type: ignore
# check if tool call content block - only for tool_use and server_tool_use blocks

28
litellm/llms/anthropic/chat/transformation.py
View file

@ -62,6 +62,7 @@ from litellm.utils import (
ModelResponse,
Usage,
add_dummy_tool,
any_assistant_message_has_thinking_blocks,
get_max_tokens,
has_tool_call_blocks,
last_assistant_with_tool_calls_has_no_thinking_blocks,
@ -1013,10 +1014,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Drop thinking param if thinking is enabled but thinking_blocks are missing
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
#
# IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
# If any message has thinking_blocks, we must keep thinking enabled, otherwise
# Anthropic errors with: "When thinking is disabled, an assistant message cannot contain thinking"
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -1162,6 +1169,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is None:
web_search_results = []
web_search_results.append(content)
## WEB FETCH TOOL RESULT - preserve web fetch results for multi-turn conversations
## Fixes: https://github.com/BerriAI/litellm/issues/18137
elif content["type"] == "web_fetch_tool_result":
if web_search_results is None:
web_search_results = []
web_search_results.append(content)
elif content.get("thinking", None) is not None:
if thinking_blocks is None:
thinking_blocks = []
@ -1265,14 +1278,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_creation_tokens=cache_creation_input_tokens,
cache_creation_token_details=cache_creation_token_details,
)
completion_token_details = (
CompletionTokensDetailsWrapper(
reasoning_tokens=token_counter(
text=reasoning_content, count_response_tokens=True
)
)
# Always populate completion_token_details, not just when there's reasoning_content
reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else None
else 0
)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else None,
text_tokens=completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens,
)
total_tokens = prompt_tokens + completion_tokens

20
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -14,6 +14,10 @@ from typing import (
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
)
from litellm.types.llms.anthropic import (
AllAnthropicToolsValues,
AnthopicMessagesAssistantMessageParam,
@ -425,15 +429,15 @@ class LiteLLMAnthropicMessagesAdapter:
) -> Optional[str]:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int}
OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default'
"""
if not isinstance(thinking, dict):
return None
thinking_type = thinking.get("type", "disabled")
if thinking_type == "disabled":
return None
elif thinking_type == "enabled":
@ -446,7 +450,7 @@ class LiteLLMAnthropicMessagesAdapter:
return "low"
else:
return "minimal"
return None
def translate_anthropic_tool_choice_to_openai(
@ -676,10 +680,10 @@ class LiteLLMAnthropicMessagesAdapter:
type="tool_use",
id=tool_call.id,
name=tool_call.function.name or "",
input=(
json.loads(tool_call.function.arguments)
if tool_call.function.arguments
else {}
input=parse_tool_call_arguments(
tool_call.function.arguments,
tool_name=tool_call.function.name,
context="Anthropic pass-through adapter",
),
)
# Add provider_specific_fields if signature is present

19
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -25,7 +25,24 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
return "gpt-5" in model or "gpt5_series" in model
def get_supported_openai_params(self, model: str) -> List[str]:
return OpenAIGPT5Config.get_supported_openai_params(self, model=model)
"""Get supported parameters for Azure OpenAI GPT-5 models.
Azure OpenAI GPT-5.2 models support logprobs, unlike OpenAI's GPT-5.
This overrides the parent class to add logprobs support back for gpt-5.2.
Reference:
- Tested with Azure OpenAI GPT-5.2 (api-version: 2025-01-01-preview)
- Azure returns logprobs successfully despite Microsoft's general
documentation stating reasoning models don't support it.
"""
params = OpenAIGPT5Config.get_supported_openai_params(self, model=model)
# Only gpt-5.2 has been verified to support logprobs on Azure
if self.is_model_gpt_5_2_model(model):
azure_supported_params = ["logprobs", "top_logprobs"]
params.extend(azure_supported_params)
return params
def map_openai_params(
self,

4
litellm/llms/base_llm/chat/transformation.py
View file

@ -132,10 +132,10 @@ class BaseConfig(ABC):
Checks 'non_default_params' for 'thinking' and 'max_tokens'
if 'thinking' is enabled and 'max_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
if 'thinking' is enabled and 'max_tokens' or 'max_completion_tokens' is not specified, set 'max_tokens' to the thinking token budget + DEFAULT_MAX_TOKENS
"""
is_thinking_enabled = self.is_thinking_enabled(optional_params)
if is_thinking_enabled and "max_tokens" not in non_default_params:
if is_thinking_enabled and ("max_tokens" not in non_default_params and "max_completion_tokens" not in non_default_params):
thinking_token_budget = cast(dict, optional_params["thinking"]).get(
"budget_tokens", None
)

93
litellm/llms/base_llm/files/transformation.py
View file

@ -2,11 +2,14 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from openai.types.file_deleted import FileDeleted
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.files import TwoStepFileUploadConfig
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileContentRequest,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
@ -75,7 +78,15 @@ class BaseFilesConfig(BaseConfig):
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
) -> Union[dict, str, bytes]:
) -> Union[dict, str, bytes, "TwoStepFileUploadConfig"]:
"""
Transform OpenAI-style file creation request into provider-specific format.
Returns:
- dict: For pre-signed single-step uploads (e.g., Bedrock S3)
- str/bytes: For traditional file uploads
- TwoStepFileUploadConfig: For two-step upload process (e.g., Manus, GCS)
"""
pass
@abstractmethod
@ -88,6 +99,86 @@ class BaseFilesConfig(BaseConfig):
) -> OpenAIFileObject:
pass
@abstractmethod
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file retrieve request into provider-specific format."""
pass
@abstractmethod
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""Transform file retrieve response into OpenAI format."""
pass
@abstractmethod
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file delete request into provider-specific format."""
pass
@abstractmethod
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> "FileDeleted":
"""Transform file delete response into OpenAI format."""
pass
@abstractmethod
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file list request into provider-specific format."""
pass
@abstractmethod
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
"""Transform file list response into OpenAI format."""
pass
@abstractmethod
def transform_file_content_request(
self,
file_content_request: "FileContentRequest",
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Transform file content request into provider-specific format."""
pass
@abstractmethod
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> "HttpxBinaryResponseContent":
"""Transform file content response into OpenAI format."""
pass
def transform_request(
self,
model: str,

72
litellm/llms/bedrock/base_aws_llm.py
View file

@ -74,6 +74,41 @@ class BaseAWSLLM:
"aws_external_id",
]
def _get_ssl_verify(self):
"""
Get SSL verification setting for boto3 clients.
This ensures that custom CA certificates are properly used for all AWS API calls,
including STS and Bedrock services.
Returns:
Union[bool, str]: SSL verification setting - False to disable, True to enable,
or a string path to a CA bundle file
"""
import litellm
from litellm.secret_managers.main import str_to_bool
# Check environment variable first (highest priority)
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
# Convert string "False"/"True" to boolean
if isinstance(ssl_verify, str):
# Check if it's a file path
if os.path.exists(ssl_verify):
return ssl_verify
# Otherwise try to convert to boolean
ssl_verify_bool = str_to_bool(ssl_verify)
if ssl_verify_bool is not None:
ssl_verify = ssl_verify_bool
# Check SSL_CERT_FILE environment variable for custom CA bundle
if ssl_verify is True or ssl_verify == "True":
ssl_cert_file = os.getenv("SSL_CERT_FILE")
if ssl_cert_file and os.path.exists(ssl_cert_file):
return ssl_cert_file
return ssl_verify
def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
"""
Generate a unique cache key based on the credential arguments.
@ -314,6 +349,12 @@ class BaseAWSLLM:
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model = model.split(".")[0]
if _split_model in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
@ -323,13 +364,9 @@ class BaseAWSLLM:
if provider is not None:
return provider
# check if provider == "nova"
if "nova" in model:
return "nova"
else:
for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider
for provider in get_args(BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider
return None
@staticmethod
@ -364,11 +401,15 @@ class BaseAWSLLM:
elif provider == "qwen3" and "qwen3/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="qwen3"
)
)
elif provider == "stability" and "stability/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="stability"
)
elif provider == "moonshot" and "moonshot/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="moonshot"
)
return model_id
@staticmethod
@ -412,7 +453,7 @@ class BaseAWSLLM:
if "nova" in model.lower():
if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova")
# Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
if "." in model:
parts = model.split(".")
@ -563,6 +604,7 @@ class BaseAWSLLM:
"sts",
region_name=aws_region_name,
endpoint_url=sts_endpoint,
verify=self._get_ssl_verify(),
)
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
@ -619,7 +661,7 @@ class BaseAWSLLM:
# Create an STS client without credentials
with tracer.trace("boto3.client(sts) for manual IRSA"):
sts_client = boto3.client("sts", region_name=region)
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
# Manually assume the IRSA role with the session name
verbose_logger.debug(
@ -642,6 +684,7 @@ class BaseAWSLLM:
aws_access_key_id=irsa_creds["AccessKeyId"],
aws_secret_access_key=irsa_creds["SecretAccessKey"],
aws_session_token=irsa_creds["SessionToken"],
verify=self._get_ssl_verify(),
)
# Get current caller identity for debugging
@ -680,7 +723,7 @@ class BaseAWSLLM:
verbose_logger.debug("Same account role assumption, using automatic IRSA")
with tracer.trace("boto3.client(sts) with automatic IRSA"):
sts_client = boto3.client("sts", region_name=region)
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
# Get current caller identity for debugging
try:
@ -803,7 +846,7 @@ class BaseAWSLLM:
# This allows the web identity token to work automatically
if aws_access_key_id is None and aws_secret_access_key is None:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client("sts")
sts_client = boto3.client("sts", verify=self._get_ssl_verify())
else:
with tracer.trace("boto3.client(sts)"):
sts_client = boto3.client(
@ -811,6 +854,7 @@ class BaseAWSLLM:
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
verify=self._get_ssl_verify(),
)
assume_role_params = {
@ -958,7 +1002,9 @@ class BaseAWSLLM:
return endpoint_url, proxy_endpoint_url
def _select_default_endpoint_url(
self, endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], aws_region_name: str
self,
endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]],
aws_region_name: str,
) -> str:
"""
Select the default endpoint url based on the endpoint type

104
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -339,6 +339,55 @@ class AmazonConverseConfig(BaseConfig):
}
}
def _handle_reasoning_effort_parameter(
self, model: str, reasoning_effort: str, optional_params: dict
) -> None:
"""
Handle the reasoning_effort parameter based on the model type.
Different model families handle reasoning effort differently:
- GPT-OSS models: Keep reasoning_effort as-is (passed to additionalModelRequestFields)
- Nova Lite 2 models: Transform to reasoningConfig structure
- Other models (Anthropic, etc.): Convert to thinking parameter
Args:
model: The model identifier
reasoning_effort: The reasoning effort value
optional_params: Dictionary of optional parameters to update in-place
Examples:
>>> config = AmazonConverseConfig()
>>> params = {}
>>> config._handle_reasoning_effort_parameter("gpt-oss-model", "high", params)
>>> params
{'reasoning_effort': 'high'}
>>> params = {}
>>> config._handle_reasoning_effort_parameter("amazon.nova-2-lite-v1:0", "high", params)
>>> params
{'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
>>> params = {}
>>> config._handle_reasoning_effort_parameter("anthropic.claude-3", "high", params)
>>> params
{'thinking': {'type': 'enabled', 'budget_tokens': 10000}}
"""
if "gpt-oss" in model:
# GPT-OSS models: keep reasoning_effort as-is
# It will be passed through to additionalModelRequestFields
optional_params["reasoning_effort"] = reasoning_effort
elif self._is_nova_lite_2_model(model):
# Nova Lite 2 models: transform to reasoningConfig
reasoning_config = self._transform_reasoning_effort_to_reasoning_config(
reasoning_effort
)
optional_params.update(reasoning_config)
else:
# Anthropic and other models: convert to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
reasoning_effort
)
def get_supported_openai_params(self, model: str) -> List[str]:
from litellm.utils import supports_function_calling
@ -353,6 +402,7 @@ class AmazonConverseConfig(BaseConfig):
"extra_headers",
"response_format",
"requestMetadata",
"service_tier",
]
if (
@ -657,25 +707,22 @@ class AmazonConverseConfig(BaseConfig):
if param == "thinking":
optional_params["thinking"] = value
elif param == "reasoning_effort" and isinstance(value, str):
if "gpt-oss" in model:
# GPT-OSS models: keep reasoning_effort as-is
# It will be passed through to additionalModelRequestFields
optional_params["reasoning_effort"] = value
elif self._is_nova_lite_2_model(model):
# Nova Lite 2 models: transform to reasoningConfig
reasoning_config = (
self._transform_reasoning_effort_to_reasoning_config(value)
)
optional_params.update(reasoning_config)
else:
# Anthropic and other models: convert to thinking parameter
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
value
)
self._handle_reasoning_effort_parameter(
model=model, reasoning_effort=value, optional_params=optional_params
)
if param == "requestMetadata":
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value) # type: ignore
optional_params["requestMetadata"] = value
if param == "service_tier" and isinstance(value, str):
# Map OpenAI service_tier (string) to Bedrock serviceTier (object)
# OpenAI values: "auto", "default", "flex", "priority"
# Bedrock values: "default", "flex", "priority" (no "auto")
bedrock_tier = value
if value == "auto":
bedrock_tier = "default" # Bedrock doesn't support "auto"
if bedrock_tier in ("default", "flex", "priority"):
optional_params["serviceTier"] = {"type": bedrock_tier}
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
# Nova Lite 2 handles token budgeting differently through reasoningConfig
@ -685,10 +732,7 @@ class AmazonConverseConfig(BaseConfig):
)
final_is_thinking_enabled = self.is_thinking_enabled(optional_params)
if (
final_is_thinking_enabled
and "tool_choice" in optional_params
):
if final_is_thinking_enabled and "tool_choice" in optional_params:
tool_choice_block = optional_params["tool_choice"]
if isinstance(tool_choice_block, dict):
if "any" in tool_choice_block or "tool" in tool_choice_block:
@ -912,20 +956,22 @@ class AmazonConverseConfig(BaseConfig):
inference_params = {
k: v for k, v in inference_params.items() if k in total_supported_params
}
# Only set the topK value in for models that support it
additional_request_params.update(
self._handle_top_k_value(model, inference_params)
)
# Filter out internal/MCP-related parameters that shouldn't be sent to the API
# These are LiteLLM internal parameters, not API parameters
additional_request_params = filter_internal_params(additional_request_params)
# Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
# from additional_request_params to prevent JSON serialization errors
# This filters: Exception objects, callable objects (functions), Logging objects, etc.
additional_request_params = filter_exceptions_from_params(additional_request_params)
additional_request_params = filter_exceptions_from_params(
additional_request_params
)
return inference_params, additional_request_params, request_metadata
@ -950,7 +996,10 @@ class AmazonConverseConfig(BaseConfig):
if original_tools:
for tool in original_tools:
tool_type = tool.get("type", "")
if tool_type in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
if tool_type in (
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
):
# Tool search not supported in Converse API - skip it
continue
filtered_tools.append(tool)
@ -1535,6 +1584,13 @@ class AmazonConverseConfig(BaseConfig):
if "trace" in completion_response:
setattr(model_response, "trace", completion_response["trace"])
# Add service_tier if present in Bedrock response
# Map Bedrock serviceTier (object) to OpenAI service_tier (string)
if "serviceTier" in completion_response:
service_tier_block = completion_response["serviceTier"]
if isinstance(service_tier_block, dict) and "type" in service_tier_block:
setattr(model_response, "service_tier", service_tier_block["type"])
return model_response
def get_error_class(

256
litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py Normal file
View file

@ -0,0 +1,256 @@
"""
Transformation for Bedrock Moonshot AI (Kimi K2) models.
Supports the Kimi K2 Thinking model available on Amazon Bedrock.
Model format: bedrock/moonshot.kimi-k2-thinking-v1:0
Reference: https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
"""
from typing import TYPE_CHECKING, Any, List, Optional, Union
import re
import httpx
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.types.utils import ModelResponse
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
"""
Configuration for Bedrock Moonshot AI (Kimi K2) models.
Reference:
https://aws.amazon.com/about-aws/whats-new/2025/12/amazon-bedrock-fully-managed-open-weight-models/
https://platform.moonshot.ai/docs/api/chat
Supported Params for the Amazon / Moonshot models:
- `max_tokens` (integer) max tokens
- `temperature` (float) temperature for model (0-1 for Moonshot)
- `top_p` (float) top p for model
- `stream` (bool) whether to stream responses
- `tools` (list) tool definitions (supported on kimi-k2-thinking)
- `tool_choice` (str|dict) tool choice specification (supported on kimi-k2-thinking)
NOT Supported on Bedrock:
- `stop` sequences (Bedrock doesn't support stopSequences field for this model)
Note: The kimi-k2-thinking model DOES support tool calls, unlike kimi-thinking-preview.
"""
def __init__(self, **kwargs):
AmazonInvokeConfig.__init__(self, **kwargs)
MoonshotChatConfig.__init__(self, **kwargs)
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
def _get_model_id(self, model: str) -> str:
"""
Extract the actual model ID from the LiteLLM model name.
Removes routing prefixes like:
- bedrock/invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
- invoke/moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
- moonshot.kimi-k2-thinking -> moonshot.kimi-k2-thinking
"""
# Remove bedrock/ prefix if present
if model.startswith("bedrock/"):
model = model[8:]
# Remove invoke/ prefix if present
if model.startswith("invoke/"):
model = model[7:]
# Remove any provider prefix (e.g., moonshot/)
if "/" in model and not model.startswith("arn:"):
parts = model.split("/", 1)
if len(parts) == 2:
model = parts[1]
return model
def get_supported_openai_params(self, model: str) -> List[str]:
"""
Get the supported OpenAI params for Moonshot AI models on Bedrock.
Bedrock-specific limitations:
- stopSequences field is not supported on Bedrock (unlike native Moonshot API)
- functions parameter is not supported (use tools instead)
- tool_choice doesn't support "required" value
Note: kimi-k2-thinking DOES support tool calls (unlike kimi-thinking-preview)
The parent MoonshotChatConfig class handles the kimi-thinking-preview exclusion.
"""
excluded_params: List[str] = ["functions", "stop"] # Bedrock doesn't support stopSequences
base_openai_params = super(MoonshotChatConfig, self).get_supported_openai_params(model=model)
final_params: List[str] = []
for param in base_openai_params:
if param not in excluded_params:
final_params.append(param)
return final_params
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to Moonshot AI parameters for Bedrock.
Handles Moonshot AI specific limitations:
- tool_choice doesn't support "required" value
- Temperature <0.3 limitation for n>1
- Temperature range is [0, 1] (not [0, 2] like OpenAI)
"""
return MoonshotChatConfig.map_openai_params(
self,
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the request for Bedrock Moonshot AI models.
Uses the Moonshot transformation logic which handles:
- Converting content lists to strings (Moonshot doesn't support list format)
- Adding tool_choice="required" message if needed
- Temperature and parameter validation
"""
# Filter out AWS credentials using the existing method from BaseAWSLLM
self._get_boto_credentials_from_optional_params(optional_params, model)
# Strip routing prefixes to get the actual model ID
clean_model_id = self._get_model_id(model)
# Use Moonshot's transform_request which handles message transformation
# and tool_choice="required" workaround
return MoonshotChatConfig.transform_request(
self,
model=clean_model_id,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
def _extract_reasoning_from_content(self, content: str) -> tuple[Optional[str], str]:
"""
Extract reasoning content from <reasoning> tags in the response.
Moonshot AI's Kimi K2 Thinking model returns reasoning in <reasoning> tags.
This method extracts that content and returns it separately.
Args:
content: The full content string from the API response
Returns:
tuple: (reasoning_content, main_content)
"""
if not content:
return None, content
# Match <reasoning>...</reasoning> tags
reasoning_match = re.match(
r"<reasoning>(.*?)</reasoning>\s*(.*)",
content,
re.DOTALL
)
if reasoning_match:
reasoning_content = reasoning_match.group(1).strip()
main_content = reasoning_match.group(2).strip()
return reasoning_content, main_content
return None, content
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: "ModelResponse",
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> "ModelResponse":
"""
Transform the response from Bedrock Moonshot AI models.
Moonshot AI uses OpenAI-compatible response format, but returns reasoning
content in <reasoning> tags. This method:
1. Calls parent class transformation
2. Extracts reasoning content from <reasoning> tags
3. Sets reasoning_content on the message object
"""
# First, get the standard transformation
model_response = MoonshotChatConfig.transform_response(
self,
model=model,
raw_response=raw_response,
model_response=model_response,
logging_obj=logging_obj,
request_data=request_data,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
api_key=api_key,
json_mode=json_mode,
)
# Extract reasoning content from <reasoning> tags
if model_response.choices and len(model_response.choices) > 0:
for choice in model_response.choices:
# Only process Choices (not StreamingChoices) which have message attribute
if isinstance(choice, Choices) and choice.message and choice.message.content:
reasoning_content, main_content = self._extract_reasoning_from_content(
choice.message.content
)
if reasoning_content:
# Set the reasoning_content field
choice.message.reasoning_content = reasoning_content
# Update the main content without reasoning tags
choice.message.content = main_content
return model_response
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BedrockError:
"""Return the appropriate error class for Bedrock."""
return BedrockError(status_code=status_code, message=error_message)

10
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -524,6 +524,12 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
if model.startswith("invoke/"):
model = model.replace("invoke/", "", 1)
# Special case: Check for "nova" in model name first (before "amazon")
# This handles amazon.nova-* models which would otherwise match "amazon" (Titan)
if "nova" in model.lower():
if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova")
_split_model = model.split(".")[0]
if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model)
@ -533,10 +539,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
if provider is not None:
return provider
# check if provider == "nova"
if "nova" in model:
return "nova"
for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
if provider in model:
return provider

177
litellm/llms/bedrock/common_utils.py
View file

@ -15,7 +15,7 @@ import litellm
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret
@ -132,6 +132,38 @@ def add_custom_header(headers):
return callback
def _get_bedrock_client_ssl_verify() -> Union[bool, str]:
"""
Get SSL verification setting for Bedrock client.
Returns the SSL verification setting which can be:
- True: Use default SSL verification
- False: Disable SSL verification
- str: Path to a custom CA bundle file
"""
from litellm.secret_managers.main import str_to_bool
ssl_verify: Union[bool, str, None] = os.getenv("SSL_VERIFY", litellm.ssl_verify)
# Convert string "False"/"True" to boolean
if isinstance(ssl_verify, str):
# Check if it's a file path
if os.path.exists(ssl_verify):
return ssl_verify # Keep the file path
# Otherwise try to convert to boolean
ssl_verify_bool = str_to_bool(ssl_verify)
if ssl_verify_bool is not None:
ssl_verify = ssl_verify_bool
# Check SSL_CERT_FILE environment variable for custom CA bundle
if ssl_verify is True or ssl_verify == "True":
ssl_cert_file = os.getenv("SSL_CERT_FILE")
if ssl_cert_file and os.path.exists(ssl_cert_file):
return ssl_cert_file
return ssl_verify if ssl_verify is not None else True
def init_bedrock_client(
region_name=None,
aws_access_key_id: Optional[str] = None,
@ -177,8 +209,7 @@ def init_bedrock_client(
aws_web_identity_token,
) = params_to_check
# SSL certificates (a.k.a CA bundle) used to verify the identity of requested hosts.
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
ssl_verify = _get_bedrock_client_ssl_verify()
### SET REGION NAME
if region_name:
@ -229,7 +260,7 @@ def init_bedrock_client(
status_code=401,
)
sts_client = boto3.client("sts")
sts_client = boto3.client("sts", verify=ssl_verify)
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
@ -359,6 +390,70 @@ def get_bedrock_tool_name(response_tool_name: str) -> str:
return response_tool_name
# Cache the global regions list at module level
_BEDROCK_GLOBAL_REGIONS: Optional[List[str]] = None
def _get_all_bedrock_regions() -> List[str]:
"""Get all Bedrock regions, cached at module level."""
global _BEDROCK_GLOBAL_REGIONS
if _BEDROCK_GLOBAL_REGIONS is None:
_BEDROCK_GLOBAL_REGIONS = AmazonBedrockGlobalConfig().get_all_regions()
return _BEDROCK_GLOBAL_REGIONS
def get_bedrock_cross_region_inference_regions() -> List[str]:
"""Abbreviations of regions AWS Bedrock supports for cross region inference."""
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
def extract_model_name_from_bedrock_arn(model: str) -> str:
"""
Extract the model name from an AWS Bedrock ARN.
Returns the string after the last '/' if 'arn' is in the input string.
"""
if "arn" in model.lower():
return model.split("/")[-1]
return model
def strip_bedrock_routing_prefix(model: str) -> str:
"""Strip LiteLLM routing prefixes from model name."""
for prefix in ["bedrock/", "converse/", "invoke/", "openai/"]:
if model.startswith(prefix):
model = model.split("/", 1)[1]
return model
def get_bedrock_base_model(model: str) -> str:
"""
Get the base model from the given model name.
Handle model names like:
- "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
- "bedrock/converse/model" -> "model"
"""
model = strip_bedrock_routing_prefix(model)
model = extract_model_name_from_bedrock_arn(model)
potential_region = model.split(".", 1)[0]
alt_potential_region = model.split("/", 1)[0]
if potential_region in get_bedrock_cross_region_inference_regions():
return model.split(".", 1)[1]
elif (
alt_potential_region in _get_all_bedrock_regions()
and len(model.split("/", 1)) > 1
):
return model.split("/", 1)[1]
return model
# Import after standalone functions to avoid circular imports
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
class BedrockModelInfo(BaseLLMModelInfo):
global_config = AmazonBedrockGlobalConfig()
all_global_regions = global_config.get_all_regions()
@ -394,76 +489,34 @@ class BedrockModelInfo(BaseLLMModelInfo):
) -> List[str]:
return []
@staticmethod
def extract_model_name_from_arn(model: str) -> str:
def get_token_counter(self) -> Optional[BaseTokenCounter]:
"""
Extract the model name from an AWS Bedrock ARN.
Returns the string after the last '/' if 'arn' is in the input string.
Args:
arn (str): The ARN string to parse
Factory method to create a Bedrock token counter.
Returns:
str: The extracted model name if 'arn' is in the string,
otherwise returns the original string
BedrockTokenCounter instance for this provider.
"""
if "arn" in model.lower():
return model.split("/")[-1]
return model
return BedrockTokenCounter()
@staticmethod
def extract_model_name_from_arn(model: str) -> str:
"""Wrapper for standalone function. See extract_model_name_from_bedrock_arn()."""
return extract_model_name_from_bedrock_arn(model)
@staticmethod
def get_non_litellm_routing_model_name(model: str) -> str:
if model.startswith("bedrock/"):
model = model.split("/", 1)[1]
if model.startswith("converse/"):
model = model.split("/", 1)[1]
if model.startswith("invoke/"):
model = model.split("/", 1)[1]
if model.startswith("openai/"):
model = model.split("/", 1)[1]
return model
"""Wrapper for standalone function. See strip_bedrock_routing_prefix()."""
return strip_bedrock_routing_prefix(model)
@staticmethod
def get_base_model(model: str) -> str:
"""
Get the base model from the given model name.
Handle model names like - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1"
"""
model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
model = BedrockModelInfo.extract_model_name_from_arn(model)
potential_region = model.split(".", 1)[0]
alt_potential_region = model.split("/", 1)[
0
] # in model cost map we store regional information like `/us-west-2/bedrock-model`
if (
potential_region
in BedrockModelInfo._supported_cross_region_inference_region()
):
return model.split(".", 1)[1]
elif (
alt_potential_region in BedrockModelInfo.all_global_regions
and len(model.split("/", 1)) > 1
):
return model.split("/", 1)[1]
return model
"""Wrapper for standalone function. See get_bedrock_base_model()."""
return get_bedrock_base_model(model)
@staticmethod
def _supported_cross_region_inference_region() -> List[str]:
"""
Abbreviations of regions AWS Bedrock supports for cross region inference
"""
return ["global", "us", "eu", "apac", "jp", "au", "us-gov"]
"""Wrapper for standalone function. See get_bedrock_cross_region_inference_regions()."""
return get_bedrock_cross_region_inference_regions()
@staticmethod
def get_bedrock_route(
@ -629,6 +682,8 @@ def get_bedrock_chat_config(model: str):
return litellm.AmazonCohereConfig()
elif bedrock_invoke_provider == "mistral":
return litellm.AmazonMistralConfig()
elif bedrock_invoke_provider == "moonshot":
return litellm.AmazonMoonshotConfig()
elif bedrock_invoke_provider == "deepseek_r1":
return litellm.AmazonDeepSeekR1Config()
elif bedrock_invoke_provider == "nova":

87
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py Normal file
View file

@ -0,0 +1,87 @@
"""
Bedrock Token Counter implementation using the CountTokens API.
"""
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.types.utils import LlmProviders, TokenCountResponse
class BedrockTokenCounter(BaseTokenCounter):
"""Token counter implementation for AWS Bedrock provider using the CountTokens API."""
def should_use_token_counting_api(
self,
custom_llm_provider: Optional[str] = None,
) -> bool:
"""
Returns True if we should use the Bedrock CountTokens API for token counting.
"""
return custom_llm_provider == LlmProviders.BEDROCK.value
async def count_tokens(
self,
model_to_use: str,
messages: Optional[List[Dict[str, Any]]],
contents: Optional[List[Dict[str, Any]]],
deployment: Optional[Dict[str, Any]] = None,
request_model: str = "",
) -> Optional[TokenCountResponse]:
"""
Count tokens using AWS Bedrock's CountTokens API.
This method calls the existing BedrockCountTokensHandler to make an API call
to Bedrock's token counting endpoint, bypassing the local tiktoken-based counting.
Args:
model_to_use: The model identifier
messages: The messages to count tokens for
contents: Alternative content format (not used for Bedrock)
deployment: Deployment configuration containing litellm_params
request_model: The original request model name
Returns:
TokenCountResponse with token count, or None if counting fails
"""
if not messages:
return None
deployment = deployment or {}
litellm_params = deployment.get("litellm_params", {})
# Build request data in the format expected by BedrockCountTokensHandler
request_data = {
"model": model_to_use,
"messages": messages,
}
# Get the resolved model (strip prefixes like bedrock/, converse/, etc.)
resolved_model = get_bedrock_base_model(model_to_use)
try:
handler = BedrockCountTokensHandler()
result = await handler.handle_count_tokens_request(
request_data=request_data,
litellm_params=litellm_params,
resolved_model=resolved_model,
)
# Transform response to TokenCountResponse
if result is not None:
return TokenCountResponse(
total_tokens=result.get("input_tokens", 0),
request_model=request_model,
model_used=model_to_use,
tokenizer_type="bedrock_api",
original_response=result,
)
except Exception as e:
verbose_logger.warning(
f"Error calling Bedrock CountTokens API: {e}, falling back to default tokenizer"
)
return None

20
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -6,10 +6,9 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
from typing import Any, Dict
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@ -70,6 +69,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
verbose_logger.debug(f"Making request to: {endpoint_url}")
# Use existing _sign_request method from BaseAWSLLM
# Extract api_key for bearer token auth if provided
api_key = litellm_params.get("api_key", None)
headers = {"Content-Type": "application/json"}
signed_headers, signed_body = self._sign_request(
service_name="bedrock",
@ -78,6 +79,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
request_data=bedrock_request,
api_base=endpoint_url,
model=resolved_model,
api_key=api_key,
)
async_client = get_async_httpx_client(llm_provider=litellm.LlmProviders.BEDROCK)
@ -94,9 +96,9 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
if response.status_code != 200:
error_text = response.text
verbose_logger.error(f"AWS Bedrock error: {error_text}")
raise HTTPException(
status_code=400,
detail={"error": f"AWS Bedrock error: {error_text}"},
raise BedrockError(
status_code=response.status_code,
message=f"AWS Bedrock error: {error_text}",
)
bedrock_response = response.json()
@ -112,12 +114,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
return final_response
except HTTPException:
# Re-raise HTTP exceptions as-is
except BedrockError:
# Re-raise Bedrock exceptions as-is
raise
except Exception as e:
verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
raise HTTPException(
raise BedrockError(
status_code=500,
detail={"error": f"CountTokens processing error: {str(e)}"},
message=f"CountTokens processing error: {str(e)}",
)

4
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -8,7 +8,7 @@ to AWS Bedrock's CountTokens API format and vice versa.
from typing import Any, Dict, List
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
class BedrockCountTokensConfig(BaseAWSLLM):
@ -141,7 +141,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
Complete endpoint URL for CountTokens API
"""
# Use existing LiteLLM function to get the base model ID (removes region prefix)
model_id = BedrockModelInfo.get_base_model(model)
model_id = get_bedrock_base_model(model)
# Remove bedrock/ prefix if present
if model_id.startswith("bedrock/"):

1
litellm/llms/bedrock/files/handler.py
View file

@ -142,6 +142,7 @@ class BedrockFilesHandler(BaseAWSLLM):
aws_secret_access_key=credentials.secret_key,
aws_session_token=credentials.token,
region_name=aws_region_name,
verify=self._get_ssl_verify(),
)
# Download file from S3

69
litellm/llms/bedrock/files/transformation.py
View file

@ -1,12 +1,14 @@
import json
import os
import time
from litellm._uuid import uuid
from typing import Any, Dict, List, Optional, Tuple, Union
import httpx
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -18,6 +20,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileTypes,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
@ -539,6 +542,70 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
status_code=status_code, message=error_message, headers=headers
)
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("BedrockFilesConfig does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("BedrockFilesConfig does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file listing")
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("BedrockFilesConfig does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("BedrockFilesConfig does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("BedrockFilesConfig does not support file content retrieval")
class BedrockJsonlFilesTransformation:
"""

44
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -24,6 +24,37 @@ class BedrockPassthroughConfig(
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in endpoint
def _encode_model_id_for_endpoint(self, model_id: str) -> str:
"""
Encode model_id (especially ARNs) for use in Bedrock endpoints.
ARNs contain special characters like colons and slashes that need to be
properly URL-encoded when used in HTTP request paths. For example:
arn:aws:bedrock:us-east-1:123:application-inference-profile/abc123
becomes:
arn:aws:bedrock:us-east-1:123:application-inference-profile%2Fabc123
Args:
model_id: The model ID or ARN to encode
Returns:
The encoded model_id suitable for use in endpoint URLs
"""
from litellm.passthrough.utils import CommonUtils
import re
# Create a temporary endpoint with the model_id to check if encoding is needed
temp_endpoint = f"/model/{model_id}/converse"
encoded_temp_endpoint = CommonUtils.encode_bedrock_runtime_modelid_arn(temp_endpoint)
# Extract the encoded model_id from the temporary endpoint
encoded_model_id_match = re.search(r'/model/([^/]+)/', encoded_temp_endpoint)
if encoded_model_id_match:
return encoded_model_id_match.group(1)
else:
# Fallback to original model_id if extraction fails
return model_id
def get_complete_url(
self,
api_base: Optional[str],
@ -34,11 +65,12 @@ class BedrockPassthroughConfig(
litellm_params: dict,
) -> Tuple["URL", str]:
optional_params = litellm_params.copy()
model_id = optional_params.get("model_id", None)
aws_region_name = self._get_aws_region_name(
optional_params=optional_params,
model=model,
model_id=None,
model_id=model_id,
)
aws_bedrock_runtime_endpoint = optional_params.get("aws_bedrock_runtime_endpoint")
@ -49,6 +81,16 @@ class BedrockPassthroughConfig(
endpoint_type="runtime",
)
# If model_id is provided (e.g., Application Inference Profile ARN), use it in the endpoint
# instead of the translated model name
if model_id is not None:
import re
# Encode the model_id if it's an ARN to properly handle special characters
encoded_model_id = self._encode_model_id_for_endpoint(model_id)
# Replace the model name in the endpoint with the encoded model_id
endpoint = re.sub(r'model/[^/]+/', f'model/{encoded_model_id}/', endpoint)
return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url
def sign_request(

737
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -14,6 +14,7 @@ from typing import (
)
import httpx # type: ignore
from openai.types.file_deleted import FileDeleted
import litellm
import litellm.litellm_core_utils
@ -71,6 +72,7 @@ from litellm.types.containers.main import (
ContainerObject,
DeleteContainerResult,
)
from litellm.types.files import TwoStepFileUploadConfig
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -82,6 +84,7 @@ from litellm.types.llms.anthropic_skills import (
from litellm.types.llms.openai import (
CreateBatchRequest,
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAIFileObject,
ResponseInputParam,
@ -2782,6 +2785,38 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
def _extract_upload_url_from_response(
self,
response: httpx.Response,
upload_url_location: str,
upload_url_key: str = "upload_url",
) -> tuple[Optional[str], Optional[dict]]:
"""
Extract upload URL from initial file creation response.
Args:
response: HTTP response from initial file creation request
upload_url_location: Where to find URL ('headers' or 'body')
upload_url_key: Key name for URL in response body (default: 'upload_url')
Returns:
Tuple of (upload_url, response_data)
- upload_url: The extracted upload URL, or None if not found
- response_data: Parsed response body (for 'body' location), or None
"""
if upload_url_location == "headers":
# Google Cloud Storage style - URL in X-Goog-Upload-URL header
upload_url = response.headers.get("X-Goog-Upload-URL")
return upload_url, None
else:
# Response body style (e.g., Manus, S3 presigned URLs)
try:
response_data = response.json()
upload_url = response_data.get(upload_url_key)
return upload_url, response_data if upload_url else None
except Exception:
return None, None
def create_file(
self,
create_file_data: CreateFileRequest,
@ -2844,14 +2879,58 @@ class BaseLLMHTTPHandler:
else:
sync_httpx_client = client
if isinstance(transformed_request, dict) and "method" in transformed_request:
if isinstance(transformed_request, dict) and "initial_request" in transformed_request:
# Handle two-step uploads (TwoStepFileUploadConfig)
# Used by providers like Manus, Google Cloud Storage
try:
# Step 1: Initial request to get upload URL
initial_response = sync_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response
upload_url, initial_response_data = self._extract_upload_url_from_response(
response=initial_response,
upload_url_location=transformed_request.get("upload_url_location", "headers"),
upload_url_key=transformed_request.get("upload_url_key", "upload_url"),
)
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_method = transformed_request["upload_request"].get("method", "POST").lower()
upload_response = getattr(sync_httpx_client, upload_method)(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
# Store initial response for transformation
if initial_response_data:
litellm_params["initial_file_response"] = initial_response_data
except Exception as e:
raise self._handle_error(
e=e,
provider_config=provider_config,
)
elif isinstance(transformed_request, dict) and "method" in transformed_request and "initial_request" not in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request = cast(Dict[str, Any], transformed_request)
upload_response = getattr(
sync_httpx_client, transformed_request["method"].lower()
sync_httpx_client, presigned_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
url=presigned_request["url"],
headers=presigned_request["headers"],
data=presigned_request["data"],
timeout=timeout,
)
elif isinstance(transformed_request, str) or isinstance(
@ -2879,36 +2958,7 @@ class BaseLLMHTTPHandler:
timeout=timeout,
)
else:
try:
# Step 1: Initial request to get upload URL
initial_response = sync_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response headers
upload_url = initial_response.headers.get("X-Goog-Upload-URL")
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_response = sync_httpx_client.post(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=provider_config,
)
raise ValueError(f"Unsupported transformed_request type: {type(transformed_request)}")
# Store the upload URL in litellm_params for the transformation method
litellm_params_with_url = dict(litellm_params)
@ -2923,7 +2973,7 @@ class BaseLLMHTTPHandler:
async def async_create_file(
self,
transformed_request: Union[bytes, str, dict],
transformed_request: Union[bytes, str, dict, "TwoStepFileUploadConfig"],
litellm_params: dict,
provider_config: BaseFilesConfig,
headers: dict,
@ -2955,14 +3005,59 @@ class BaseLLMHTTPHandler:
},
)
if isinstance(transformed_request, dict) and "method" in transformed_request:
if isinstance(transformed_request, dict) and "initial_request" in transformed_request:
# Handle two-step uploads (TwoStepFileUploadConfig)
# Used by providers like Manus, Google Cloud Storage
try:
# Step 1: Initial request to get upload URL
initial_response = await async_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response
upload_url, initial_response_data = self._extract_upload_url_from_response(
response=initial_response,
upload_url_location=transformed_request.get("upload_url_location", "headers"),
upload_url_key=transformed_request.get("upload_url_key", "upload_url"),
)
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_method = transformed_request["upload_request"].get("method", "POST").lower()
upload_response = await getattr(async_httpx_client, upload_method)(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
# Store initial response for transformation
if initial_response_data:
litellm_params["initial_file_response"] = initial_response_data
except Exception as e:
verbose_logger.exception(f"Error creating file: {e}")
raise self._handle_error(
e=e,
provider_config=provider_config,
)
elif isinstance(transformed_request, dict) and "method" in transformed_request and "initial_request" not in transformed_request:
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request = cast(Dict[str, Any], transformed_request)
upload_response = await getattr(
async_httpx_client, transformed_request["method"].lower()
async_httpx_client, presigned_request["method"].lower()
)(
url=transformed_request["url"],
headers=transformed_request["headers"],
data=transformed_request["data"],
url=presigned_request["url"],
headers=presigned_request["headers"],
data=presigned_request["data"],
timeout=timeout,
)
elif isinstance(transformed_request, str) or isinstance(
@ -2990,37 +3085,7 @@ class BaseLLMHTTPHandler:
timeout=timeout,
)
else:
try:
# Step 1: Initial request to get upload URL
initial_response = await async_httpx_client.post(
url=api_base,
headers={
**headers,
**transformed_request["initial_request"]["headers"],
},
data=json.dumps(transformed_request["initial_request"]["data"]),
timeout=timeout,
)
# Extract upload URL from response headers
upload_url = initial_response.headers.get("X-Goog-Upload-URL")
if not upload_url:
raise ValueError("Failed to get upload URL from initial request")
# Step 2: Upload the actual file
upload_response = await async_httpx_client.post(
url=upload_url,
headers=transformed_request["upload_request"]["headers"],
data=transformed_request["upload_request"]["data"],
timeout=timeout,
)
except Exception as e:
verbose_logger.exception(f"Error creating file: {e}")
raise self._handle_error(
e=e,
provider_config=provider_config,
)
raise ValueError(f"Unsupported transformed_request type: {type(transformed_request)}")
return provider_config.transform_create_file_response(
model=None,
@ -3734,29 +3799,525 @@ class BaseLLMHTTPHandler:
logging_obj=logging_obj,
)
def list_files(self):
def retrieve_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]:
"""
Lists all files
Retrieve file metadata by ID
"""
pass
if _is_async:
return self.async_retrieve_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
def delete_file(self):
"""
Deletes a file
"""
pass
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
def retrieve_file(self):
"""
Returns the metadata of the file
"""
pass
# Get URL and params from provider config
url, params = provider_config.transform_retrieve_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
def retrieve_file_content(self):
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_retrieve_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_retrieve_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> OpenAIFileObject:
"""
Returns the content of the file
Async retrieve file metadata by ID
"""
pass
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_retrieve_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_retrieve_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def delete_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union["FileDeleted", Coroutine[Any, Any, "FileDeleted"]]:
"""
Delete a file by ID
"""
if _is_async:
return self.async_delete_file(
file_id=file_id,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_delete_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = sync_httpx_client.delete(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_delete_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_delete_file(
self,
file_id: str,
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> "FileDeleted":
"""
Async delete a file by ID
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_delete_file_request(
file_id=file_id,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_id,
},
)
try:
response = await async_httpx_client.delete(
url=url, headers=headers, params=params, timeout=timeout
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_delete_file_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def list_files(
self,
purpose: Optional[str],
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[List[OpenAIFileObject], Coroutine[Any, Any, List[OpenAIFileObject]]]:
"""
List all files
"""
if _is_async:
return self.async_list_files(
purpose=purpose,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_list_files_request(
purpose=purpose,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"purpose": purpose,
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_list_files_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_list_files(
self,
purpose: Optional[str],
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> List[OpenAIFileObject]:
"""
Async list all files
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_list_files_request(
purpose=purpose,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"purpose": purpose,
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_list_files_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def retrieve_file_content(
self,
file_content_request: "FileContentRequest",
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union["HttpxBinaryResponseContent", Coroutine[Any, Any, "HttpxBinaryResponseContent"]]:
"""
Retrieve file content by ID
"""
if _is_async:
return self.async_retrieve_file_content(
file_content_request=file_content_request,
provider_config=provider_config,
litellm_params=litellm_params,
headers=headers,
logging_obj=logging_obj,
client=client,
timeout=timeout,
)
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client()
else:
sync_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_file_content_request(
file_content_request=file_content_request,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_content_request.get("file_id"),
},
)
try:
response = sync_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_file_content_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
async def async_retrieve_file_content(
self,
file_content_request: "FileContentRequest",
provider_config: BaseFilesConfig,
litellm_params: dict,
headers: dict,
logging_obj: LiteLLMLoggingObj,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> "HttpxBinaryResponseContent":
"""
Async retrieve file content by ID
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=provider_config.custom_llm_provider
)
else:
async_httpx_client = client
# Get URL and params from provider config
url, params = provider_config.transform_file_content_request(
file_content_request=file_content_request,
optional_params={},
litellm_params=litellm_params,
)
# Validate environment and get headers
headers = provider_config.validate_environment(
api_key=litellm_params.get("api_key"),
headers=headers,
model="",
messages=[],
optional_params={},
litellm_params=litellm_params,
)
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"file_id": file_content_request.get("file_id"),
},
)
try:
response = await async_httpx_client.get(
url=url, headers=headers, params=params
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
return provider_config.transform_file_content_response(
raw_response=response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def _prepare_fake_stream_request(
self,

77
litellm/llms/deepinfra/chat/transformation.py
View file

@ -1,9 +1,11 @@
from typing import Optional, Tuple, Union
import json
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
import litellm
from litellm.constants import MIN_NON_ZERO_TEMPERATURE
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
class DeepInfraConfig(OpenAIGPTConfig):
@ -117,6 +119,79 @@ class DeepInfraConfig(OpenAIGPTConfig):
optional_params[param] = value
return optional_params
def _transform_tool_message_content(self, messages: List[AllMessageValues]) -> List[AllMessageValues]:
"""
Transform tool message content from array to string format for DeepInfra compatibility.
DeepInfra requires tool message content to be a string, not an array.
This method converts tool message content from array format to string format.
Example transformation:
- Input: {"role": "tool", "content": [{"type": "text", "text": "20"}]}
- Output: {"role": "tool", "content": "20"}
Or if content is complex:
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
if message.get("role") == "tool":
content = message.get("content")
# If content is a list/array, convert it to string
if isinstance(content, list):
# Check if it's a simple single text item
if (
len(content) == 1
and isinstance(content[0], dict)
and content[0].get("type") == "text"
and "text" in content[0]
):
# Extract just the text value for simple cases
message["content"] = content[0]["text"]
else:
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[False] = False
) -> List[AllMessageValues]:
...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
"""
Transform messages for DeepInfra compatibility.
Handles both sync and async transformations.
"""
if is_async:
# For async case, create an async function that awaits parent and applies our transformation
async def _async_transform():
# Call parent with is_async=True (literal) for async case
parent_result = super(DeepInfraConfig, self)._transform_messages(
messages=messages, model=model, is_async=cast(Literal[True], True)
)
transformed_messages = await parent_result
return self._transform_tool_message_content(transformed_messages)
return _async_transform()
else:
# Call parent with is_async=False (literal) for sync case
parent_result = super()._transform_messages(
messages=messages, model=model, is_async=cast(Literal[False], False)
)
# For sync case, parent_result is already the transformed messages
return self._transform_tool_message_content(parent_result)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:

1
litellm/llms/gemini/chat/transformation.py
View file

@ -87,6 +87,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"stop",
"logprobs",
"frequency_penalty",
"presence_penalty",
"modalities",
"parallel_tool_calls",
"web_search_options",

66
litellm/llms/gemini/files/transformation.py
View file

@ -7,6 +7,7 @@ import time
from typing import List, Optional
import httpx
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
@ -17,6 +18,7 @@ from litellm.llms.base_llm.files.transformation import (
from litellm.types.llms.gemini import GeminiCreateFilesResponseObject
from litellm.types.llms.openai import (
CreateFileRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
)
@ -171,3 +173,67 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
except Exception as e:
verbose_logger.exception(f"Error parsing file upload response: {str(e)}")
raise ValueError(f"Error parsing file upload response: {str(e)}")
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file listing")
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("GoogleAIStudioFilesHandler does not support file content retrieval")

1
litellm/llms/gemini/google_genai/transformation.py
View file

@ -89,6 +89,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"audio_timestamp",
"automatic_function_calling",
"thinking_config",
"image_config",
]
def map_generate_content_optional_params(

2
litellm/llms/manus/__init__.py Normal file
View file

@ -0,0 +1,2 @@
# Manus provider implementation

2
litellm/llms/manus/files/__init__.py Normal file
View file

@ -0,0 +1,2 @@
# Manus Files API implementation

439
litellm/llms/manus/files/transformation.py Normal file
View file

@ -0,0 +1,439 @@
"""
Manus Files API implementation.
Manus has an OpenAI-compatible Files API with some differences:
- Uses API_KEY header instead of Authorization: Bearer
- File upload is a two-step process:
1. Create file record to get upload URL
2. Upload file content to the upload URL
Reference: https://open.manus.im/docs/openai-compatibility#file-management
"""
import time
from typing import Any, Dict, List, Optional, Union
import httpx
from openai.types.file_deleted import FileDeleted
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
LiteLLMLoggingObj,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.secret_managers.main import get_secret_str
from litellm.types.files import TwoStepFileUploadConfig, TwoStepFileUploadRequest
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
)
from litellm.types.utils import LlmProviders
MANUS_API_BASE = "https://api.manus.im"
class ManusFilesConfig(BaseFilesConfig):
"""
Configuration for Manus Files API.
Manus uses:
- API_KEY header for authentication (not Authorization: Bearer)
- Two-step file upload process
- Content-Type: application/json for all requests
Reference: https://open.manus.im/docs/openai-compatibility#file-management
"""
def __init__(self):
pass
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.MANUS
def validate_environment(
self,
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for Manus API.
Manus uses API_KEY header instead of Authorization: Bearer.
For file uploads, don't set Content-Type - httpx will set it for multipart.
"""
api_key = (
api_key
or litellm.api_key
or get_secret_str("MANUS_API_KEY")
)
if not api_key:
raise ValueError(
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
)
# Manus uses API_KEY header, not Authorization: Bearer
# Manus requires Content-Type: application/json for all requests (even GET)
headers.update(
{
"API_KEY": api_key,
"Content-Type": "application/json",
}
)
return headers
def get_supported_openai_params(
self, model: str
) -> List[OpenAICreateFileRequestOptionalParams]:
"""
Return supported OpenAI file creation parameters for Manus.
Manus supports the standard 'purpose' parameter.
"""
return ["purpose"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to Manus-specific parameters.
Manus is OpenAI-compatible, so no special mapping needed.
"""
return optional_params
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for Manus Files API endpoint.
Returns:
str: The full URL for the Manus /v1/files endpoint
"""
api_base = (
api_base
or litellm.api_base
or get_secret_str("MANUS_API_BASE")
or MANUS_API_BASE
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
# Manus API uses /v1/files endpoint
if api_base.endswith("/v1"):
return f"{api_base}/files"
return f"{api_base}/v1/files"
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
"""
Return the appropriate error class for Manus API errors.
Uses OpenAIError since Manus is OpenAI-compatible.
"""
return OpenAIError(
status_code=status_code,
message=error_message,
headers=headers,
)
def transform_create_file_request(
self,
model: str,
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
) -> TwoStepFileUploadConfig:
"""
Transform OpenAI-style file creation request into Manus's two-step format.
Manus API spec (https://open.manus.im/docs/openai-compatibility#file-management):
1. POST /v1/files with JSON {"filename": "..."} → returns {"id": "...", "upload_url": "..."}
2. PUT to upload_url with raw file content
"""
# Extract file data
file_data = create_file_data.get("file")
if file_data is None:
raise ValueError("File data is required")
extracted_data = extract_file_data(file_data)
filename = extracted_data["filename"] or f"file_{int(time.time())}"
content = extracted_data["content"]
# Get API base URL
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
)
# Get API key
api_key = (
litellm_params.get("api_key")
or litellm.api_key
or get_secret_str("MANUS_API_KEY")
)
if not api_key:
raise ValueError(
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
)
# Build typed two-step upload config
return TwoStepFileUploadConfig(
initial_request=TwoStepFileUploadRequest(
method="POST",
url=api_base,
headers={
"API_KEY": api_key,
"Content-Type": "application/json",
},
data={"filename": filename},
),
upload_request=TwoStepFileUploadRequest(
method="PUT",
url="", # Will be populated from initial_request response
headers={},
data=content,
),
upload_url_location="body",
upload_url_key="upload_url",
)
def transform_create_file_response(
self,
model: Optional[str],
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""
Transform Manus's file upload response into OpenAI-style FileObject.
For two-step uploads, the handler stores the initial response in litellm_params.
We need to return the file object from the initial POST, not the final PUT.
Manus initial response format:
{
"id": "file-abc123xyz",
"object": "file",
"filename": "document.pdf",
"status": "pending",
"upload_url": "https://...",
"upload_expires_at": "...",
"created_at": "..."
}
"""
try:
# For two-step uploads, get the initial response from litellm_params
initial_response_data = litellm_params.get("initial_file_response")
if initial_response_data:
response_json = initial_response_data
else:
# Log raw response for debugging
verbose_logger.debug(f"Manus raw response text: {raw_response.text}")
response_json = raw_response.json()
verbose_logger.debug(f"Manus file response: {response_json}")
# Parse created_at timestamp
created_at_str = response_json.get("created_at", "")
if created_at_str:
try:
# Try parsing ISO format
created_at = int(
time.mktime(
time.strptime(
created_at_str.replace("Z", "+00:00")[:19],
"%Y-%m-%dT%H:%M:%S",
)
)
)
except (ValueError, TypeError):
created_at = int(time.time())
else:
created_at = int(time.time())
return OpenAIFileObject(
id=response_json.get("id", ""),
bytes=response_json.get("bytes", 0),
created_at=created_at,
filename=response_json.get("filename", ""),
object="file",
purpose=response_json.get("purpose", "assistants"),
status="uploaded", # After successful upload, status is uploaded
status_details=response_json.get("status_details"),
)
except Exception as e:
verbose_logger.exception(f"Error parsing Manus file response: {str(e)}")
raise ValueError(f"Error parsing Manus file response: {str(e)}")
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for retrieving a file."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
def transform_retrieve_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
"""Transform retrieve file response."""
return self.transform_create_file_response(
model=None,
raw_response=raw_response,
logging_obj=logging_obj,
litellm_params=litellm_params,
)
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for deleting a file."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
def transform_delete_file_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
"""Transform delete file response."""
response_json = raw_response.json()
return FileDeleted(**response_json)
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for listing files."""
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
params = {}
if purpose:
params["purpose"] = purpose
return api_base, params
def transform_list_files_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
"""Transform list files response."""
response_json = raw_response.json()
files_data = response_json.get("data", [])
return [self._parse_file_dict(f) for f in files_data]
def _parse_file_dict(self, file_dict: Dict[str, Any]) -> OpenAIFileObject:
"""Parse a file dict into OpenAIFileObject."""
created_at_str = file_dict.get("created_at", "")
if created_at_str:
try:
created_at = int(
time.mktime(
time.strptime(
created_at_str.replace("Z", "+00:00")[:19],
"%Y-%m-%dT%H:%M:%S",
)
)
)
except (ValueError, TypeError):
created_at = int(time.time())
else:
created_at = int(time.time())
return OpenAIFileObject(
id=file_dict.get("id", ""),
bytes=file_dict.get("bytes", 0),
created_at=created_at,
filename=file_dict.get("filename", ""),
object="file",
purpose=file_dict.get("purpose", "assistants"),
status=file_dict.get("status", "uploaded"),
status_details=file_dict.get("status_details"),
)
def transform_file_content_request(
self,
file_content_request: FileContentRequest,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
"""Get URL and params for retrieving file content."""
file_id = file_content_request.get("file_id")
api_base = self.get_complete_url(
api_base=litellm_params.get("api_base"),
api_key=litellm_params.get("api_key"),
model="",
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}/content", {}
def transform_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
"""Transform file content response."""
return HttpxBinaryResponseContent(response=raw_response)

2
litellm/llms/manus/responses/__init__.py Normal file
View file

@ -0,0 +1,2 @@
# Manus Responses API implementation

340
litellm/llms/manus/responses/transformation.py Normal file
View file

@ -0,0 +1,340 @@
import uuid
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseInputParam,
ResponsesAPIResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
MANUS_API_BASE = "https://api.manus.im"
class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
Configuration for Manus API's Responses API.
Manus API is OpenAI-compatible but has some differences:
- API key passed via `API_KEY` header (not `Authorization: Bearer`)
- Model format: `manus/{agent_profile}` (e.g., `manus/manus-1.6`)
- Requires `extra_body` with `task_mode: "agent"` and `agent_profile`
Reference: https://open.manus.im/docs/openai-compatibility
"""
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.MANUS
def should_fake_stream(
self,
model: Optional[str],
stream: Optional[bool],
custom_llm_provider: Optional[str] = None,
) -> bool:
"""
Manus API doesn't support real-time streaming.
It returns a task that runs asynchronously.
We fake streaming by converting the response into streaming events.
"""
return stream is True
def _extract_agent_profile(self, model: str) -> str:
"""
Extract agent profile from model name.
Model format: `manus/{agent_profile}`
Examples: `manus/manus-1.6`, `manus/manus-1.6-lite`, `manus/manus-1.6-max`
Returns:
str: The agent profile (e.g., "manus-1.6")
"""
if "/" in model:
return model.split("/", 1)[1]
# If no slash, assume the model name itself is the agent profile
return model
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
"""
Validate environment and set up headers for Manus API.
Manus uses `API_KEY` header instead of `Authorization: Bearer`.
"""
litellm_params = litellm_params or GenericLiteLLMParams()
api_key = (
litellm_params.api_key
or litellm.api_key
or get_secret_str("MANUS_API_KEY")
)
if not api_key:
raise ValueError(
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
)
# Manus uses API_KEY header, not Authorization: Bearer
# Content-Type is required for all requests (including GET)
headers.update(
{
"API_KEY": api_key,
"Content-Type": "application/json",
}
)
return headers
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for Manus Responses API endpoint.
Returns:
str: The full URL for the Manus /v1/responses endpoint
"""
api_base = (
api_base
or litellm.api_base
or get_secret_str("MANUS_API_BASE")
or MANUS_API_BASE
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
# Manus API uses /v1/responses endpoint (OpenAI-compatible)
if api_base.endswith("/v1"):
return f"{api_base}/responses"
return f"{api_base}/v1/responses"
def transform_responses_api_request(
self,
model: str,
input: Union[str, ResponseInputParam],
response_api_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
"""
Transform the request for Manus API.
Manus requires:
- `task_mode: "agent"` in the request body
- `agent_profile` extracted from model name in the request body
"""
# First, get the base OpenAI request
base_request = super().transform_responses_api_request(
model=model,
input=input,
response_api_optional_request_params=response_api_optional_request_params,
litellm_params=litellm_params,
headers=headers,
)
# Extract agent profile from model name
agent_profile = self._extract_agent_profile(model=model)
# Add Manus-specific parameters directly to the request body
# These will be sent as part of the request
base_request["task_mode"] = "agent"
base_request["agent_profile"] = agent_profile
# Merge any existing extra_body into the request
extra_body = response_api_optional_request_params.get("extra_body", {}) or {}
if extra_body:
base_request.update(extra_body)
verbose_logger.debug(
f"Manus: Using agent_profile={agent_profile}, task_mode=agent"
)
return base_request
def transform_response_api_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
"""
Transform Manus API response to OpenAI-compatible format.
Manus uses camelCase (createdAt) instead of snake_case (created_at).
"""
try:
logging_obj.post_call(
original_response=raw_response.text,
additional_args={"complete_input_dict": {}},
)
raw_response_json = raw_response.json()
# Manus uses camelCase "createdAt" instead of snake_case "created_at"
if "createdAt" in raw_response_json and "created_at" not in raw_response_json:
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["createdAt"]
)
# Ensure created_at is set
if "created_at" in raw_response_json:
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["created_at"]
)
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
)
raw_response_headers = dict(raw_response.headers)
processed_headers = process_response_headers(raw_response_headers)
# Ensure reasoning is an empty dict if not present, OpenAI SDK does not allow None
if "reasoning" not in raw_response_json or raw_response_json.get("reasoning") is None:
raw_response_json["reasoning"] = {}
if "text" not in raw_response_json or raw_response_json.get("text") is None:
raw_response_json["text"] = {}
if "output" not in raw_response_json or raw_response_json.get("output") is None:
raw_response_json["output"] = []
# Ensure usage is present with default values if not provided
if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
raw_response_json["usage"] = ResponseAPIUsage(
input_tokens=0,
output_tokens=0,
total_tokens=0,
)
# Ensure id is present - failed responses may not include it
if "id" not in raw_response_json or raw_response_json.get("id") is None:
# Generate a placeholder id for failed responses
# This allows the response object to be created even when the API doesn't return an id
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
except Exception:
verbose_logger.debug(
f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
)
response = ResponsesAPIResponse.model_construct(**raw_response_json)
# Store processed headers in additional_headers so they get returned to the client
response._hidden_params["additional_headers"] = processed_headers
response._hidden_params["headers"] = raw_response_headers
return response
def transform_get_response_api_request(
self,
response_id: str,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
"""
Transform the get response API request into a URL and data.
Manus API follows OpenAI-compatible format:
- GET /v1/responses/{response_id}
Reference: https://open.manus.im/docs/openai-compatibility
"""
url = f"{api_base}/{response_id}"
data: Dict = {}
return url, data
def transform_get_response_api_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ResponsesAPIResponse:
"""
Transform Manus API GET response to OpenAI-compatible format.
Manus uses camelCase (createdAt) instead of snake_case (created_at).
Same transformation as transform_response_api_response.
"""
try:
logging_obj.post_call(
original_response=raw_response.text,
additional_args={"complete_input_dict": {}},
)
raw_response_json = raw_response.json()
# Manus uses camelCase "createdAt" instead of snake_case "created_at"
if "createdAt" in raw_response_json and "created_at" not in raw_response_json:
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["createdAt"]
)
# Ensure created_at is set
if "created_at" in raw_response_json:
raw_response_json["created_at"] = _safe_convert_created_field(
raw_response_json["created_at"]
)
except Exception:
raise OpenAIError(
message=raw_response.text, status_code=raw_response.status_code
)
raw_response_headers = dict(raw_response.headers)
processed_headers = process_response_headers(raw_response_headers)
# Ensure reasoning, text, output, and usage are present with defaults
if "reasoning" not in raw_response_json or raw_response_json.get("reasoning") is None:
raw_response_json["reasoning"] = {}
if "text" not in raw_response_json or raw_response_json.get("text") is None:
raw_response_json["text"] = {}
if "output" not in raw_response_json or raw_response_json.get("output") is None:
raw_response_json["output"] = []
if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
raw_response_json["usage"] = ResponseAPIUsage(
input_tokens=0,
output_tokens=0,
total_tokens=0,
)
# Ensure id is present - failed responses may not include it
if "id" not in raw_response_json or raw_response_json.get("id") is None:
# Generate a placeholder id for failed responses
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
except Exception:
verbose_logger.debug(
f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
)
response = ResponsesAPIResponse.model_construct(**raw_response_json)
# Store processed headers in additional_headers so they get returned to the client
response._hidden_params["additional_headers"] = processed_headers
response._hidden_params["headers"] = raw_response_headers
return response

5
litellm/llms/oci/chat/transformation.py
View file

@ -1124,8 +1124,11 @@ def adapt_messages_to_generic_oci_standard_content_message(
elif type == "image_url":
image_url = content_item.get("image_url")
# Handle both OpenAI format (object with url) and string format
if isinstance(image_url, dict):
image_url = image_url.get("url")
if not isinstance(image_url, str):
raise Exception("Prop `image_url` is not a string")
raise Exception("Prop `image_url` must be a string or an object with a `url` property")
new_content.append(OCIImageContentPart(imageUrl=image_url))
return OCIMessage(

6
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -771,9 +771,9 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
return ModelResponseStream(
id=chunk["id"],
object="chat.completion.chunk",
created=chunk["created"],
model=chunk["model"],
choices=chunk["choices"],
created=chunk.get("created"),
model=chunk.get("model"),
choices=chunk.get("choices", []),
)
except Exception as e:
raise e

4
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -83,6 +83,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["structured_messages"] = (
messages # pass the openai /chat/completions messages to the guardrail, as-is
)
# Pass tools (function definitions) to the guardrail
tools = data.get("tools")
if tools:
inputs["tools"] = tools
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,

4
litellm/llms/openai_like/providers.json
View file

@ -61,6 +61,10 @@
"max_completion_tokens": "max_tokens"
}
},
"abliteration": {
"base_url": "https://api.abliteration.ai/v1",
"api_key_env": "ABLITERATION_API_KEY"
},
"llamagate": {
"base_url": "https://api.llamagate.dev/v1",
"api_key_env": "LLAMAGATE_API_KEY",

182
litellm/llms/openrouter/embedding/transformation.py Normal file
View file

@ -0,0 +1,182 @@
"""
OpenRouter Embedding API Configuration.
This module provides the configuration for OpenRouter's Embedding API.
OpenRouter is OpenAI-compatible and supports embeddings via the /v1/embeddings endpoint.
Docs: https://openrouter.ai/docs
"""
from typing import TYPE_CHECKING, Any, Optional
import httpx
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
from litellm.types.llms.openai import AllEmbeddingInputValues
from litellm.types.utils import EmbeddingResponse
from litellm.utils import convert_to_model_response_object
from ..common_utils import OpenRouterException
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OpenrouterEmbeddingConfig(BaseEmbeddingConfig):
"""
Configuration for OpenRouter's Embedding API.
Reference: https://openrouter.ai/docs
"""
def validate_environment(
self,
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for OpenRouter API.
OpenRouter requires:
- Authorization header with Bearer token
- HTTP-Referer header (site URL)
- X-Title header (app name)
"""
from litellm import get_secret
# Get OpenRouter-specific headers
openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai"
openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM"
openrouter_headers = {
"HTTP-Referer": openrouter_site_url,
"X-Title": openrouter_app_name,
"Content-Type": "application/json",
}
# Add Authorization header if api_key is provided
if api_key:
openrouter_headers["Authorization"] = f"Bearer {api_key}"
# Merge with existing headers (user's extra_headers take priority)
merged_headers = {**openrouter_headers, **headers}
return merged_headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for OpenRouter Embedding API endpoint.
"""
# api_base is already set to https://openrouter.ai/api/v1 in main.py
# Remove trailing slashes
if api_base:
api_base = api_base.rstrip("/")
else:
api_base = "https://openrouter.ai/api/v1"
# Return the embeddings endpoint
return f"{api_base}/embeddings"
def transform_embedding_request(
self,
model: str,
input: AllEmbeddingInputValues,
optional_params: dict,
headers: dict,
) -> dict:
"""
Transform embedding request to OpenRouter format (OpenAI-compatible).
"""
# Ensure input is a list
if isinstance(input, str):
input = [input]
# OpenRouter expects the full model name (e.g., google/gemini-embedding-001)
# Strip 'openrouter/' prefix if present
if model.startswith("openrouter/"):
model = model.replace("openrouter/", "", 1)
return {
"model": model,
"input": input,
**optional_params,
}
def transform_embedding_response(
self,
model: str,
raw_response: httpx.Response,
model_response: EmbeddingResponse,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str],
request_data: dict,
optional_params: dict,
litellm_params: dict,
) -> EmbeddingResponse:
"""
Transform embedding response from OpenRouter format (OpenAI-compatible).
"""
logging_obj.post_call(original_response=raw_response.text)
# OpenRouter returns standard OpenAI-compatible embedding response
response_json = raw_response.json()
return convert_to_model_response_object(
response_object=response_json,
model_response_object=model_response,
response_type="embedding",
)
def get_supported_openai_params(self, model: str) -> list:
"""
Get list of supported OpenAI parameters for OpenRouter embeddings.
"""
return [
"timeout",
"dimensions",
"encoding_format",
"user",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to OpenRouter format.
"""
for param, value in non_default_params.items():
if param in self.get_supported_openai_params(model):
optional_params[param] = value
return optional_params
def get_error_class(
self, error_message: str, status_code: int, headers: Any
) -> Any:
"""
Get the error class for OpenRouter errors.
"""
return OpenRouterException(
message=error_message,
status_code=status_code,
headers=headers,
)

4
litellm/llms/replicate/chat/handler.py
View file

@ -213,7 +213,7 @@ def completion(
response = httpx_client.get(url=prediction_url, headers=headers)
if (
response.status_code == 200
and response.json().get("status") == "processing"
and response.json().get("status") in ["processing", "starting"]
):
continue
return litellm.ReplicateConfig().transform_response(
@ -284,7 +284,7 @@ async def async_completion(
response = await async_handler.get(url=prediction_url, headers=headers)
if (
response.status_code == 200
and response.json().get("status") == "processing"
and response.json().get("status") in ["processing", "starting"]
):
continue
return litellm.ReplicateConfig().transform_response(

68
litellm/llms/vertex_ai/files/transformation.py
View file

@ -1,11 +1,12 @@
import json
import os
import time
from litellm._uuid import uuid
from typing import Any, Dict, List, Optional, Tuple, Union
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from litellm._uuid import uuid
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -24,6 +25,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileTypes,
HttpxBinaryResponseContent,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
@ -333,6 +335,70 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
status_code=status_code, message=error_message, headers=headers
)
def transform_retrieve_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file retrieval")
def transform_retrieve_file_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> OpenAIFileObject:
raise NotImplementedError("VertexAIFilesConfig does not support file retrieval")
def transform_delete_file_request(
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file deletion")
def transform_delete_file_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> FileDeleted:
raise NotImplementedError("VertexAIFilesConfig does not support file deletion")
def transform_list_files_request(
self,
purpose: Optional[str],
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file listing")
def transform_list_files_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> List[OpenAIFileObject]:
raise NotImplementedError("VertexAIFilesConfig does not support file listing")
def transform_file_content_request(
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
) -> tuple[str, dict]:
raise NotImplementedError("VertexAIFilesConfig does not support file content retrieval")
def transform_file_content_response(
self,
raw_response: Response,
logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> HttpxBinaryResponseContent:
raise NotImplementedError("VertexAIFilesConfig does not support file content retrieval")
class VertexAIJsonlFilesTransformation(VertexGeminiConfig):
"""

2
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -115,7 +115,7 @@ def _process_gemini_image(
and (image_type := format or _get_image_mime_type_from_url(image_url))
is not None
):
file_data = FileDataType(file_uri=image_url, mime_type=image_type)
file_data = FileDataType(mime_type=image_type, file_uri=image_url)
part = {"file_data": file_data}
if media_resolution_enum is not None and model is not None:

Some files were not shown because too many files have changed in this diff Show more