scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'origin/main' into sap-ai-hub-module-config-list

Browse source 
# Conflicts:
#	litellm/llms/sap/chat/transformation.py
This commit is contained in:
Vasilisa Parshikova 2026-03-05 13:02:41 +04:00
commit 2968d471d3
303 changed files with 27005 additions and 9273 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
.circleci/config.yml
View file

@ -3689,6 +3689,114 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_azure_batches_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.12
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.12 -y
conda activate myenv
python --version
- run:
name: Install Poetry
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
pip install poetry
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=llmproxy \
-e POSTGRES_PASSWORD=dbpassword9090 \
-e POSTGRES_DB=litellm \
-p 5432:5432 \
postgres:15
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- run:
name: Install system dependencies
command: |
sudo apt-get update -y
sudo apt-get install -y libpq-dev
- run:
name: Install Dependencies
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- run:
name: Setup litellm-enterprise
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run pip install --force-reinstall --no-deps -e enterprise/
- run:
name: Generate Prisma client
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
poetry run prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run Prisma migrations
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- run:
name: Run Azure Batch E2E Tests
command: |
export PATH="$HOME/miniconda/bin:$PATH"
source $HOME/miniconda/etc/profile.d/conda.sh
conda activate myenv
export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
export USE_LOCAL_LITELLM=true
export USE_MOCK_MODELS=true
export USE_STATE_TRACKER=true
export LITELLM_LOG=DEBUG
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10 \
--junitxml=test-results/junit.xml
no_output_timeout: 30m
upload-coverage:
docker:
- image: cimg/python:3.9
@ -4458,6 +4566,12 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_e2e_azure_batches_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- llm_translation_testing:
filters:
branches:

1
.github/workflows/test-linting.yml vendored
View file

@ -32,7 +32,6 @@ jobs:
run: |
poetry lock
poetry install --with dev
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |

2
.github/workflows/test-litellm.yml vendored
View file

@ -38,7 +38,7 @@ jobs:
poetry run pip install "google-genai==1.22.0"
poetry run pip install "google-cloud-aiplatform>=1.38"
poetry run pip install "fastapi-offline==1.7.3"
poetry run pip install "python-multipart==0.0.22"
poetry run pip install "python-multipart>=0.0.20"
poetry run pip install "openapi-core"
- name: Setup litellm-enterprise as local package
run: |

90
.github/workflows/test-proxy-e2e-azure-batches.yml vendored Normal file
View file

@ -0,0 +1,90 @@
name: Proxy E2E Azure Batches Tests
on:
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
proxy_e2e_azure_batches_tests:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:15
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Cache Poetry dependencies
uses: actions/cache@v4
with:
path: |
~/.cache/pypoetry
~/.cache/pip
.venv
key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
restore-keys: |
${{ runner.os }}-poetry-e2e-batches-
${{ runner.os }}-poetry-
- name: Install dependencies
run: |
poetry config virtualenvs.in-project true
poetry install --with dev,proxy-dev --extras "proxy"
poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
- name: Setup litellm-enterprise
run: |
poetry run pip install --force-reinstall --no-deps -e enterprise/
- name: Generate Prisma client
run: |
poetry run prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
run: |
cd litellm/proxy
poetry run prisma migrate deploy --schema schema.prisma
cd ../..
- name: Run Azure Batch E2E Tests
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
USE_LOCAL_LITELLM: "true"
USE_MOCK_MODELS: "true"
USE_STATE_TRACKER: "true"
LITELLM_LOG: DEBUG
run: |
poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
-vv -s -k "test_e2e_managed_batch" \
--tb=short \
--maxfail=3 \
--durations=10

3
AGENTS.md
View file

@ -109,6 +109,8 @@ Key files:
- `litellm/proxy/auth/` - Authentication logic
- `litellm/proxy/management_endpoints/` - Admin API endpoints
**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
LiteLLM supports MCP for agent workflows:
@ -176,6 +178,7 @@ When opening issues or pull requests, follow these templates:
5. **Dependencies**: Keep dependencies minimal and well-justified
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.

16
CLAUDE.md
View file

@ -107,7 +107,21 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Proxy database access
- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features
- Separate licensing and authentication for enterprise features
### Troubleshooting: DB schema out of sync after proxy restart
`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
**Fix options:**
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

13
dev_config.yaml Normal file
View file

@ -0,0 +1,13 @@
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False

71
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -244,6 +244,47 @@ response = litellm.image_edit(
print(response)
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
#### Basic Image Edit
```python showLineNumbers title="OpenRouter Image Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Add aurora borealis to the night sky",
)
print(response)
```
#### Multiple Images Edit
```python showLineNumbers title="OpenRouter Multiple Images Edit"
import os
from litellm import image_edit
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene",
size="1536x1024", # mapped to aspect_ratio 3:2
quality="high", # mapped to image_size 4K
)
print(response)
```
</TabItem>
</Tabs>
@ -398,6 +439,34 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-F "size=1024x1024"
```
</TabItem>
<TabItem value="openrouter" label="OpenRouter">
1. Add the OpenRouter image edit model to your `config.yaml`:
```yaml showLineNumbers title="OpenRouter Proxy Configuration"
model_list:
- model_name: openrouter-image-edit
litellm_params:
model: openrouter/google/gemini-2.5-flash-image
api_key: os.environ/OPENROUTER_API_KEY
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="OpenRouter Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=openrouter-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Make the sky a vibrant purple sunset" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

170
docs/my-website/docs/mcp.md
View file

@ -336,175 +336,9 @@ litellm_settings:
## Converting OpenAPI Specs to MCP Servers
LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
LiteLLM can convert OpenAPI specifications into MCP servers, exposing any REST API as MCP tools without writing custom server code.
**Benefits:**
- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
- **Easy Testing**: Test and iterate on API integrations quickly
**Configuration:**
Add your OpenAPI-based MCP server to your `config.yaml`:
```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: sk-xxxxxxx
mcp_servers:
# OpenAPI Spec Example - Petstore API
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
# OpenAPI Spec with API Key Authentication
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
# OpenAPI Spec with Bearer Token
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
**Configuration Parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | The base URL of your API endpoint |
| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
| `auth_value` | No | Authentication value (required if `auth_type` is set) |
| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
| `description` | No | Optional description for the MCP server |
| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
### Usage Example
Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
# Standard MCP configuration
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
# Create a client that connects to the server
client = Client(config)
async def main():
async with client:
# List available tools generated from OpenAPI spec
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
# Example: Get a pet by ID (from Petstore API)
response = await client.call_tool(
name="getpetbyid",
arguments={"petId": "1"}
)
print(f"Response:\n{response}\n")
# Example: Find pets by status
response = await client.call_tool(
name="findpetsbystatus",
arguments={"status": "available"}
)
print(f"Response:\n{response}\n")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets in the petstore",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>
**How It Works**
1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
5. **Response Translation**: API responses are converted back to MCP format
**OpenAPI Spec Requirements**
Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
- **Required fields**: `paths`, `info` sections should be properly defined
- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
- **Parameters**: Request parameters should be properly documented with types and descriptions
See the **[MCP from OpenAPI Specs guide](./mcp_openapi.md)** for full setup, usage examples, and how to override tool names and descriptions.
## MCP OAuth

226
docs/my-website/docs/mcp_openapi.md Normal file
View file

@ -0,0 +1,226 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# MCP from OpenAPI Specs
LiteLLM can convert any OpenAPI/Swagger spec into an MCP server — no custom MCP server code required.
## Step 1 — Add the MCP Server
Add your OpenAPI-based server in `config.yaml`:
```yaml title="config.yaml" showLineNumbers
mcp_servers:
petstore_mcp:
url: "https://petstore.swagger.io/v2"
spec_path: "/path/to/openapi.json"
auth_type: "none"
my_api_mcp:
url: "http://0.0.0.0:8090"
spec_path: "/path/to/openapi.json"
auth_type: "api_key"
auth_value: "your-api-key-here"
secured_api_mcp:
url: "https://api.example.com"
spec_path: "/path/to/openapi.json"
auth_type: "bearer_token"
auth_value: "your-bearer-token"
```
Or from the UI: go to **MCP Servers → Add New MCP Server**, fill in the URL and spec path, and LiteLLM will fetch the spec and load all endpoints as tools.
**Configuration parameters:**
| Parameter | Required | Description |
|-----------|----------|-------------|
| `url` | Yes | Base URL of your API |
| `spec_path` | Yes | Path or URL to your OpenAPI spec (JSON or YAML) |
| `auth_type` | No | `none`, `api_key`, `bearer_token`, `basic`, `authorization`, `oauth2` |
| `auth_value` | No | Auth value (required if `auth_type` is set) |
| `description` | No | Optional description |
| `allowed_tools` | No | Allowlist of specific tools |
| `disallowed_tools` | No | Blocklist of specific tools |
**Supported spec versions:** OpenAPI 3.0.x, 3.1.x, Swagger 2.0. Each operation's `operationId` becomes the tool name — make sure they're unique.
Once tools are loaded, you'll see them in the Tool Configuration section:
<Image
img={require('../img/mcp_openapi_tools_loaded.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
## Step 2 — Optionally Override Tool Names and Descriptions
By default, tool names and descriptions come from the `operationId` and description fields in your spec. You can rename or rewrite them so MCP clients see something cleaner — without touching the upstream spec.
### From the UI
Each tool card has a pencil icon. Click it to open the inline editor:
<Image
img={require('../img/mcp_openapi_tool_edit_panel.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
- **Display Name** — overrides the name MCP clients see
- **Description** — overrides the description MCP clients see
- Leave a field blank to keep the original from the spec
After setting overrides, a purple **Custom name** badge appears on the tool card:
<Image
img={require('../img/mcp_openapi_custom_name_badge.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
<br/>
### From the API
Pass `tool_name_to_display_name` and `tool_name_to_description` in the create or update request:
```bash title="Create server with tool name overrides" showLineNumbers
curl -X POST http://localhost:4000/v1/mcp/server \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"name": "petstore_mcp",
"url": "https://petstore.swagger.io/v2",
"spec_path": "/path/to/openapi.json",
"tool_name_to_display_name": {
"getPetById": "Get Pet",
"findPetsByStatus": "List Available Pets"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID",
"findPetsByStatus": "Returns all pets matching a given status (available, pending, sold)"
}
}'
```
```bash title="Update overrides on an existing server" showLineNumbers
curl -X PUT http://localhost:4000/v1/mcp/server/{server_id} \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"tool_name_to_display_name": {
"getPetById": "Get Pet"
},
"tool_name_to_description": {
"getPetById": "Look up a pet by its ID"
}
}'
```
The map key is the **original `operationId`** from the spec — not the prefixed tool name. LiteLLM strips the server prefix before doing the lookup.
For example, if your server is `petstore_mcp`, the tool is exposed as `petstore_mcp-getPetById`. The map key is still `getPetById`.
**Before and after:**
```
# Without overrides
Tool: "petstore_mcp-getPetById"
Description: "Returns a single pet"
Tool: "petstore_mcp-findPetsByStatus"
Description: "Finds Pets by status"
# After overrides
Tool: "Get Pet"
Description: "Look up a pet by its ID"
Tool: "List Available Pets"
Description: "Returns all pets matching a given status (available, pending, sold)"
```
## Using the Server
<Tabs>
<TabItem value="fastmcp" label="Python FastMCP">
```python title="Using OpenAPI-based MCP Server" showLineNumbers
from fastmcp import Client
import asyncio
config = {
"mcpServers": {
"petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer sk-1234"
}
}
}
}
client = Client(config)
async def main():
async with client:
tools = await client.list_tools()
print(f"Available tools: {[tool.name for tool in tools]}")
response = await client.call_tool(
name="Get Pet", # overridden name
arguments={"petId": "1"}
)
print(f"Response: {response}")
if __name__ == "__main__":
asyncio.run(main())
```
</TabItem>
<TabItem value="cursor" label="Cursor IDE">
```json title="Cursor MCP Configuration" showLineNumbers
{
"mcpServers": {
"Petstore": {
"url": "http://localhost:4000/petstore_mcp/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
}
}
}
```
</TabItem>
<TabItem value="openai" label="OpenAI Responses API">
```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
curl --location 'https://api.openai.com/v1/responses' \
--header 'Content-Type: application/json' \
--header "Authorization: Bearer $OPENAI_API_KEY" \
--data '{
"model": "gpt-4o",
"tools": [
{
"type": "mcp",
"server_label": "petstore",
"server_url": "http://localhost:4000/petstore_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
}
}
],
"input": "Find all available pets",
"tool_choice": "required"
}'
```
</TabItem>
</Tabs>

1
docs/my-website/docs/providers/openai.md
View file

@ -191,6 +191,7 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5.2 | `response = completion(model="gpt-5.2", messages=messages)` |
| gpt-5.2-2025-12-11 | `response = completion(model="gpt-5.2-2025-12-11", messages=messages)` |
| gpt-5.2-chat-latest | `response = completion(model="gpt-5.2-chat-latest", messages=messages)` |
| gpt-5.3-chat-latest | `response = completion(model="gpt-5.3-chat-latest", messages=messages)` |
| gpt-5.2-pro | `response = completion(model="gpt-5.2-pro", messages=messages)` |
| gpt-5.2-pro-2025-12-11 | `response = completion(model="gpt-5.2-pro-2025-12-11", messages=messages)` |
| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |

87
docs/my-website/docs/providers/openrouter.md
View file

@ -210,3 +210,90 @@ response = image_generation(
# Cost is available in the response metadata
print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
```
## Image Edit
OpenRouter supports image editing through select models like Google Gemini image models. LiteLLM routes image edit requests to OpenRouter's chat completions endpoint with the source image sent as a base64 data URL and `modalities: ["image", "text"]`.
### Supported Models
| Model | Description |
|-------|-------------|
| `openrouter/google/gemini-2.5-flash-image` | Gemini 2.5 Flash with image editing |
See all available image models on [OpenRouter's model list](https://openrouter.ai/models?modality=image).
### Supported Parameters
| Parameter | OpenRouter Mapping | Notes |
|-----------|--------------------|-------|
| `size` | `image_config.aspect_ratio` | `1024x1024` → `1:1`, `1536x1024` → `3:2`, `1024x1536` → `2:3`, `1792x1024` → `16:9`, `1024x1792` → `9:16` |
| `quality` | `image_config.image_size` | `low`/`standard` → `1K`, `medium` → `2K`, `high`/`hd` → `4K` |
| `n` | `n` | Number of images |
:::note
`quality=high` (4K) is only supported by `google/gemini-3-pro-image-preview` and `google/gemini-3.1-flash-image-preview`. The `google/gemini-2.5-flash-image` model supports up to `medium` (2K).
:::
### Usage
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Basic image edit
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("original_image.png", "rb"),
prompt="Make the sky a vibrant purple sunset",
)
print(response)
```
### Advanced Usage with Parameters
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
# Edit with size and quality parameters
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=open("photo.png", "rb"),
prompt="Add northern lights to the sky",
size="1536x1024", # Maps to aspect_ratio 3:2
quality="high", # Maps to image_size 4K
)
# Access the edited image
image_data = response.data[0]
if image_data.b64_json:
import base64
with open("edited.png", "wb") as f:
f.write(base64.b64decode(image_data.b64_json))
```
### Multiple Images Edit
```python
from litellm import image_edit
import os
os.environ["OPENROUTER_API_KEY"] = "your-api-key"
response = image_edit(
model="openrouter/google/gemini-2.5-flash-image",
image=[
open("scene.png", "rb"),
open("style_reference.png", "rb"),
],
prompt="Blend the reference style into the scene",
)
print(response)
```

13
docs/my-website/docs/proxy/guardrails/azure_content_guardrail.md
View file

@ -100,6 +100,19 @@ AzureHarmCategories:
n/a
## Important Notes
### Azure Content Safety Character Limit
Both Azure Prompt Shield and Azure Text Moderation have a **10,000 character limit** per request. When text exceeds this limit:
- LiteLLM automatically splits the text into chunks at word boundaries (no words are broken)
- Each chunk is sent separately to the Azure Content Safety API for analysis
- If any chunk is flagged (attack detected or severity threshold exceeded), the entire request is blocked
- If all chunks are safe, the request is allowed to proceed
This applies to both `pre_call` and `post_call` hooks and ensures that long prompts are properly analyzed without breaking words or losing context.
## Further Reading

10
docs/my-website/docs/proxy/guardrails/quick_start.md
View file

@ -358,13 +358,13 @@ response = client.chat.completions.create(
}
],
extra_body={
"guardrails": [
"guardrails": {
"aporia-pre-guard": {
"extra_body": {
"success_threshold": 0.9
}
}
]
}
}
)
@ -387,13 +387,13 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"content": "what llm are you"
}
],
"guardrails": [
"guardrails": {
"aporia-pre-guard": {
"extra_body": {
"success_threshold": 0.9
}
}
]
}
}'
```
</TabItem>
@ -451,7 +451,6 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Content-Type: application/json' \
-d '{
"guardrails": ["aporia-pre-guard", "aporia-post-guard"]
}
}'
```
@ -465,7 +464,6 @@ curl --location 'http://0.0.0.0:4000/key/update' \
--data '{
"key": "sk-jNm1Zar7XfNdZXp49Z1kSQ",
"guardrails": ["aporia-pre-guard", "aporia-post-guard"]
}
}'
```

89
docs/my-website/docs/proxy/token_auth.md
View file

@ -1054,6 +1054,95 @@ curl -X GET 'http://0.0.0.0:4000/user/info?user_id=user-123' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>'
```
## [BETA] JWT-to-Virtual-Key Mapping
Map JWT identities to LiteLLM virtual keys so that JWT-authenticated users get per-user budgets, rate limits, model access controls, and spend tracking.
When a JWT comes in, LiteLLM looks up a configured claim (e.g. `email`, `sub`) in a mapping table. If a mapping exists, the request is treated as if it arrived with the corresponding virtual key — all virtual key features apply.
### Setup
Add `virtual_key_claim_field` to your JWT auth config:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
virtual_key_claim_field: "email" # JWT claim to look up (supports dot notation)
virtual_key_mapping_cache_ttl: 300 # Cache TTL in seconds (default: 300)
```
### Managing Mappings
All endpoints require admin auth (`Authorization: Bearer <master_key>`).
**Create a mapping** — link a JWT claim value to an existing virtual key:
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/new \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"jwt_claim_name": "email",
"jwt_claim_value": "user@example.com",
"key": "sk-virtual-key-from-key-generate"
}'
```
**List mappings** (paginated):
```bash
curl http://localhost:4000/jwt/key/mapping/list?page=1&size=50 \
-H "Authorization: Bearer sk-1234"
```
**Get a specific mapping:**
```bash
curl "http://localhost:4000/jwt/key/mapping/info?id=<mapping-id>" \
-H "Authorization: Bearer sk-1234"
```
**Update a mapping:**
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/update \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"id": "<mapping-id>",
"description": "Updated description",
"is_active": true
}'
```
**Delete a mapping:**
```bash
curl -X POST http://localhost:4000/jwt/key/mapping/delete \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{"id": "<mapping-id>"}'
```
### How It Works
1. A request arrives with a JWT bearer token
2. LiteLLM validates the JWT signature
3. Extracts the configured claim (e.g. `email` → `user@example.com`)
4. Looks up the claim value in the `LiteLLM_JWTKeyMapping` table
5. If a mapping exists, the request proceeds as if the mapped virtual key was used — budgets, rate limits, model access, and spend tracking all apply
6. If no mapping exists, falls back to standard JWT auth (team-level controls)
### Error Codes
| Code | Meaning |
|------|---------|
| 409 | Duplicate mapping — a mapping for that claim name + value already exists |
| 400 | The provided key does not match an existing virtual key |
| 404 | Mapping not found (for update/delete/info) |
| 403 | Non-admin user attempted a mapping operation |
## All JWT Params
[**See Code**](https://github.com/BerriAI/litellm/blob/b204f0c01c703317d812a1553363ab0cb989d5b6/litellm/proxy/_types.py#L95)

242
docs/my-website/docs/response_api.md
View file

@ -14,6 +14,7 @@ Requests to /chat/completions may be bridged here automatically when the provide
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Streaming | ✅ | |
| WebSocket Mode | ✅ | Lower-latency persistent connections for all providers |
| Image Generation Streaming | ✅ | Progressive image generation with partial images (1-3) |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
@ -810,6 +811,245 @@ for event in response:
</TabItem>
</Tabs>
## WebSocket Mode
The Responses API supports **WebSocket mode** for lower-latency, persistent connections ideal for agentic workflows. WebSocket mode works with **all LiteLLM providers**, not just those with native WebSocket support.
### Architecture
LiteLLM provides two WebSocket modes:
1. **Native WebSocket**: Direct `wss://` connection to providers that support it (OpenAI, Azure)
2. **Managed WebSocket**: HTTP streaming over WebSocket for all other providers (Anthropic, Gemini, Bedrock, etc.)
The system automatically selects the appropriate mode based on provider capabilities.
### Usage
<Tabs>
<TabItem value="python" label="Python (websocket-client)">
```python showLineNumbers title="WebSocket with Python"
import json
from websocket import create_connection # pip install websocket-client
# Connect to LiteLLM proxy WebSocket endpoint
ws = create_connection(
"ws://localhost:4000/v1/responses?model=gemini-2.5-flash",
header=["Authorization: Bearer sk-1234"]
)
try:
# Send initial message
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"store": True,
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "My favorite color is blue."}]
}]
}))
# Collect response events
response_id = None
while True:
event = json.loads(ws.recv())
print(f"Event: {event['type']}")
if event["type"] == "response.completed":
response_id = event["response"]["id"]
break
elif event["type"] == "response.output_text.delta":
print(f"Text: {event.get('delta', '')}", end="", flush=True)
print(f"\nResponse ID: {response_id}")
# Send follow-up with previous_response_id for multi-turn
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"previous_response_id": response_id,
"input": [{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "What is my favorite color?"}]
}]
}))
# Collect follow-up response
while True:
event = json.loads(ws.recv())
if event["type"] == "response.completed":
break
elif event["type"] == "response.output_text.delta":
print(event.get("delta", ""), end="", flush=True)
finally:
ws.close()
```
</TabItem>
<TabItem value="javascript" label="JavaScript (ws)">
```javascript showLineNumbers title="WebSocket with JavaScript"
const WebSocket = require('ws'); // npm install ws
const ws = new WebSocket(
'ws://localhost:4000/v1/responses?model=gemini-2.5-flash',
{
headers: {
'Authorization': 'Bearer sk-1234'
}
}
);
ws.on('open', () => {
// Send initial message
ws.send(JSON.stringify({
type: 'response.create',
model: 'gemini-2.5-flash',
store: true,
input: [{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'My favorite color is blue.' }]
}]
}));
});
let responseId = null;
ws.on('message', (data) => {
const event = JSON.parse(data.toString());
console.log(`Event: ${event.type}`);
if (event.type === 'response.completed') {
responseId = event.response.id;
console.log(`Response ID: ${responseId}`);
// Send follow-up
ws.send(JSON.stringify({
type: 'response.create',
model: 'gemini-2.5-flash',
previous_response_id: responseId,
input: [{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'What is my favorite color?' }]
}]
}));
} else if (event.type === 'response.output_text.delta') {
process.stdout.write(event.delta || '');
}
});
ws.on('error', (error) => {
console.error('WebSocket error:', error);
});
```
</TabItem>
<TabItem value="curl" label="curl (websocat)">
```bash showLineNumbers title="WebSocket with websocat"
# Install websocat: brew install websocat (macOS) or cargo install websocat
# Connect to WebSocket endpoint
websocat "ws://localhost:4000/v1/responses?model=gemini-2.5-flash" \
-H="Authorization: Bearer sk-1234"
# Then send JSON events (paste and press Enter):
{"type":"response.create","model":"gemini-2.5-flash","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Hello!"}]}]}
# You'll receive streaming events back:
# {"type":"response.created",...}
# {"type":"response.in_progress",...}
# {"type":"response.output_text.delta","delta":"Hello",...}
# {"type":"response.completed",...}
```
</TabItem>
</Tabs>
### Event Types
WebSocket connections receive Server-Sent Events (SSE) formatted as JSON:
| Event Type | Description |
|------------|-------------|
| `response.created` | Response generation started |
| `response.in_progress` | Response is being generated |
| `response.output_item.added` | New output item (message, tool call, etc.) added |
| `response.output_text.delta` | Incremental text chunk |
| `response.output_text.done` | Text output completed |
| `response.content_part.done` | Content part completed |
| `response.output_item.done` | Output item completed |
| `response.completed` | Full response completed successfully |
| `response.failed` | Response generation failed |
| `response.incomplete` | Response incomplete (e.g., max tokens reached) |
| `error` | Error occurred |
### Multi-Turn Conversations
Use `previous_response_id` to maintain conversation context across multiple WebSocket messages:
```python showLineNumbers title="Multi-turn WebSocket Conversation"
# Turn 1
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"store": True, # Required for multi-turn
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]}]
}))
# ... collect events and get response_id from response.completed event ...
# Turn 2 - reference previous response
ws.send(json.dumps({
"type": "response.create",
"model": "gemini-2.5-flash",
"previous_response_id": response_id, # Links to previous turn
"input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Continue"}]}]
}))
```
### Provider Support
| Provider | WebSocket Mode | Notes |
|----------|----------------|-------|
| OpenAI | Native | Direct `wss://` connection to OpenAI |
| Azure OpenAI | Native | Direct `wss://` connection to Azure |
| Anthropic | Managed | HTTP streaming over WebSocket |
| Google AI Studio (Gemini) | Managed | HTTP streaming over WebSocket |
| Vertex AI | Managed | HTTP streaming over WebSocket |
| AWS Bedrock | Managed | HTTP streaming over WebSocket |
| All other providers | Managed | HTTP streaming over WebSocket |
**Note**: Both native and managed modes provide the same event stream format. The difference is transparent to clients.
### Configuration
No special configuration needed. WebSocket mode is automatically available on the `/v1/responses` endpoint when accessed via WebSocket protocol (`ws://` or `wss://`).
For LiteLLM Proxy, ensure your models are configured normally:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-2.5-flash
litellm_params:
model: gemini/gemini-2.5-flash
api_key: os.environ/GEMINI_API_KEY
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
Both models will automatically support WebSocket mode at `ws://localhost:4000/v1/responses`.
## Response ID Security
By default, LiteLLM Proxy prevents users from accessing other users' response IDs.
@ -930,7 +1170,7 @@ For Responses API with load balancing across deployments with **different API ke
Notes:
- User-key affinity is keyed on `metadata.user_api_key_hash` (the API key hash). The OpenAI `user` request parameter is an end-user identifier and is intentionally not used for deployment affinity.
- Session-ID affinity is keyed on `metadata.session_id`. For proxy requests, this can be passed via the `x-litellm-session-id` HTTP header. For Python SDK requests, you can pass it via `litellm_metadata={"session_id": "value"}` in request args.
- Session-ID affinity is keyed on `metadata.session_id`. For proxy requests, this can be passed via the `x-litellm-session-id` or `x-litellm-trace-id` HTTP header (they are interchangeable for call chaining). For Python SDK requests, you can pass it via `litellm_metadata={"session_id": "value"}` in request args.
- `user_api_key_hash` is already SHA-256, and is used as-is (no double hashing).
- Affinity is scoped by a stable model identifier (the model-map key, e.g. `model_map_information.model_map_key`) so model aliases map to the same stickiness bucket.
- The mapping TTL is controlled by `deployment_affinity_ttl_seconds` (configured on Router init / proxy startup).

7
docs/my-website/docs/search/index.md
View file

@ -2,7 +2,7 @@
| Feature | Supported |
|---------|-----------|
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup` |
| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi` |
| Cost Tracking | ✅ |
| Logging | ✅ |
| Load Balancing | ❌ |
@ -210,7 +210,7 @@ See the [official Perplexity Search documentation](https://docs.perplexity.ai/ap
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `query` | string or array | Yes | Search query. Can be a single string or array of strings |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, or `"linkup"` |
| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, or `"searchapi"` |
| `search_tool_name` | string | Yes (Proxy) | Name of the search tool configured in `config.yaml` |
| `max_results` | integer | No | Maximum number of results to return (1-20). Default: 10 |
| `search_domain_filter` | array | No | List of domains to filter results (max 20 domains) |
@ -276,7 +276,8 @@ The response follows Perplexity's search format with the following structure:
| Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
| SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
| Linkup | `LINKUP_API_KEY` | `linkup` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
| DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
| SearchAPI.io | `SEARCHAPI_API_KEY` | `searchapi` |
See the individual provider documentation for detailed setup instructions and provider-specific parameters.

197
docs/my-website/docs/search/searchapi.md Normal file
View file

@ -0,0 +1,197 @@
# SearchAPI.io (Google Search)
Get started by creating a free API key via https://www.searchapi.io/.
SearchAPI.io provides access to Google Search results with a simple API. It supports all Google Search parameters including location, language, time filters, and more.
For complete documentation on all supported parameters, visit https://www.searchapi.io/docs/google.
## LiteLLM Python SDK
```python showLineNumbers title="SearchAPI.io Search"
import os
from litellm import search
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
response = search(
query="latest AI developments",
search_provider="searchapi",
max_results=10
)
# Access search results
for result in response.results:
print(f"{result.title}: {result.url}")
print(f"Snippet: {result.snippet}\n")
```
### Advanced Usage with SearchAPI.io Parameters
SearchAPI.io supports many Google Search-specific parameters:
```python showLineNumbers title="Advanced SearchAPI.io Parameters"
import os
from litellm import search
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
response = search(
query="machine learning research",
search_provider="searchapi",
max_results=10,
# Unified parameters
country="US",
search_domain_filter=["arxiv.org", "nature.com"],
# SearchAPI.io specific parameters
gl="us", # Country code
hl="en", # Interface language
time_period="last_month", # Time filter
safe="active", # SafeSearch
device="desktop", # Device type
location="New York" # Geographic location
)
```
## LiteLLM AI Gateway
### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
search_tools:
- search_tool_name: google-search
litellm_params:
search_provider: searchapi
api_key: os.environ/SEARCHAPI_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test the search endpoint
```bash showLineNumbers title="Test Request"
curl http://0.0.0.0:4000/v1/search/google-search \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "latest AI developments",
"max_results": 10,
"country": "US"
}'
```
## SearchAPI.io Specific Parameters
SearchAPI.io supports many Google Search parameters. Here are some commonly used ones:
| Parameter | Type | Description |
|-----------|------|-------------|
| `gl` | string | Country code (e.g., 'us', 'uk', 'de') |
| `hl` | string | Interface language (e.g., 'en', 'es', 'fr') |
| `location` | string | Geographic location (e.g., 'New York', 'London') |
| `device` | string | Device type: 'desktop', 'mobile', 'tablet' |
| `time_period` | string | Time filter: 'last_hour', 'last_day', 'last_week', 'last_month', 'last_year' |
| `time_period_min` | string | Start date (MM/DD/YYYY) |
| `time_period_max` | string | End date (MM/DD/YYYY) |
| `safe` | string | SafeSearch: 'active' or 'off' |
| `lr` | string | Language restriction (e.g., 'lang_en', 'lang_es') |
| `cr` | string | Country restriction |
| `page` | integer | Page number for pagination |
### Example with Time Filters
```python showLineNumbers title="Search with Time Filter"
response = search(
query="AI breakthroughs",
search_provider="searchapi",
max_results=10,
time_period="last_month"
)
```
### Example with Custom Date Range
```python showLineNumbers title="Search with Custom Date Range"
response = search(
query="AI research papers",
search_provider="searchapi",
max_results=10,
time_period_min="01/01/2024",
time_period_max="03/01/2024"
)
```
### Example with Location
```python showLineNumbers title="Search with Location"
response = search(
query="AI conferences",
search_provider="searchapi",
max_results=10,
location="San Francisco",
gl="us"
)
```
## Response Format
SearchAPI.io returns results in the standard LiteLLM search format:
```json
{
"object": "search",
"results": [
{
"title": "Latest AI Developments",
"url": "https://example.com/ai-news",
"snippet": "Recent breakthroughs in artificial intelligence...",
"date": "2024-01-15"
}
]
}
```
## Rate Limits
SearchAPI.io has different rate limits based on your plan:
- Free tier: 100 requests/month
- Paid plans: Higher limits available
Check your current usage at https://www.searchapi.io/dashboard.
## Error Handling
```python showLineNumbers title="Error Handling"
from litellm import search
import os
os.environ["SEARCHAPI_API_KEY"] = "your-api-key"
try:
response = search(
query="test query",
search_provider="searchapi",
max_results=10
)
print(f"Found {len(response.results)} results")
except Exception as e:
print(f"Search failed: {str(e)}")
```
## Additional Resources
- SearchAPI.io Documentation: https://www.searchapi.io/docs
- API Dashboard: https://www.searchapi.io/dashboard
- Pricing: https://www.searchapi.io/pricing

BIN
docs/my-website/img/mcp_openapi_custom_name_badge.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 144 KiB

BIN
docs/my-website/img/mcp_openapi_tool_edit_panel.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 151 KiB

BIN
docs/my-website/img/mcp_openapi_tools_loaded.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 103 KiB

5305
docs/my-website/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

6
docs/my-website/package.json
View file

@ -15,10 +15,10 @@
},
"dependencies": {
"@docusaurus/core": "3.8.1",
"@docusaurus/plugin-google-gtag": "^3.5.2",
"@docusaurus/plugin-google-gtag": "3.8.1",
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "^3.5.2",
"@docusaurus/theme-mermaid": "^3.5.2",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",

4
docs/my-website/release_notes/v1.81.14.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.81.14 - New Gateway Level Guardrails & Compliance Playground"
title: "v1.81.14 - New Gateway Level Guardrails & Compliance Playground"
slug: "v1-81-14"
date: 2026-02-21T00:00:00
authors:
@ -27,7 +27,7 @@ import Image from '@theme/IdealImage';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.81.14.rc.1
ghcr.io/berriai/litellm:main-v1.81.14-stable
```
</TabItem>

7
docs/my-website/release_notes/v1.82.0.md
View file

@ -1,5 +1,5 @@
---
title: "v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
title: "[Preview] v1.82.0 - Realtime Guardrails, Projects Management, and 10+ Performance Optimizations"
slug: "v1-82-0"
date: 2026-02-28T00:00:00
authors:
@ -46,6 +46,11 @@ pip install litellm==1.82.0
- **Guardrail ecosystem expansion** — [Noma v2, Lakera v2 post-call, Singapore regulatory policies (PDPA + MAS), employment discrimination blockers, code execution blocker, guardrail policy versioning, and production monitoring](../../docs/proxy/guardrails) - [PR #21400](https://github.com/BerriAI/litellm/pull/21400), [PR #21783](https://github.com/BerriAI/litellm/pull/21783), [PR #21948](https://github.com/BerriAI/litellm/pull/21948)
- **OpenAI Codex 5.3 — day 0** — [Full support for `gpt-5.3-codex` on OpenAI and Azure, plus `gpt-audio-1.5` and `gpt-realtime-1.5` model coverage](../../docs/providers/openai) - [PR #22035](https://github.com/BerriAI/litellm/pull/22035)
- **10+ performance optimizations** — Streaming hot-path fixes, Redis pipeline batching, database task batching, ModelResponse init skip, and router cache improvements — lower latency and CPU on every request
- **`/v1/messages` → `/responses` routing** — `/v1/messages` requests are now routed to the [Responses API](../../docs/response_api) by default for OpenAI/Azure models
:::danger v1/messages routing change
This version starts routing `/v1/messages` requests to the `/responses` API by default. To opt out and continue using chat/completions, set `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true` or `litellm_settings.use_chat_completions_url_for_anthropic_messages: true` in your config.
:::
---

1
docs/my-website/sidebars.js
View file

@ -608,6 +608,7 @@ const sidebars = {
items: [
"mcp",
"mcp_usage",
"mcp_openapi",
"mcp_oauth",
"mcp_public_internet",
"mcp_semantic_filter",

239
docs/my-website/src/pages/index.md
View file

@ -7,42 +7,41 @@ https://github.com/BerriAI/litellm
## **Call 100+ LLMs using the OpenAI Input/Output Format**
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more)
- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
## How to use LiteLLM
You can use litellm through either:
1. [LiteLLM Proxy Server](#litellm-proxy-server-llm-gateway) - Server (LLM Gateway) to call 100+ LLMs, load balance, cost tracking across projects
2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking
### **When to use LiteLLM Proxy Server (LLM Gateway)**
You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
:::tip
Use LiteLLM Proxy Server if you want a **central service (LLM Gateway) to access multiple LLMs**
Typically used by Gen AI Enablement / ML PLatform Teams
:::
- LiteLLM Proxy gives you a unified interface to access multiple LLMs (100+ LLMs)
- Track LLM Usage and setup guardrails
- Customize Logging, Guardrails, Caching per project
### **When to use LiteLLM Python SDK**
:::tip
Use LiteLLM Python SDK if you want to use LiteLLM in your **python code**
Typically used by developers building llm projects
:::
- LiteLLM SDK gives you a unified interface to access multiple LLMs (100+ LLMs)
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
<table style={{width: '100%', tableLayout: 'fixed'}}>
<thead>
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="#litellm-proxy-server-llm-gateway">LiteLLM Proxy Server</a></strong></th>
<th style={{width: '43%'}}><strong><a href="#basic-usage">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>
<tr>
<td style={{width: '14%'}}><strong>Use Case</strong></td>
<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
<td style={{width: '43%'}}>Developers building LLM projects</td>
</tr>
<tr>
<td style={{width: '14%'}}><strong>Key Features</strong></td>
<td style={{width: '43%'}}>• Centralized API gateway with authentication & authorization<br />• Multi-tenant cost tracking and spend management per project/user<br />• Per-project customization (logging, guardrails, caching)<br />• Virtual keys for secure access control<br />• Admin dashboard UI for monitoring and management</td>
<td style={{width: '43%'}}>• Direct Python library integration in your codebase<br />• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a><br />• Application-level load balancing and cost tracking<br />• Exception handling with OpenAI-compatible errors<br />• Observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
</tr>
</tbody>
</table>
## **LiteLLM Python SDK**
@ -67,7 +66,7 @@ import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
@ -83,13 +82,27 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
response = completion(
model="claude-2",
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["XAI_API_KEY"] = "your-api-key"
response = completion(
model="xai/grok-2-latest",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="vertex" label="VertexAI">
```python
@ -97,11 +110,11 @@ from litellm import completion
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "hardy-device-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
@ -212,8 +225,61 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for instructions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>
### Response Format (OpenAI Chat Completions Format)
```json
{
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
"created": 1734366691,
"model": "gpt-5",
"object": "chat.completion",
"system_fingerprint": null,
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"usage": {
"completion_tokens": 43,
"prompt_tokens": 13,
"total_tokens": 56,
"completion_tokens_details": null,
"prompt_tokens_details": {
"audio_tokens": null,
"cached_tokens": 0
},
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0
}
}
```
### Responses API
Use `litellm.responses()` for advanced models that support reasoning content like GPT-5, o3, etc.
@ -265,11 +331,11 @@ from litellm import responses
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "jr-smith-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "jr-smith-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = responses(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "What is the capital of France?","role": "user"}]
)
```
@ -314,7 +380,7 @@ import os
os.environ["OPENAI_API_KEY"] = "your-api-key"
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
@ -331,14 +397,29 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
response = completion(
model="claude-2",
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["XAI_API_KEY"] = "your-api-key"
response = completion(
model="xai/grok-2-latest",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
<TabItem value="vertex" label="VertexAI">
```python
@ -346,11 +427,11 @@ from litellm import completion
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEX_PROJECT"] = "hardy-device-386718"
os.environ["VERTEX_LOCATION"] = "us-central1"
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="chat-bison",
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
@ -370,7 +451,7 @@ os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
response = completion(
model="nvidia_nim/<model_name>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
@ -466,22 +547,74 @@ response = completion(
```
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for instructions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-5",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
</Tabs>
### Streaming Response Format (OpenAI Format)
```json
{
"id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
"created": 1734366925,
"model": "claude-sonnet-4-5-20250929",
"object": "chat.completion.chunk",
"system_fingerprint": null,
"choices": [
{
"finish_reason": null,
"index": 0,
"delta": {
"content": "Hello",
"role": "assistant",
"function_call": null,
"tool_calls": null,
"audio": null
},
"logprobs": null
}
]
}
```
### Exception handling
LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM.
```python
from openai.error import OpenAIError
import litellm
from litellm import completion
import os
os.environ["ANTHROPIC_API_KEY"] = "bad-key"
try:
# some code
completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
except OpenAIError as e:
print(e)
completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
except litellm.AuthenticationError as e:
# Thrown when the API key is invalid
print(f"Authentication failed: {e}")
except litellm.RateLimitError as e:
# Thrown when you've exceeded your rate limit
print(f"Rate limited: {e}")
except litellm.APIError as e:
# Thrown for general API errors
print(f"API error: {e}")
```
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
@ -502,7 +635,7 @@ os.environ["OPENAI_API_KEY"]
litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
response = completion(model="openai/gpt-5", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
```
### Track Costs, Usage, Latency for streaming
@ -527,7 +660,7 @@ litellm.success_callback = [track_cost_callback] # set custom callback function
# litellm.completion() call
response = completion(
model="gpt-3.5-turbo",
model="openai/gpt-5",
messages=[
{
"role": "user",
@ -584,7 +717,7 @@ Example `litellm_config.yaml`
```yaml
model_list:
- model_name: gpt-3.5-turbo
- model_name: gpt-5
litellm_params:
model: azure/<your-azure-model-deployment>
api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE")
@ -621,7 +754,7 @@ docker run \
import openai # openai v1.0.0+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
response = client.chat.completions.create(model="gpt-5", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"

53
enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py
View file

@ -1,13 +1,13 @@
"""
AUDIT LOGGING
All /audit logging endpoints. Attempting to write these as CRUD endpoints.
All /audit logging endpoints. Attempting to write these as CRUD endpoints.
GET - /audit/{id} - Get audit log by id
GET - /audit - Get all audit logs
"""
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
#### AUDIT LOGGING ####
from fastapi import APIRouter, Depends, HTTPException, Query
@ -22,6 +22,27 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
router = APIRouter()
def _build_json_field_or_condition(json_key: str, value: str) -> Dict[str, Any]:
"""
Build an OR condition that matches a value inside a JSON column at the
given key, checking both before_value and updated_values.
Uses Prisma's JSON path filtering (PostgreSQL only).
Example result (team_id="t1"):
{"OR": [
{"before_value": {"path": ["team_id"], "string_contains": "t1"}},
{"updated_values": {"path": ["team_id"], "string_contains": "t1"}},
]}
"""
return {
"OR": [
{"before_value": {"path": [json_key], "string_contains": value}},
{"updated_values": {"path": [json_key], "string_contains": value}},
]
}
@router.get(
"/audit",
tags=["Audit Logging"],
@ -49,6 +70,14 @@ async def get_audit_logs(
),
start_date: Optional[str] = Query(None, description="Filter logs after this date"),
end_date: Optional[str] = Query(None, description="Filter logs before this date"),
object_team_id: Optional[str] = Query(
None,
description="Filter by team_id present in before_value or updated_values JSON (PostgreSQL only)",
),
object_key_hash: Optional[str] = Query(
None,
description="Filter by token (key hash) present in before_value or updated_values JSON (PostgreSQL only)",
),
# Sorting parameters
sort_by: Optional[str] = Query(
None,
@ -60,6 +89,9 @@ async def get_audit_logs(
Get all audit logs with filtering and pagination.
Returns a paginated response of audit logs matching the specified filters.
Note: object_team_id and object_key_hash use Prisma JSON path filtering,
which requires PostgreSQL.
"""
from litellm.proxy.proxy_server import prisma_client
@ -82,18 +114,29 @@ async def get_audit_logs(
if object_id:
where_conditions["object_id"] = object_id
if start_date or end_date:
date_filter = {}
date_filter: Dict[str, Any] = {}
if start_date:
date_filter["gte"] = start_date
if end_date:
date_filter["lte"] = end_date
where_conditions["updated_at"] = date_filter
# JSON field filters (PostgreSQL only) — each filter is AND'd with the
# others, but checks both before_value and updated_values internally (OR).
if object_team_id:
where_conditions["AND"] = where_conditions.get("AND", []) + [
_build_json_field_or_condition("team_id", object_team_id)
]
if object_key_hash:
where_conditions["AND"] = where_conditions.get("AND", []) + [
_build_json_field_or_condition("token", object_key_hash)
]
# Build sort conditions
order_by = {}
order_by: Dict[str, Any] = {}
if sort_by and isinstance(sort_by, str):
order_by[sort_by] = sort_order
elif sort_order and isinstance(sort_order, str):
else:
order_by["updated_at"] = sort_order # Default sort by updated_at
# Get paginated results

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260226000000_add_blocked_tools_to_object_permission/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN "blocked_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

11
litellm-proxy-extras/litellm_proxy_extras/migrations/20260226120000_add_spend_log_tool_index/migration.sql Normal file
View file

@ -0,0 +1,11 @@
-- CreateTable
CREATE TABLE "LiteLLM_SpendLogToolIndex" (
"request_id" TEXT NOT NULL,
"tool_name" TEXT NOT NULL,
"start_time" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LiteLLM_SpendLogToolIndex_pkey" PRIMARY KEY ("request_id","tool_name")
);
-- CreateIndex
CREATE INDEX "LiteLLM_SpendLogToolIndex_tool_name_start_time_idx" ON "LiteLLM_SpendLogToolIndex"("tool_name", "start_time");

20
litellm-proxy-extras/litellm_proxy_extras/migrations/20260303000000_update_tool_table_policies/migration.sql Normal file
View file

@ -0,0 +1,20 @@
-- Rename call_policy to input_policy
ALTER TABLE "LiteLLM_ToolTable" RENAME COLUMN "call_policy" TO "input_policy";
-- Add output_policy column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "output_policy" TEXT NOT NULL DEFAULT 'untrusted';
-- Add user_agent column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "user_agent" TEXT;
-- Add last_used_at column
ALTER TABLE "LiteLLM_ToolTable" ADD COLUMN "last_used_at" TIMESTAMP(3);
-- Drop old index on call_policy
DROP INDEX IF EXISTS "LiteLLM_ToolTable_call_policy_idx";
-- CreateIndex
CREATE INDEX "LiteLLM_ToolTable_input_policy_idx" ON "LiteLLM_ToolTable"("input_policy");
-- CreateIndex
CREATE INDEX "LiteLLM_ToolTable_output_policy_idx" ON "LiteLLM_ToolTable"("output_policy");

47
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -260,6 +260,7 @@ model LiteLLM_ObjectPermissionTable {
vector_stores String[] @default([])
agents String[] @default([])
agent_access_groups String[] @default([])
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
teams LiteLLM_TeamTable[]
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
@ -276,6 +277,7 @@ model LiteLLM_MCPServerTable {
alias String?
description String?
url String?
spec_path String?
transport String @default("sse")
auth_type String?
credentials Json? @default("{}")
@ -928,6 +930,16 @@ model LiteLLM_SpendLogGuardrailIndex {
@@index([policy_id, start_time])
}
// Index for fast "last N logs for tool" from SpendLogs – see how a tool is called in production
model LiteLLM_SpendLogToolIndex {
request_id String
tool_name String // matches LiteLLM_ToolTable.tool_name; join for input_policy/output_policy etc.
start_time DateTime
@@id([request_id, tool_name])
@@index([tool_name, start_time])
}
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
@ -1065,26 +1077,31 @@ model LiteLLM_PolicyAttachmentTable {
updated_by String?
}
// Global tool registry - auto-discovered from LLM responses; admins set call_policy here
// Global tool registry - auto-discovered from LLM responses; admins set input_policy/output_policy here
model LiteLLM_ToolTable {
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
call_policy String @default("untrusted") // "trusted" | "untrusted" | "dual_llm" | "blocked"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
tool_id String @id @default(uuid())
tool_name String @unique // e.g. "huggingface_remote-mcp__dynamic_space"
origin String? // MCP server name or "user_defined"
input_policy String @default("untrusted") // "trusted" | "untrusted" | "blocked"
output_policy String @default("untrusted") // "trusted" | "untrusted"
call_count Int @default(0) // cumulative number of times this tool was seen
assignments Json? @default("{}")
key_hash String? // hash of the virtual key that first called this tool
team_id String? // team that first called this tool
key_alias String? // human-readable alias of the virtual key
user_agent String? // user-agent of the first request that discovered this tool
last_used_at DateTime? // timestamp of the most recent call
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
@@index([call_policy])
@@index([input_policy])
@@index([output_policy])
@@index([team_id])
}
// Per-(tool, team/key) policy overrides. When present, override replaces global tool policy for that scope.
//Unified Access Groups table for storing unified access groups
model LiteLLM_AccessGroupTable {
access_group_id String @id @default(uuid())

1
litellm/__init__.py
View file

@ -1246,6 +1246,7 @@ from .ocr.main import *
from .rag.main import *
from .search.main import *
from .realtime_api.main import _arealtime
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (

96
litellm/a2a_protocol/main.py
View file

@ -24,11 +24,7 @@ from litellm.utils import client
if TYPE_CHECKING:
from a2a.client import A2AClient as A2AClientType
from a2a.types import (
AgentCard,
SendMessageRequest,
SendStreamingMessageRequest,
)
from a2a.types import AgentCard, SendMessageRequest, SendStreamingMessageRequest
# Runtime imports with availability check
A2A_SDK_AVAILABLE = False
@ -124,13 +120,48 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
litellm_logging_obj.model = model
litellm_logging_obj.custom_llm_provider = custom_llm_provider
litellm_logging_obj.model_call_details["model"] = model
litellm_logging_obj.model_call_details[
"custom_llm_provider"
] = custom_llm_provider
litellm_logging_obj.model_call_details["custom_llm_provider"] = (
custom_llm_provider
)
return agent_name
async def _send_message_via_completion_bridge(
request: "SendMessageRequest",
custom_llm_provider: str,
api_base: Optional[str],
litellm_params: Dict[str, Any],
) -> LiteLLMSendMessageResponse:
"""
Route a send_message through the LiteLLM completion bridge (e.g. LangGraph, Bedrock AgentCore).
Requires request; api_base is optional for providers that derive endpoint from model.
"""
verbose_logger.info(
f"A2A using completion bridge: provider={custom_llm_provider}, api_base={api_base}"
)
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
A2ACompletionBridgeHandler,
)
params = (
request.params.model_dump(mode="json")
if hasattr(request.params, "model_dump")
else dict(request.params)
)
response_dict = await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=str(request.id),
params=params,
litellm_params=litellm_params,
api_base=api_base,
)
return LiteLLMSendMessageResponse.from_dict(response_dict)
@client
async def asend_message(
a2a_client: Optional["A2AClientType"] = None,
@ -193,39 +224,21 @@ async def asend_message(
```
"""
litellm_params = litellm_params or {}
logging_obj = kwargs.get("litellm_logging_obj")
trace_id = getattr(logging_obj, "litellm_trace_id", None) if logging_obj else None
custom_llm_provider = litellm_params.get("custom_llm_provider")
# Route through completion bridge if custom_llm_provider is set
if custom_llm_provider:
if request is None:
raise ValueError("request is required for completion bridge")
# api_base is optional for providers that derive endpoint from model (e.g., bedrock/agentcore)
verbose_logger.info(
f"A2A using completion bridge: provider={custom_llm_provider}, api_base={api_base}"
)
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
A2ACompletionBridgeHandler,
)
# Extract params from request
params = (
request.params.model_dump(mode="json")
if hasattr(request.params, "model_dump")
else dict(request.params)
)
response_dict = await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=str(request.id),
params=params,
litellm_params=litellm_params,
return await _send_message_via_completion_bridge(
request=request,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=litellm_params,
)
# Convert to LiteLLMSendMessageResponse
return LiteLLMSendMessageResponse.from_dict(response_dict)
# Standard A2A client flow
if request is None:
raise ValueError("request is required")
@ -236,11 +249,13 @@ async def asend_message(
raise ValueError(
"Either a2a_client or api_base is required for standard A2A flow"
)
trace_id = str(uuid.uuid4())
trace_id = trace_id or str(uuid.uuid4())
extra_headers = {"X-LiteLLM-Trace-Id": trace_id}
if agent_id:
extra_headers["X-LiteLLM-Agent-Id"] = agent_id
a2a_client = await create_a2a_client(base_url=api_base, extra_headers=extra_headers)
a2a_client = await create_a2a_client(
base_url=api_base, extra_headers=extra_headers
)
# Type assertion: a2a_client is guaranteed to be non-None here
assert a2a_client is not None
@ -255,6 +270,15 @@ async def asend_message(
)
card_url = getattr(agent_card, "url", None) if agent_card else None
context_id = trace_id or str(uuid.uuid4())
message = request.params.message
if isinstance(message, dict):
if message.get("context_id") is None:
message["context_id"] = context_id
else:
if getattr(message, "context_id", None) is None:
message.context_id = context_id
# Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL
a2a_response = None
for _ in range(2): # max 2 attempts: original + 1 retry
@ -606,7 +630,9 @@ async def create_a2a_client(
if extra_headers:
httpx_client.headers.update(extra_headers)
verbose_proxy_logger.debug(f"A2A client created with extra_headers={extra_headers}")
verbose_proxy_logger.debug(
f"A2A client created with extra_headers={extra_headers}"
)
# Resolve agent card
resolver = A2ACardResolver(

6
litellm/batches/batch_utils.py
View file

@ -1,14 +1,10 @@
import json
import time
from typing import Any, List, Literal, Optional, Tuple
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.types.llms.openai import Batch
from litellm.types.utils import CallTypes, ModelInfo, ModelResponse, Usage
from litellm.types.utils import CallTypes, ModelInfo, Usage
from litellm.utils import token_counter

7
litellm/batches/main.py
View file

@ -112,6 +112,7 @@ async def acreate_batch(
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
output_expires_after: Optional[Dict[str, Any]] = None,
**kwargs,
) -> LiteLLMBatch:
"""
@ -133,6 +134,7 @@ async def acreate_batch(
metadata,
extra_headers,
extra_body,
output_expires_after,
**kwargs,
)
@ -152,7 +154,7 @@ async def acreate_batch(
@client
def create_batch(
def create_batch( # noqa: PLR0915
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
input_file_id: str,
@ -160,6 +162,7 @@ def create_batch(
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
output_expires_after: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
"""
@ -215,6 +218,8 @@ def create_batch(
extra_headers=extra_headers,
extra_body=extra_body,
)
if output_expires_after is not None:
_create_batch_request["output_expires_after"] = output_expires_after
if model is not None:
provider_config = ProviderConfigManager.get_provider_batches_config(
model=model,

345
litellm/files/main.py
View file

@ -7,7 +7,6 @@ https://platform.openai.com/docs/api-reference/files
import asyncio
import contextvars
import os
import time
import uuid as uuid_module
from functools import partial
@ -20,10 +19,12 @@ from litellm import get_secret_str
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.files.handler import AnthropicFilesHandler
from litellm.llms.azure.common_utils import get_azure_credentials
from litellm.llms.azure.files.handler import AzureOpenAIFilesAPI
from litellm.llms.bedrock.files.handler import BedrockFilesHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
from litellm.types.llms.openai import (
@ -185,95 +186,36 @@ def create_file(
timeout=timeout,
)
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
create_file_data=_create_file_request,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base = optional_params.api_base or ""
vertex_ai_project = (
optional_params.vertex_project
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.vertex_location
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
response = vertex_ai_files_instance.create_file(
_is_async=_is_async,
api_base=api_base,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
timeout=timeout,
max_retries=optional_params.max_retries,
create_file_data=_create_file_request,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_file'. Only ['openai', 'azure', 'vertex_ai', 'manus'] are supported.".format(
@ -336,7 +278,7 @@ async def afile_retrieve(
@client
def file_retrieve(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "manus"] = "openai",
custom_llm_provider: Literal["openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "manus"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -367,64 +309,31 @@ def file_retrieve(
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.retrieve_file(
file_id=file_id,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.retrieve_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_id=file_id,
@ -576,63 +485,31 @@ def file_delete(
timeout = 600.0
_is_async = kwargs.pop("is_async", False) is True
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
response = openai_files_instance.delete_file(
file_id=file_id,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.delete_file(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_id=file_id,
@ -815,64 +692,31 @@ def file_list(
)
return response
elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.list_files(
purpose=purpose,
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.list_files(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
purpose=purpose,
@ -1003,64 +847,31 @@ def file_content(
return response
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
optional_params.api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
openai_creds = get_openai_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
organization=optional_params.organization,
)
organization = (
optional_params.organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
)
# set API KEY
api_key = (
optional_params.api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
response = openai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
api_key=api_key,
api_base=openai_creds.api_base,
api_key=openai_creds.api_key,
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
organization=openai_creds.organization,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # type: ignore
api_version = (
optional_params.api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
) # type: ignore
api_key = (
optional_params.api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
) # type: ignore
extra_body = optional_params.get("extra_body", {})
if extra_body is not None:
extra_body.pop("azure_ad_token", None)
else:
get_secret_str("AZURE_AD_TOKEN") # type: ignore
azure_creds = get_azure_credentials(
api_base=optional_params.api_base,
api_key=optional_params.api_key,
api_version=optional_params.api_version,
)
response = azure_files_instance.file_content(
_is_async=_is_async,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_base=azure_creds.api_base,
api_key=azure_creds.api_key,
api_version=azure_creds.api_version,
timeout=timeout,
max_retries=optional_params.max_retries,
file_content_request=_file_content_request,

6
litellm/integrations/helicone.py
View file

@ -167,12 +167,12 @@ class HeliconeLogger:
if "claude" in model and not is_vertex_ai:
url = f"{self.api_base}/anthropic/v1/log"
provider_url = "https://api.anthropic.com/v1/messages"
elif "gemini" in model:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://generativelanguage.googleapis.com/v1beta"
elif is_vertex_ai:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://aiplatform.googleapis.com/v1"
elif "gemini" in model:
url = f"{self.api_base}/custom/v1/log"
provider_url = "https://generativelanguage.googleapis.com/v1beta"
headers = {
"Authorization": f"Bearer {self.key}",
"Content-Type": "application/json",

8
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -1,6 +1,5 @@
from typing import Optional
# Pre-define optional kwargs keys as frozenset for O(1) lookups
# These are extracted from kwargs only if present, avoiding unnecessary .get() calls
_OPTIONAL_KWARGS_KEYS = frozenset({
@ -95,6 +94,13 @@ def get_litellm_params(
litellm_request_debug: Optional[bool] = None,
**kwargs,
) -> dict:
# Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining)
_meta = metadata or {}
if litellm_session_id is None:
litellm_session_id = _meta.get("session_id") or _meta.get("trace_id")
if litellm_trace_id is None:
litellm_trace_id = _meta.get("trace_id") or _meta.get("session_id")
# Build base dict with explicit parameters (always included)
litellm_params = {
"acompletion": acompletion,

306
litellm/litellm_core_utils/litellm_logging.py
View file

@ -133,8 +133,8 @@ from ..integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from ..integrations.custom_prompt_management import CustomPromptManagement
from ..integrations.datadog.datadog import DataDogLogger
from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger
from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
from ..integrations.datadog.datadog_metrics import DatadogMetricsLogger
from ..integrations.dotprompt import DotpromptManager
from ..integrations.dynamodb import DyanmoDBLogger
from ..integrations.galileo import GalileoObserve
@ -352,9 +352,9 @@ class Logging(LiteLLMLoggingBaseClass):
)
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[
Any
] = [] # for generating complete stream response
self.sync_streaming_chunks: List[Any] = (
[]
) # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
# Initialize dynamic callbacks
@ -746,9 +746,9 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_spec=prompt_spec,
dynamic_callback_params=dynamic_callback_params,
):
self.model_call_details[
"prompt_integration"
] = logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
logger.__class__.__name__
)
return logger
except Exception:
# If check fails, continue to next logger
@ -816,9 +816,9 @@ class Logging(LiteLLMLoggingBaseClass):
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
non_default_params
):
self.model_call_details[
"prompt_integration"
] = anthropic_cache_control_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
anthropic_cache_control_logger.__class__.__name__
)
return anthropic_cache_control_logger
#########################################################
@ -830,9 +830,9 @@ class Logging(LiteLLMLoggingBaseClass):
internal_usage_cache=None,
llm_router=None,
)
self.model_call_details[
"prompt_integration"
] = vector_store_custom_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
vector_store_custom_logger.__class__.__name__
)
# Add to global callbacks so post-call hooks are invoked
if (
vector_store_custom_logger
@ -892,9 +892,9 @@ class Logging(LiteLLMLoggingBaseClass):
model
): # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"][
"api_base"
] = self._get_masked_api_base(additional_args.get("api_base", ""))
self.model_call_details["litellm_params"]["api_base"] = (
self._get_masked_api_base(additional_args.get("api_base", ""))
)
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
# Log the exact input to the LLM API
@ -923,10 +923,10 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata[
"raw_request"
] = "redacted by litellm. \
_metadata["raw_request"] = (
"redacted by litellm. \
'litellm.turn_off_message_logging=True'"
)
else:
curl_command = self._get_request_curl_command(
api_base=additional_args.get("api_base", ""),
@ -937,34 +937,34 @@ class Logging(LiteLLMLoggingBaseClass):
_metadata["raw_request"] = str(curl_command)
# split up, so it's easier to parse in the UI
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
)
)
except Exception as e:
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
error=str(e),
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
error=str(e),
)
)
_metadata[
"raw_request"
] = "Unable to Log \
_metadata["raw_request"] = (
"Unable to Log \
raw request: {}".format(
str(e)
str(e)
)
)
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
@ -1265,13 +1265,13 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
response: Optional[
MCPPostCallResponseObject
] = await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
response: Optional[MCPPostCallResponseObject] = (
await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
)
######################################################################
# if any of the callbacks modify the response, use the modified response
@ -1466,9 +1466,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
try:
@ -1494,9 +1494,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
@ -1652,10 +1652,8 @@ class Logging(LiteLLMLoggingBaseClass):
result=logging_result
)
self.model_call_details[
"standard_logging_object"
] = self._build_standard_logging_payload(
logging_result, start_time, end_time
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(logging_result, start_time, end_time)
)
if (
@ -1734,9 +1732,9 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details[
"completion_start_time"
] = self.completion_start_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
@ -1773,10 +1771,10 @@ class Logging(LiteLLMLoggingBaseClass):
end_time=end_time,
)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details[
"standard_logging_object"
] = self._build_standard_logging_payload(
result, start_time, end_time
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(
result, start_time, end_time
)
)
if (
standard_logging_payload := self.model_call_details.get(
@ -1785,9 +1783,9 @@ class Logging(LiteLLMLoggingBaseClass):
) is not None:
emit_standard_logging_payload(standard_logging_payload)
elif standard_logging_object is not None:
self.model_call_details[
"standard_logging_object"
] = standard_logging_object
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
else:
self.model_call_details["response_cost"] = None
@ -1945,17 +1943,17 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
"Logging Details LiteLLM-Success Call streaming complete"
)
self.model_call_details[
"complete_streaming_response"
] = complete_streaming_response
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=complete_streaming_response)
self.model_call_details["complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=complete_streaming_response)
)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
)
if (
standard_logging_payload := self.model_call_details.get(
@ -2289,10 +2287,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
openMeterLogger.log_success_event(
@ -2316,10 +2314,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
@ -2458,9 +2456,9 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
print_verbose("Async success callbacks: Got a complete streaming response")
self.model_call_details[
"async_complete_streaming_response"
] = complete_streaming_response
self.model_call_details["async_complete_streaming_response"] = (
complete_streaming_response
)
try:
if self.model_call_details.get("cache_hit", False) is True:
@ -2471,10 +2469,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=self.model_call_details
)
# base_model defaults to None if not set on model_info
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(
result=complete_streaming_response
self.model_call_details["response_cost"] = (
self._response_cost_calculator(
result=complete_streaming_response
)
)
verbose_logger.debug(
@ -2487,10 +2485,10 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = None
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(
complete_streaming_response, start_time, end_time
)
)
# print standard logging payload
@ -2517,10 +2515,8 @@ class Logging(LiteLLMLoggingBaseClass):
# _success_handler_helper_fn
if self.model_call_details.get("standard_logging_object") is None:
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = self._build_standard_logging_payload(
result, start_time, end_time
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(result, start_time, end_time)
)
# print standard logging payload
@ -2764,18 +2760,18 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
return start_time, end_time
@ -3739,9 +3735,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
service_name=arize_config.project_name,
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, ArizeLogger)
@ -3767,13 +3763,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
)
else:
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={arize_phoenix_config.project_name}"
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={arize_phoenix_config.project_name}"
)
# Set Phoenix project name from environment variable
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
@ -3781,19 +3777,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
)
else:
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={phoenix_project_name}"
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={phoenix_project_name}"
)
# auth can be disabled on local deployments of arize phoenix
if arize_phoenix_config.otlp_auth_headers is not None:
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = arize_phoenix_config.otlp_auth_headers
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
arize_phoenix_config.otlp_auth_headers
)
for callback in _in_memory_loggers:
if (
@ -3969,9 +3965,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
exporter="otlp_http",
endpoint="https://langtrace.ai/api/trace",
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
@ -4204,8 +4200,7 @@ def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list) -> None:
litellm.logging_callback_manager.add_litellm_callback(phoenix_logger)
verbose_logger.info(
"Auto-initialized Arize Phoenix logger alongside otel "
"(endpoint=%s)",
"Auto-initialized Arize Phoenix logger alongside otel " "(endpoint=%s)",
arize_phoenix_config.endpoint,
)
except Exception as e:
@ -4768,9 +4763,11 @@ class StandardLoggingPayloadSetup:
).model_dump()
if isinstance(_raw, dict):
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_raw
).model_dump()
return (
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
_raw
).model_dump()
)
return _raw
if isinstance(_raw, Usage):
return _raw.model_dump()
@ -4884,10 +4881,10 @@ class StandardLoggingPayloadSetup:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params[
"additional_headers"
] = StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
@ -5039,14 +5036,22 @@ class StandardLoggingPayloadSetup:
dynamic_litellm_session_id = litellm_params.get("litellm_session_id")
dynamic_litellm_trace_id = litellm_params.get("litellm_trace_id")
# Note: we recommend using `litellm_session_id` for session tracking
# `litellm_trace_id` is an internal litellm param
if dynamic_litellm_session_id:
return str(dynamic_litellm_session_id)
elif dynamic_litellm_trace_id:
return str(dynamic_litellm_trace_id)
else:
return logging_obj.litellm_trace_id
# Fallback: use metadata.session_id or metadata.trace_id for call chaining
metadata = litellm_params.get("metadata") or {}
metadata_session_id = metadata.get("session_id")
metadata_trace_id = metadata.get("trace_id")
if metadata_session_id:
return str(metadata_session_id)
if metadata_trace_id:
return str(metadata_trace_id)
return logging_obj.litellm_trace_id
@staticmethod
def _get_user_agent_tags(proxy_server_request: dict) -> Optional[List[str]]:
@ -5502,9 +5507,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
):
for k, v in metadata["user_api_key_metadata"].items():
if k == "logging": # prevent logging user logging keys
cleaned_user_api_key_metadata[
k
] = "scrubbed_by_litellm_for_sensitive_keys"
cleaned_user_api_key_metadata[k] = (
"scrubbed_by_litellm_for_sensitive_keys"
)
else:
cleaned_user_api_key_metadata[k] = v
@ -5616,4 +5621,3 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
model_parameters={"stream": True},
hidden_params=hidden_params,
)

95
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -20,6 +20,7 @@ from typing import (
cast,
)
from litellm import verbose_logger
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
@ -1278,16 +1279,76 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
return images
def _attempt_json_repair(s: str) -> Optional[Any]:
"""
Attempt to repair truncated JSON produced by LLM tool calls.
Handles the most common truncation patterns where the model generates
valid JSON that is cut short (missing closing brackets/braces).
Returns the parsed value on success, or None if repair fails.
"""
import json
stripped = s.rstrip()
if not stripped:
return None
# Track the stack of unmatched openers to respect nesting order
opener_stack: list = []
in_string = False
escape_next = False
for ch in stripped:
if escape_next:
escape_next = False
continue
if ch == "\\":
if in_string:
escape_next = True
continue
if ch == '"':
in_string = not in_string
continue
if in_string:
continue
if ch == "{":
opener_stack.append("}")
elif ch == "[":
opener_stack.append("]")
elif ch in ("}", "]"):
if opener_stack and opener_stack[-1] == ch:
opener_stack.pop()
if not opener_stack:
return None
# Remove trailing comma before we close brackets
candidate = stripped.rstrip(",")
# Close in reverse order of opening (respects nesting)
candidate += "".join(reversed(opener_stack))
try:
return json.loads(candidate)
except json.JSONDecodeError:
pass
return None
def parse_tool_call_arguments(
arguments: Optional[str],
tool_name: Optional[str] = None,
context: Optional[str] = None,
) -> Dict[str, Any]:
) -> Any:
"""
Parse tool call arguments from a JSON string.
This function handles malformed JSON gracefully by raising a ValueError
with context about what failed and what the problematic input was.
When the JSON is malformed (e.g. truncated by the model), this function
attempts a lightweight repair (closing unmatched brackets/braces) before
raising an error. A warning is logged whenever repair succeeds so that
callers are aware the arguments were not perfectly formed.
Args:
arguments: The JSON string containing tool arguments, or None.
@ -1295,19 +1356,34 @@ def parse_tool_call_arguments(
context: Optional context string (e.g., "Anthropic Messages API").
Returns:
Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty.
Parsed arguments (usually a dict, but may be any JSON-deserializable
type such as list, str, int, float, or None). Returns empty dict if
arguments is None or empty.
Raises:
ValueError: If the arguments string is not valid JSON.
ValueError: If the arguments string is not valid JSON and cannot be repaired.
"""
import json
if not arguments:
if not arguments or not arguments.strip():
return {}
try:
return json.loads(arguments)
except json.JSONDecodeError as e:
except json.JSONDecodeError as original_error:
repaired = _attempt_json_repair(arguments)
if repaired is not None:
verbose_logger.warning(
"Repaired truncated tool call arguments for tool '%s' (%s). "
"Original (%d chars): %.200s%s",
tool_name or "<unknown>",
context or "unknown context",
len(arguments),
arguments,
"..." if len(arguments) > 200 else "",
)
return repaired
error_parts = ["Failed to parse tool call arguments"]
if tool_name:
@ -1316,10 +1392,11 @@ def parse_tool_call_arguments(
error_parts.append(f"({context})")
error_message = (
" ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}"
" ".join(error_parts)
+ f". Error: {str(original_error)}. Arguments: {arguments}"
)
raise ValueError(error_message) from e
raise ValueError(error_message) from original_error
def split_concatenated_json_objects(raw: str) -> List[Dict[str, Any]]:

10
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1035,9 +1035,13 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
)
parameters = "".join(
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
)
if isinstance(parsed_args, dict):
parameters = "".join(
f"<{param}>{val}</{param}>\n"
for param, val in parsed_args.items()
)
else:
parameters = f"<result>{parsed_args}</result>\n"
invokes += (
"<invoke>\n"
f"<tool_name>{tool_name}</tool_name>\n"

9
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1099,7 +1099,14 @@ class CustomStreamWrapper:
and self.custom_llm_provider in litellm._custom_providers
):
if self.received_finish_reason is not None:
if "provider_specific_fields" not in chunk:
_chunk_has_content = isinstance(chunk, dict) and (
bool(chunk.get("text", ""))
or chunk.get("tool_use") is not None
)
if not _chunk_has_content and (
not isinstance(chunk, dict)
or "provider_specific_fields" not in chunk
):
raise StopIteration
anthropic_response_obj: GChunk = cast(GChunk, chunk)
completion_obj["content"] = anthropic_response_obj["text"]

10
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -75,7 +75,7 @@ class AnthropicMessagesHandler(BaseTranslation):
if messages is None:
return data
chat_completion_compatible_request, tool_name_mapping = (
chat_completion_compatible_request, _tool_name_mapping = (
LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
# Use a shallow copy to avoid mutating request data (pop on litellm_metadata).
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
@ -141,6 +141,14 @@ class AnthropicMessagesHandler(BaseTranslation):
return data
def extract_request_tool_names(self, data: dict) -> List[str]:
"""Extract tool names from Anthropic messages request (tools[].name)."""
names: List[str] = []
for tool in data.get("tools") or []:
if isinstance(tool, dict) and tool.get("name"):
names.append(str(tool["name"]))
return names
def _extract_input_text_and_images(
self,
message: Dict[str, Any],

136
litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py
View file

@ -41,7 +41,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
type="text",
text="",
)
pending_new_content_block: bool = False
chunk_queue: deque = deque() # Queue for buffering multiple chunks
def __init__(
@ -80,38 +79,40 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
from .transformation import LiteLLMAnthropicMessagesAdapter
try:
# Always return queued chunks first
if self.chunk_queue:
return self.chunk_queue.popleft()
# Queue initial chunks if not sent yet
if self.sent_first_chunk is False:
self.sent_first_chunk = True
return {
"type": "message_start",
"message": {
"id": "msg_{}".format(uuid.uuid4()),
"type": "message",
"role": "assistant",
"content": [],
"model": self.model,
"stop_reason": None,
"stop_sequence": None,
"usage": self._create_initial_usage_delta(),
},
}
self.chunk_queue.append(
{
"type": "message_start",
"message": {
"id": "msg_{}".format(uuid.uuid4()),
"type": "message",
"role": "assistant",
"content": [],
"model": self.model,
"stop_reason": None,
"stop_sequence": None,
"usage": self._create_initial_usage_delta(),
},
}
)
return self.chunk_queue.popleft()
if self.sent_content_block_start is False:
self.sent_content_block_start = True
return {
"type": "content_block_start",
"index": self.current_content_block_index,
"content_block": {"type": "text", "text": ""},
}
# Handle pending new content block start
if self.pending_new_content_block:
self.pending_new_content_block = False
self.sent_content_block_finish = False # Reset for new block
return {
"type": "content_block_start",
"index": self.current_content_block_index,
"content_block": self.current_content_block_start,
}
self.chunk_queue.append(
{
"type": "content_block_start",
"index": self.current_content_block_index,
"content_block": {"type": "text", "text": ""},
}
)
return self.chunk_queue.popleft()
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
@ -126,45 +127,65 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
)
# Check if we need to start a new content block
# This is where you'd add your logic to detect when a new content block should start
# For example, if the chunk indicates a tool call or different content type
if should_start_new_block and not self.sent_content_block_finish:
# End current content block and prepare for new one
self.holding_chunk = processed_chunk
self.sent_content_block_finish = True
self.pending_new_content_block = True
return {
"type": "content_block_stop",
"index": max(self.current_content_block_index - 1, 0),
}
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
self.chunk_queue.append(
{
"type": "content_block_stop",
"index": max(self.current_content_block_index - 1, 0),
}
)
self.chunk_queue.append(
{
"type": "content_block_start",
"index": self.current_content_block_index,
"content_block": self.current_content_block_start,
}
)
self.sent_content_block_finish = False
return self.chunk_queue.popleft()
if (
processed_chunk["type"] == "message_delta"
and self.sent_content_block_finish is False
):
self.holding_chunk = processed_chunk
# Queue both the content_block_stop and the message_delta
self.chunk_queue.append(
{
"type": "content_block_stop",
"index": self.current_content_block_index,
}
)
self.sent_content_block_finish = True
return {
"type": "content_block_stop",
"index": self.current_content_block_index,
}
self.chunk_queue.append(processed_chunk)
return self.chunk_queue.popleft()
elif self.holding_chunk is not None:
return_chunk = self.holding_chunk
self.holding_chunk = processed_chunk
return return_chunk
self.chunk_queue.append(self.holding_chunk)
self.chunk_queue.append(processed_chunk)
self.holding_chunk = None
return self.chunk_queue.popleft()
else:
return processed_chunk
self.chunk_queue.append(processed_chunk)
return self.chunk_queue.popleft()
# Handle any remaining held chunks after stream ends
if self.holding_chunk is not None:
return_chunk = self.holding_chunk
self.chunk_queue.append(self.holding_chunk)
self.holding_chunk = None
return return_chunk
if self.sent_last_message is False:
if not self.sent_last_message:
self.sent_last_message = True
return {"type": "message_stop"}
self.chunk_queue.append({"type": "message_stop"})
if self.chunk_queue:
return self.chunk_queue.popleft()
raise StopIteration
except StopIteration:
if self.chunk_queue:
return self.chunk_queue.popleft()
if self.sent_last_message is False:
self.sent_last_message = True
return {"type": "message_stop"}
@ -265,7 +286,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
if not self.queued_usage_chunk:
if should_start_new_block and not self.sent_content_block_finish:
# Queue the sequence: content_block_stop -> content_block_start -> current_chunk
# Queue the sequence: content_block_stop -> content_block_start
# The trigger chunk itself is not emitted as a delta since the
# content_block_start already carries the relevant information.
# 1. Stop current content block
self.chunk_queue.append(
@ -284,9 +307,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
}
)
# 3. Queue the current chunk (don't lose it!)
self.chunk_queue.append(processed_chunk)
# Reset state for new block
self.sent_content_block_finish = False

8
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -43,8 +43,12 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
if "tool_choice" not in params:
params.append("tool_choice")
# Only gpt-5.2 has been verified to support logprobs on Azure
if self.is_model_gpt_5_2_model(model):
# Only gpt-5.2 has been verified to support logprobs on Azure.
# The base OpenAI class includes logprobs for gpt-5.1+, but Azure
# hasn't verified support for gpt-5.1, so remove them unless gpt-5.2.
if self.is_model_gpt_5_1_model(model) and not self.is_model_gpt_5_2_model(model):
params = [p for p in params if p not in ["logprobs", "top_logprobs"]]
elif self.is_model_gpt_5_2_model(model):
azure_supported_params = ["logprobs", "top_logprobs"]
params.extend(azure_supported_params)

38
litellm/llms/azure/common_utils.py
View file

@ -1,6 +1,6 @@
import json
import os
from typing import Any, Callable, Dict, Literal, Optional, Union, cast
from typing import Any, Callable, Dict, Literal, NamedTuple, Optional, Union, cast
import httpx
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
@ -789,3 +789,39 @@ class BaseAzureLLM(BaseOpenAILLM):
return param_value
return os.getenv(env_var_key)
class AzureCredentials(NamedTuple):
api_base: Optional[str]
api_key: Optional[str]
api_version: Optional[str]
def get_azure_credentials(
api_base: Optional[str] = None,
api_key: Optional[str] = None,
api_version: Optional[str] = None,
) -> AzureCredentials:
"""Resolve Azure credentials from params, litellm globals, and env vars."""
resolved_api_base = (
api_base
or litellm.api_base
or get_secret_str("AZURE_API_BASE")
)
resolved_api_version = (
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
resolved_api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
return AzureCredentials(
api_base=resolved_api_base,
api_key=resolved_api_key,
api_version=resolved_api_version,
)

7
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -98,3 +98,10 @@ class BaseTranslation(ABC):
Optional to override in subclasses.
"""
return responses_so_far
def extract_request_tool_names(self, data: dict) -> List[str]:
"""
Extract tool names from the request body for allowlist/policy checks.
Override in tool-capable handlers; default returns [].
"""
return []

12
litellm/llms/base_llm/responses/transformation.py
View file

@ -218,6 +218,18 @@ class BaseResponsesAPIConfig(ABC):
"""Returns True if litellm should fake a stream for the given model and stream value"""
return False
def supports_native_websocket(self) -> bool:
"""
Returns True if the provider has a native WebSocket endpoint for Responses API.
Providers with native websocket support can connect directly to wss:// endpoints.
Providers without native support will use the ManagedResponsesWebSocketHandler
which makes HTTP streaming calls and forwards events over the websocket.
Default: False (use managed websocket handler)
"""
return False
#########################################################
########## CANCEL RESPONSE API TRANSFORMATION ##########
#########################################################

10
litellm/llms/chatgpt/responses/transformation.py
View file

@ -1,14 +1,14 @@
import json
from typing import Any, Optional
from litellm.exceptions import AuthenticationError
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
@ -200,3 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base = api_base or self.authenticator.get_api_base() or CHATGPT_API_BASE
api_base = api_base.rstrip("/")
return f"{api_base}/responses"
def supports_native_websocket(self) -> bool:
"""ChatGPT does not support native WebSocket for Responses API"""
return False

118
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -69,6 +69,7 @@ from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
MockResponsesAPIStreamingIterator,
ResponsesAPIStreamingIterator,
ResponsesWebSocketStreaming,
SyncResponsesAPIStreamingIterator,
)
from litellm.types.containers.main import (
@ -4731,6 +4732,123 @@ class BaseLLMHTTPHandler:
f"Unexpected error while closing WebSocket: {close_error}"
)
async def async_responses_websocket(
self,
model: str,
websocket: Any,
logging_obj: LiteLLMLoggingObj,
responses_api_provider_config: Optional[BaseResponsesAPIConfig],
api_base: Optional[str] = None,
api_key: Optional[str] = None,
timeout: Optional[float] = None,
user_api_key_dict: Optional[Any] = None,
litellm_metadata: Optional[Dict[str, Any]] = None,
custom_llm_provider: Optional[str] = None,
**kwargs: Any,
):
"""
Handles Responses API WebSocket mode.
For providers with native websocket support (OpenAI, Azure):
- Opens a persistent WebSocket to the provider's /v1/responses endpoint
- Proxies response.create events bidirectionally for lower-latency agentic workflows
For providers without native websocket support (all others):
- Uses ManagedResponsesWebSocketHandler which makes HTTP streaming calls
- Forwards events over the websocket connection
"""
if responses_api_provider_config is None or not responses_api_provider_config.supports_native_websocket():
from litellm.responses.streaming_iterator import (
ManagedResponsesWebSocketHandler,
)
handler = ManagedResponsesWebSocketHandler(
websocket=websocket,
model=model,
logging_obj=logging_obj,
user_api_key_dict=user_api_key_dict,
litellm_metadata=litellm_metadata,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,
)
await handler.run()
return
import websockets
from websockets.asyncio.client import ClientConnection
litellm_params = GenericLiteLLMParams()
headers = responses_api_provider_config.validate_environment(
headers={},
model=model,
litellm_params=litellm_params,
)
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
http_url = responses_api_provider_config.get_complete_url(
api_base=api_base,
litellm_params={},
)
ws_url = http_url.replace("https://", "wss://").replace("http://", "ws://")
try:
ssl_context = get_shared_realtime_ssl_context()
if ws_url.startswith("wss://") and ssl_context is False:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
logging_obj.pre_call(
input=None,
api_key=api_key or "",
additional_args={
"api_base": ws_url,
"headers": headers,
"complete_input_dict": {"mode": "responses_websocket"},
},
)
async with websockets.connect( # type: ignore
ws_url,
additional_headers=headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
) as backend_ws:
_request_data: Dict[str, Any] = {}
if litellm_metadata:
_request_data["litellm_metadata"] = litellm_metadata
streaming = ResponsesWebSocketStreaming(
websocket=websocket,
backend_ws=cast(ClientConnection, backend_ws),
logging_obj=logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=_request_data,
)
await streaming.bidirectional_forward()
except websockets.exceptions.InvalidStatusCode as e: # type: ignore
verbose_logger.exception(f"Error connecting to responses WS backend: {e}")
await websocket.close(code=e.status_code, reason=str(e))
except Exception as e:
verbose_logger.exception(f"Error in responses WS: {e}")
try:
await websocket.close(
code=1011, reason=f"Internal server error: {str(e)}"
)
except RuntimeError as close_error:
if "already completed" in str(close_error) or "websocket.close" in str(
close_error
):
pass
else:
raise Exception(
f"Unexpected error while closing WebSocket: {close_error}"
)
def image_edit_handler(
self,
model: str,

4
litellm/llms/databricks/responses/transformation.py
View file

@ -98,3 +98,7 @@ class DatabricksResponsesAPIConfig(DatabricksBase, OpenAIResponsesAPIConfig):
litellm_params=litellm_params,
headers=headers,
)
def supports_native_websocket(self) -> bool:
"""Databricks does not support native WebSocket for Responses API"""
return False

6
litellm/llms/github_copilot/responses/transformation.py
View file

@ -22,8 +22,8 @@ from litellm.types.utils import LlmProviders
from ..authenticator import Authenticator
from ..common_utils import (
GetAPIKeyError,
GITHUB_COPILOT_API_BASE,
GetAPIKeyError,
get_copilot_default_headers,
)
@ -329,3 +329,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
return False
def supports_native_websocket(self) -> bool:
"""GitHub Copilot does not support native WebSocket for Responses API"""
return False

4
litellm/llms/hosted_vllm/responses/transformation.py
View file

@ -69,3 +69,7 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
return f"{api_base}/responses"
return f"{api_base}/v1/responses"
def supports_native_websocket(self) -> bool:
"""Hosted vLLM does not support native WebSocket for Responses API"""
return False

4
litellm/llms/litellm_proxy/responses/transformation.py
View file

@ -46,3 +46,7 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base = api_base.rstrip("/")
return f"{api_base}/responses"
def supports_native_websocket(self) -> bool:
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
return False

4
litellm/llms/manus/responses/transformation.py
View file

@ -247,6 +247,10 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
response._hidden_params["headers"] = raw_response_headers
return response
def supports_native_websocket(self) -> bool:
"""Manus does not support native WebSocket for Responses API"""
return False
def transform_get_response_api_request(
self,
response_id: str,

13
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -135,6 +135,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return data
def extract_request_tool_names(self, data: dict) -> List[str]:
"""Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name)."""
names: List[str] = []
for tool in data.get("tools") or []:
if isinstance(tool, dict) and tool.get("type") == "function":
fn = tool.get("function")
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
for fn in data.get("functions") or []:
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
return names
def _extract_inputs(
self,
message: Dict[str, Any],

39
litellm/llms/openai/common_utils.py
View file

@ -5,8 +5,9 @@ Common helpers / utils across al OpenAI endpoints
import hashlib
import inspect
import json
import os
import ssl
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Dict, List, Literal, NamedTuple, Optional, Tuple, Union
import httpx
import openai
@ -244,3 +245,39 @@ class BaseOpenAILLM:
)
class OpenAICredentials(NamedTuple):
api_base: str
api_key: Optional[str]
organization: Optional[str]
def get_openai_credentials(
api_base: Optional[str] = None,
api_key: Optional[str] = None,
organization: Optional[str] = None,
) -> OpenAICredentials:
"""Resolve OpenAI credentials from params, litellm globals, and env vars."""
resolved_api_base = (
api_base
or litellm.api_base
or os.getenv("OPENAI_BASE_URL")
or os.getenv("OPENAI_API_BASE")
or "https://api.openai.com/v1"
)
resolved_organization = (
organization
or litellm.organization
or os.getenv("OPENAI_ORGANIZATION", None)
or None
)
resolved_api_key = (
api_key
or litellm.api_key
or litellm.openai_key
or os.getenv("OPENAI_API_KEY")
)
return OpenAICredentials(
api_base=resolved_api_base,
api_key=resolved_api_key,
organization=resolved_organization,
)

39
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -30,27 +30,22 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_tool_call import \
ResponseFunctionToolCall
from pydantic import BaseModel
from litellm._logging import verbose_proxy_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
OutputFunctionToolCall,
OutputText,
)
OpenAiResponsesToChatCompletionStreamIterator)
from litellm.llms.base_llm.guardrail_translation.base_translation import \
BaseTranslation
from litellm.responses.litellm_completion_transformation.transformation import \
LiteLLMCompletionResponsesConfig
from litellm.types.llms.openai import (ChatCompletionToolCallChunk,
ChatCompletionToolParam)
from litellm.types.responses.main import (GenericResponseOutputItem,
OutputFunctionToolCall, OutputText)
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
@ -188,6 +183,18 @@ class OpenAIResponsesHandler(BaseTranslation):
return data
def extract_request_tool_names(self, data: dict) -> List[str]:
"""Extract tool names from Responses API request (tools[].name for function, tools[].server_label for mcp)."""
names: List[str] = []
for tool in data.get("tools") or []:
if not isinstance(tool, dict):
continue
if tool.get("type") == "function" and tool.get("name"):
names.append(str(tool["name"]))
elif tool.get("type") == "mcp" and tool.get("server_label"):
names.append(str(tool["server_label"]))
return names
def _extract_and_transform_tools(
self,
tools: List[Dict[str, Any]],

9
litellm/llms/openai/responses/transformation.py
View file

@ -344,6 +344,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
return False
def supports_native_websocket(self) -> bool:
"""OpenAI supports native WebSocket for Responses API"""
return True
#########################################################
########## DELETE RESPONSE API TRANSFORMATION ##############
#########################################################
@ -524,7 +528,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- POST /v1/responses/compact
"""
url = f"{api_base}/compact"
# Preserve query params (e.g., api-version) while appending /compact.
parsed_url = httpx.URL(api_base)
compact_path = parsed_url.path.rstrip("/") + "/compact"
url = str(parsed_url.copy_with(path=compact_path))
input = self._validate_input_param(input)
data = dict(

11
litellm/llms/openrouter/image_edit/__init__.py Normal file
View file

@ -0,0 +1,11 @@
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .transformation import OpenRouterImageEditConfig
__all__ = [
"OpenRouterImageEditConfig",
]
def get_openrouter_image_edit_config(model: str) -> BaseImageEditConfig:
return OpenRouterImageEditConfig()

367
litellm/llms/openrouter/image_edit/transformation.py Normal file
View file

@ -0,0 +1,367 @@
"""
OpenRouter Image Edit Support
OpenRouter provides image editing through chat completion endpoints.
The source image is sent as a base64 data URL in the message content,
and the response contains edited images in the message's images array.
Request format:
{
"model": "google/gemini-2.5-flash-image",
"messages": [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}},
{"type": "text", "text": "Edit this image by..."}
]
}],
"modalities": ["image", "text"]
}
Response format:
{
"choices": [{
"message": {
"content": "Here is the edited image.",
"role": "assistant",
"images": [{
"image_url": {"url": "data:image/png;base64,..."},
"type": "image_url"
}]
}
}],
"usage": {
"completion_tokens": 1299,
"prompt_tokens": 300,
"total_tokens": 1599,
"completion_tokens_details": {"image_tokens": 1290},
"cost": 0.0387243
}
}
"""
import base64
from io import BufferedReader, BytesIO
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.openrouter.common_utils import OpenRouterException
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OpenRouterImageEditConfig(BaseImageEditConfig):
"""
Configuration for OpenRouter image editing via chat completions.
OpenRouter uses the chat completions endpoint for image editing.
The source image is sent as a base64 data URL in the message content,
and the response contains edited images in the message's images array.
"""
def get_supported_openai_params(self, model: str) -> list:
return ["size", "quality", "n"]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
supported_params = self.get_supported_openai_params(model)
mapped_params: Dict[str, Any] = {}
for key, value in image_edit_optional_params.items():
if key in supported_params:
if key == "size":
if "image_config" not in mapped_params:
mapped_params["image_config"] = {}
mapped_params["image_config"]["aspect_ratio"] = self._map_size_to_aspect_ratio(value)
elif key == "quality":
image_size = self._map_quality_to_image_size(value)
if image_size:
if "image_config" not in mapped_params:
mapped_params["image_config"] = {}
mapped_params["image_config"]["image_size"] = image_size
else:
mapped_params[key] = value
return mapped_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or get_secret_str("OPENROUTER_API_KEY")
)
if not api_key:
raise ValueError("OPENROUTER_API_KEY is not set")
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
)
return headers
def use_multipart_form_data(self) -> bool:
"""OpenRouter uses JSON requests, not multipart/form-data."""
return False
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
base_url = api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1"
base_url = base_url.rstrip("/")
if not base_url.endswith("/chat/completions"):
return f"{base_url}/chat/completions"
return base_url
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],
image: Optional[FileTypes],
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
content_parts: List[Dict[str, Any]] = []
# Add source image(s) as base64 data URLs
if image is not None:
images = image if isinstance(image, list) else [image]
for img in images:
if img is None:
continue
mime_type = ImageEditRequestUtils.get_image_content_type(img)
image_bytes = self._read_image_bytes(img)
b64_data = base64.b64encode(image_bytes).decode("utf-8")
content_parts.append(
{
"type": "image_url",
"image_url": {
"url": f"data:{mime_type};base64,{b64_data}"
},
}
)
# Add the text prompt
if prompt:
content_parts.append({"type": "text", "text": prompt})
request_body: Dict[str, Any] = {
"model": model,
"messages": [
{
"role": "user",
"content": content_parts,
}
],
"modalities": ["image", "text"],
}
# Add mapped optional params (image_config, n, etc.)
for key, value in image_edit_optional_request_params.items():
if key not in ("model", "messages", "modalities"):
request_body[key] = value
empty_files = cast(RequestFiles, [])
return request_body, empty_files
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ImageResponse:
try:
response_json = raw_response.json()
except Exception as e:
raise OpenRouterException(
message=f"Error parsing OpenRouter response: {str(e)}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
model_response = ImageResponse()
model_response.data = []
try:
choices = response_json.get("choices", [])
for choice in choices:
message = choice.get("message", {})
images = message.get("images", [])
for image_data in images:
image_url_obj = image_data.get("image_url", {})
image_url = image_url_obj.get("url")
if image_url:
if image_url.startswith("data:"):
# Extract base64 data from data URL
parts = image_url.split(",", 1)
b64_data = parts[1] if len(parts) > 1 else None
model_response.data.append(
ImageObject(
b64_json=b64_data,
url=None,
revised_prompt=None,
)
)
else:
model_response.data.append(
ImageObject(
b64_json=None,
url=image_url,
revised_prompt=None,
)
)
except Exception as e:
raise OpenRouterException(
message=f"Error transforming OpenRouter image edit response: {str(e)}",
status_code=500,
headers={},
)
self._set_usage_and_cost(model_response, response_json, model)
return model_response
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
return OpenRouterException(
message=error_message,
status_code=status_code,
headers=headers,
)
# Private helper methods
def _map_size_to_aspect_ratio(self, size: str) -> str:
"""
Map OpenAI size format to OpenRouter aspect_ratio format.
Uses the same mapping as image generation since OpenRouter
handles both through the same chat completions endpoint.
"""
size_to_aspect_ratio = {
"256x256": "1:1",
"512x512": "1:1",
"1024x1024": "1:1",
"1536x1024": "3:2",
"1792x1024": "16:9",
"1024x1536": "2:3",
"1024x1792": "9:16",
"auto": "1:1",
}
return size_to_aspect_ratio.get(size, "1:1")
def _map_quality_to_image_size(self, quality: str) -> Optional[str]:
"""
Map OpenAI quality to OpenRouter image_size format.
Uses the same mapping as image generation since OpenRouter
handles both through the same chat completions endpoint.
"""
quality_to_image_size = {
"low": "1K",
"standard": "1K",
"medium": "2K",
"high": "4K",
"hd": "4K",
"auto": "1K",
}
return quality_to_image_size.get(quality)
def _set_usage_and_cost(
self,
model_response: ImageResponse,
response_json: dict,
model: str,
) -> None:
"""Extract and set usage and cost information from OpenRouter response."""
usage_data = response_json.get("usage", {})
if usage_data:
prompt_tokens = usage_data.get("prompt_tokens", 0)
total_tokens = usage_data.get("total_tokens", 0)
completion_tokens_details = usage_data.get("completion_tokens_details", {})
image_tokens = completion_tokens_details.get("image_tokens", 0)
# For image edit, input may include image tokens
input_image_tokens = 0
prompt_tokens_details = usage_data.get("prompt_tokens_details", {})
if prompt_tokens_details:
input_image_tokens = prompt_tokens_details.get("image_tokens", 0)
model_response.usage = ImageUsage(
input_tokens=prompt_tokens,
input_tokens_details=ImageUsageInputTokensDetails(
image_tokens=input_image_tokens,
text_tokens=prompt_tokens - input_image_tokens,
),
output_tokens=image_tokens,
total_tokens=total_tokens,
)
cost = usage_data.get("cost")
if cost is not None:
if not hasattr(model_response, "_hidden_params"):
model_response._hidden_params = {}
if "additional_headers" not in model_response._hidden_params:
model_response._hidden_params["additional_headers"] = {}
model_response._hidden_params["additional_headers"][
"llm_provider-x-litellm-response-cost"
] = float(cost)
cost_details = usage_data.get("cost_details", {})
if cost_details:
if "response_cost_details" not in model_response._hidden_params:
model_response._hidden_params["response_cost_details"] = {}
model_response._hidden_params["response_cost_details"].update(cost_details)
model_response._hidden_params["model"] = response_json.get("model", model)
def _read_image_bytes(self, image: FileTypes) -> bytes:
"""Read raw bytes from various image input types."""
if isinstance(image, bytes):
return image
if isinstance(image, BytesIO):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
if isinstance(image, BufferedReader):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
raise ValueError("Unsupported image type for OpenRouter image edit.")

4
litellm/llms/openrouter/responses/transformation.py
View file

@ -75,3 +75,7 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig):
api_base = api_base.rstrip("/")
return f"{api_base}/responses"
def supports_native_websocket(self) -> bool:
"""OpenRouter does not support native WebSocket for Responses API"""
return False

4
litellm/llms/perplexity/responses/transformation.py
View file

@ -490,3 +490,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
verbose_logger.debug("Failed to transform Perplexity cost object: %s", e)
return chunk
def supports_native_websocket(self) -> bool:
"""Perplexity does not support native WebSocket for Responses API"""
return False

2
litellm/llms/sap/chat/handler.py
View file

@ -181,7 +181,7 @@ class AsyncSAPStreamIterator:
def __init__(
self,
response:AsyncIterator,
response: AsyncIterator,
event_prefix: str = "data: ",
final_msg: str = "[DONE]",
):

14
litellm/llms/sap/chat/models.py
View file

@ -46,9 +46,21 @@ class FunctionObj(BaseModel):
class FunctionTool(BaseModel):
description: str = ""
name: str
parameters: dict = {}
parameters: dict = {"type": "object", "properties": {}}
strict: bool = False
@field_validator("parameters", mode="before")
@classmethod
def ensure_object_type(cls, v: dict) -> dict:
"""Ensure parameters has type='object' as required by SAP Orchestration Service."""
if not v:
return {"type": "object", "properties": {}}
if "type" not in v:
v = {"type": "object", **v}
if "properties" not in v:
v["properties"] = {}
return v
class ChatCompletionTool(BaseModel):
type_: Literal["function"] = Field(default="function", alias="type")

47
litellm/llms/sap/chat/transformation.py
View file

@ -159,9 +159,9 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
"response_format",
"timeout",
]
# Remove response_format for providers that don't support it on SAP GenAI Hub
if (
model.startswith('anthropic')
or model.startswith("amazon")
model.startswith("amazon")
or model.startswith("cohere")
or model.startswith("alephalpha")
or model == "gpt-4"
@ -171,6 +171,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
params.remove("tool_choice")
return params
def validate_environment(
self,
headers: dict,
@ -207,6 +208,12 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
) -> dict:
optional_params.pop("deployment_url", None)
# Filter strict for GPT models only - SAP AI Core doesn't accept it as a model param
# LangChain agents pass strict=true at top level, which fails for GPT models
# Anthropic models accept strict, so preserve it for them
if model.startswith("gpt"):
optional_params.pop("strict")
def _build_prompt_module(
*,
model_name: str,
@ -324,7 +331,37 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
original_response=raw_response.text,
additional_args={"complete_input_dict": request_data},
)
return ModelResponse.model_validate(raw_response.json()["final_result"])
response = ModelResponse.model_validate(raw_response.json()["final_result"])
# Strip markdown code blocks if JSON response_format was used with Anthropic models
# SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ```
# based on prompt phrasing. GPT/Gemini models don't exhibit this behavior,
# so we gate the stripping to avoid accidentally modifying valid responses.
response_format = optional_params.get("response_format", {})
if response_format.get("type") in ("json_object", "json_schema"):
if model.startswith("anthropic"):
response = self._strip_markdown_json(response)
return response
def _strip_markdown_json(self, response: ModelResponse) -> ModelResponse:
"""Strip markdown code block wrapper from JSON content if present.
SAP GenAI Hub with Anthropic models sometimes returns JSON wrapped in
markdown code blocks (```json ... ```) depending on prompt phrasing.
This method strips that wrapper to ensure consistent JSON output.
"""
import re
for choice in response.choices or []:
if choice.message and choice.message.content:
content = choice.message.content.strip()
# Match ```json ... ``` or ``` ... ```
match = re.match(r'^```(?:json)?\s*\n?(.*?)\n?```$', content, re.DOTALL)
if match:
choice.message.content = match.group(1).strip()
return response
def get_model_response_iterator(
self,
@ -333,6 +370,6 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
json_mode: Optional[bool] = False,
):
if sync_stream:
return SAPStreamIterator(response=streaming_response) # type: ignore
return SAPStreamIterator(response=streaming_response) # type: ignore
else:
return AsyncSAPStreamIterator(response=streaming_response) # type: ignore
return AsyncSAPStreamIterator(response=streaming_response) # type: ignore

1
litellm/llms/searchapi/__init__.py Normal file
View file

@ -0,0 +1 @@
"""SearchAPI.io integration for LiteLLM."""

4
litellm/llms/searchapi/search/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""SearchAPI.io search integration for LiteLLM."""
from litellm.llms.searchapi.search.transformation import SearchAPIConfig
__all__ = ["SearchAPIConfig"]

232
litellm/llms/searchapi/search/transformation.py Normal file
View file

@ -0,0 +1,232 @@
"""
Calls SearchAPI.io's Google Search API endpoint.
SearchAPI.io API Reference: https://www.searchapi.io/docs/google
"""
from typing import Dict, List, Literal, Optional, TypedDict, Union
from urllib.parse import urlencode
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
class _SearchAPIRequestRequired(TypedDict):
"""Required fields for SearchAPI.io request."""
engine: str # Required - search engine (e.g., 'google')
q: str # Required - search query
class SearchAPIRequest(_SearchAPIRequestRequired, total=False):
"""
SearchAPI.io request format for Google Search.
Based on: https://www.searchapi.io/docs/google
"""
kgmid: str # Optional - Knowledge Graph identifier
device: str # Optional - device type ('desktop', 'mobile', 'tablet')
location: str # Optional - geographic location
uule: str # Optional - Google-encoded location
google_domain: str # Optional - Google domain (deprecated)
gl: str # Optional - country code (e.g., 'us', 'uk')
hl: str # Optional - interface language (e.g., 'en', 'es')
lr: str # Optional - language restriction (e.g., 'lang_en')
cr: str # Optional - country restriction
nfpr: int # Optional - exclude auto-corrected results (0 or 1)
filter: int # Optional - duplicate/host crowding filter (0 or 1)
safe: str # Optional - SafeSearch ('active', 'off')
time_period: str # Optional - time period ('last_hour', 'last_day', 'last_week', 'last_month', 'last_year')
time_period_min: str # Optional - start date (MM/DD/YYYY)
time_period_max: str # Optional - end date (MM/DD/YYYY)
num: int # Optional - number of results (phased out by Google, constant 10)
page: int # Optional - page number for pagination
optimization_strategy: str # Optional - 'performance' or 'ads'
class SearchAPIConfig(BaseSearchConfig):
SEARCHAPI_API_BASE = "https://www.searchapi.io/api/v1/search"
@staticmethod
def ui_friendly_name() -> str:
return "SearchAPI.io (Google Search)"
def get_http_method(self) -> Literal["GET", "POST"]:
"""
SearchAPI.io uses GET requests for search.
"""
return "GET"
def validate_environment(
self,
headers: Dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
**kwargs,
) -> Dict:
"""
Validate environment and return headers.
"""
api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
if not api_key:
raise ValueError(
"SEARCHAPI_API_KEY is not set. Set `SEARCHAPI_API_KEY` environment variable."
)
headers["Content-Type"] = "application/json"
return headers
def get_complete_url(
self,
api_base: Optional[str],
optional_params: dict,
data: Optional[Union[Dict, List[Dict]]] = None,
**kwargs,
) -> str:
"""
Get complete URL for Search endpoint with query parameters.
SearchAPI.io uses GET requests and includes api_key in query params.
"""
api_base = api_base or get_secret_str("SEARCHAPI_API_BASE") or self.SEARCHAPI_API_BASE
# Build query parameters from the transformed request body
if data and isinstance(data, dict) and "_searchapi_params" in data:
params = data["_searchapi_params"]
query_string = urlencode(params, doseq=True)
return f"{api_base}?{query_string}"
return api_base
def transform_search_request(
self,
query: Union[str, List[str]],
optional_params: dict,
api_key: Optional[str] = None,
search_engine_id: Optional[str] = None,
**kwargs,
) -> Dict:
"""
Transform Search request to SearchAPI.io format.
Transforms unified spec parameters:
- query → q
- max_results → num (limited to 10 by Google)
- search_domain_filter → q (append site: filters)
- country → gl
Args:
query: Search query (string or list of strings)
optional_params: Optional parameters for the request
api_key: API key for authentication
Returns:
Dict with typed request data following SearchAPI.io spec
"""
if isinstance(query, list):
query = " ".join(query)
# Get API key from parameter or environment
api_key = api_key or get_secret_str("SEARCHAPI_API_KEY")
if not api_key:
raise ValueError(
"SEARCHAPI_API_KEY is not set. Set `SEARCHAPI_API_KEY` environment variable."
)
request_data: SearchAPIRequest = {
"engine": "google",
"q": query,
}
# Add API key to request
result_data = dict(request_data)
result_data["api_key"] = api_key
# Transform unified spec parameters to SearchAPI.io format
if "max_results" in optional_params:
# Google now returns constant 10 results, but we can still set num
num_results = min(optional_params["max_results"], 10)
result_data["num"] = num_results
if "search_domain_filter" in optional_params:
# Convert to multiple "site:domain" clauses
domains = optional_params["search_domain_filter"]
if isinstance(domains, list) and len(domains) > 0:
result_data["q"] = self._append_domain_filters(
result_data["q"], domains
)
if "country" in optional_params:
# Map to gl parameter
result_data["gl"] = optional_params["country"].lower()
# Pass through all other SearchAPI.io-specific parameters
for param, value in optional_params.items():
if (
param not in self.get_supported_perplexity_optional_params()
and param not in result_data
):
result_data[param] = value
# Store params in special key for URL building (GET request)
return {
"_searchapi_params": result_data,
}
@staticmethod
def _append_domain_filters(query: str, domains: List[str]) -> str:
"""
Add site: filters to restrict search to specific domains.
"""
domain_clauses = [f"site:{domain}" for domain in domains]
domain_query = " OR ".join(domain_clauses)
return f"({query}) AND ({domain_query})"
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: Optional[LiteLLMLoggingObj],
**kwargs,
) -> SearchResponse:
"""
Transform SearchAPI.io response to LiteLLM unified SearchResponse format.
SearchAPI.io → LiteLLM mappings:
- organic_results[].title → SearchResult.title
- organic_results[].link → SearchResult.url
- organic_results[].snippet → SearchResult.snippet
- organic_results[].date → SearchResult.date
"""
response_json = raw_response.json()
# Transform results to SearchResult objects
results: List[SearchResult] = []
# Process organic results
for result in response_json.get("organic_results", []):
title = result.get("title", "")
url = result.get("link", "")
snippet = result.get("snippet", "")
date = result.get("date") # SearchAPI.io provides date in some results
search_result = SearchResult(
title=title,
url=url,
snippet=snippet,
date=date,
last_updated=None, # SearchAPI.io doesn't provide last_updated
)
results.append(search_result)
return SearchResponse(
results=results,
object="search",
)

5
litellm/llms/vertex_ai/batches/handler.py
View file

@ -115,9 +115,10 @@ class VertexAIBatchPrediction(VertexLLM):
data=json.dumps(vertex_batch_request),
)
except httpx.HTTPStatusError as e:
error_body = e.response.text if hasattr(e, 'response') else "N/A"
error_body = e.response.text
litellm.verbose_logger.error(
f"Vertex AI batch create failed: status={e.response.status_code}, body={error_body[:1000]}"
"Vertex AI batch create failed: status=%s, body=%s",
e.response.status_code, error_body[:1000],
)
raise
if response.status_code != 200:

36
litellm/llms/vertex_ai/common_utils.py
View file

@ -571,14 +571,38 @@ def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
return schema_dict
def _is_any_type_schema(schema: dict) -> bool:
"""
Detect schemas that represent "any JSON value" (no type constraints).
In JSON Schema, an empty schema {} means "any value is valid".
Schemas with only metadata keys (title, description, default, examples)
but no type-constraining keywords also represent "any type".
Gemini's Schema proto uses TYPE_UNSPECIFIED (0) as default,
so omitting the type field is valid and means "any type".
"""
type_constraining_keys = {
"type",
"properties",
"items",
"anyOf",
"oneOf",
"allOf",
"enum",
"required",
"$ref",
"$schema",
}
return not any(key in type_constraining_keys for key in schema.keys())
def process_items(schema, depth=0):
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
)
if isinstance(schema, dict):
if "items" in schema and schema["items"] == {}:
schema["items"] = {"type": "object"}
for key, value in schema.items():
if isinstance(value, dict):
process_items(value, depth + 1)
@ -677,9 +701,8 @@ def convert_anyof_null_to_nullable(schema, depth=0):
# remove null type
anyof.remove(atype)
contains_null = True
elif "type" not in atype and len(atype) == 0:
# Handle empty object case
atype["type"] = "object"
elif isinstance(atype, dict) and _is_any_type_schema(atype):
pass # preserve "any type" semantics — don't coerce to object
if len(anyof) == 0:
# Edge case: response schema with only null type present is invalid in Vertex AI
@ -714,7 +737,8 @@ def add_object_type(schema):
# Gemini requires all function parameters to be type OBJECT
# Handle case where schema has no properties and no type (e.g. tools with no arguments)
if "type" not in schema and "anyOf" not in schema and "oneOf" not in schema and "allOf" not in schema:
schema["type"] = "object"
if not _is_any_type_schema(schema):
schema["type"] = "object"
properties = schema.get("properties", None)
if properties is not None:

5
litellm/llms/vertex_ai/files/transformation.py
View file

@ -408,10 +408,11 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
file_id = "deleted"
if hasattr(raw_response, "request") and raw_response.request:
url = str(raw_response.request.url)
if "/o/" in url:
if "/b/" in url and "/o/" in url:
import urllib.parse
bucket_part = url.split("/b/")[-1].split("/o/")[0]
encoded_name = url.split("/o/")[-1].split("?")[0]
file_id = f"gs://{urllib.parse.unquote(encoded_name)}"
file_id = f"gs://{bucket_part}/{urllib.parse.unquote(encoded_name)}"
return FileDeleted(id=file_id, deleted=True, object="file")
def transform_list_files_request(

35
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -2905,6 +2905,7 @@ class ModelResponseIterator:
self.logging_obj = logging_obj
self.is_function_call = check_is_function_call(logging_obj)
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]:
try:
@ -2943,6 +2944,40 @@ class ModelResponseIterator:
cumulative_tool_call_index=self.cumulative_tool_call_index,
)
# Track whether tool_calls have been seen across streaming chunks.
# Gemini sends tool_calls and finishReason in separate chunks,
# so we need to remember if earlier chunks contained tool_calls
# to correctly set finish_reason="tool_calls" per the OpenAI spec.
if not self.has_seen_tool_calls:
for choice in model_response.choices:
if hasattr(choice, "delta") and choice.delta and choice.delta.tool_calls:
self.has_seen_tool_calls = True
break
# Handle final chunk with finishReason but no content.
# _process_candidates skips candidates without "content",
# so the finish_reason from the final chunk is lost.
if not model_response.choices and _candidates:
from litellm.types.utils import Delta, StreamingChoices
for candidate in _candidates:
finish_reason_str = candidate.get("finishReason")
if finish_reason_str is not None:
if self.has_seen_tool_calls:
mapped_finish_reason = "tool_calls"
else:
mapped_finish_reason = VertexGeminiConfig._check_finish_reason(
None, finish_reason_str
)
choice = StreamingChoices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", 0),
delta=Delta(content=None, role=None),
logprobs=None,
enhancements=None,
)
model_response.choices.append(choice)
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore

9
litellm/llms/volcengine/responses/transformation.py
View file

@ -16,16 +16,17 @@ from pydantic import fields as pyd_fields
import litellm
from litellm._logging import verbose_logger
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIStreamingResponse
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamingResponse,
)
from litellm.types.responses.main import DeleteResponseResult
from litellm.types.router import GenericLiteLLMParams
@ -555,3 +556,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Fall back to the first candidate
return candidates[0]
def supports_native_websocket(self) -> bool:
"""VolcEngine does not support native WebSocket for Responses API"""
return False

4
litellm/llms/xai/responses/transformation.py
View file

@ -252,3 +252,7 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
return f"{api_base}/responses"
def supports_native_websocket(self) -> bool:
"""XAI does not support native WebSocket for Responses API"""
return False

86
litellm/main.py
View file

@ -107,6 +107,7 @@ from litellm.realtime_api.main import _realtime_health_check
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
CustomPricingLiteLLMParams,
ModelResponseStream,
RawRequestTypedDict,
StreamingChoices,
@ -418,6 +419,8 @@ async def acompletion( # noqa: PLR0915
web_search_options: Optional[OpenAIWebSearchOptions] = None,
# Session management
shared_session: Optional["ClientSession"] = None,
# Per-request JSON schema validation (overrides litellm.enable_json_schema_validation)
enable_json_schema_validation: Optional[bool] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@ -562,6 +565,7 @@ async def acompletion( # noqa: PLR0915
"thinking": thinking,
"web_search_options": web_search_options,
"shared_session": shared_session,
"enable_json_schema_validation": enable_json_schema_validation,
}
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = get_llm_provider(
@ -996,6 +1000,32 @@ def _drop_input_examples_from_tools(
return cleaned_tools
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
model_info: Optional[dict] = None,
) -> dict:
"""Build a complete model cost entry from kwargs and model_info.
Collects all CustomPricingLiteLLMParams fields present in kwargs and
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
so that register_model() receives the full pricing configuration.
"""
entry: dict = {"litellm_provider": custom_llm_provider}
for field_name in CustomPricingLiteLLMParams.model_fields:
value = kwargs.get(field_name)
if value is not None:
entry[field_name] = value
if model_info and isinstance(model_info, dict):
for key in ("mode", "supports_prompt_caching", "max_tokens"):
if key in model_info and model_info[key] is not None:
entry.setdefault(key, model_info[key])
return entry
@tracer.wrap()
@client
def completion( # type: ignore # noqa: PLR0915
@ -1047,6 +1077,8 @@ def completion( # type: ignore # noqa: PLR0915
thinking: Optional[AnthropicThinkingParam] = None,
# Session management
shared_session: Optional["ClientSession"] = None,
# Per-request JSON schema validation (overrides litellm.enable_json_schema_validation)
enable_json_schema_validation: Optional[bool] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@ -1167,6 +1199,7 @@ def completion( # type: ignore # noqa: PLR0915
thinking=thinking,
web_search_options=web_search_options,
shared_session=shared_session,
enable_json_schema_validation=enable_json_schema_validation,
**kwargs,
)
api_base = kwargs.get("api_base", None)
@ -1351,27 +1384,16 @@ def completion( # type: ignore # noqa: PLR0915
timeout = float(timeout) # type: ignore
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if input_cost_per_token is not None and output_cost_per_token is not None:
if (
input_cost_per_token is not None and output_cost_per_token is not None
) or input_cost_per_second is not None:
litellm.register_model(
{
f"{custom_llm_provider}/{model}": {
"input_cost_per_token": input_cost_per_token,
"output_cost_per_token": output_cost_per_token,
"litellm_provider": custom_llm_provider,
}
}
)
elif (
input_cost_per_second is not None
): # time based pricing just needs cost in place
output_cost_per_second = output_cost_per_second
litellm.register_model(
{
f"{custom_llm_provider}/{model}": {
"input_cost_per_second": input_cost_per_second,
"output_cost_per_second": output_cost_per_second,
"litellm_provider": custom_llm_provider,
}
f"{custom_llm_provider}/{model}": _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
}
)
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
@ -4644,7 +4666,6 @@ def embedding( # noqa: PLR0915
input_cost_per_token = kwargs.get("input_cost_per_token", None)
output_cost_per_token = kwargs.get("output_cost_per_token", None)
input_cost_per_second = kwargs.get("input_cost_per_second", None)
output_cost_per_second = kwargs.get("output_cost_per_second", None)
openai_params = [
"user",
"dimensions",
@ -4694,25 +4715,16 @@ def embedding( # noqa: PLR0915
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if input_cost_per_token is not None and output_cost_per_token is not None:
if (
input_cost_per_token is not None and output_cost_per_token is not None
) or input_cost_per_second is not None:
litellm.register_model(
{
f"{custom_llm_provider}/{model}": {
"input_cost_per_token": input_cost_per_token,
"output_cost_per_token": output_cost_per_token,
"litellm_provider": custom_llm_provider,
}
}
)
if input_cost_per_second is not None: # time based pricing just needs cost in place
output_cost_per_second = output_cost_per_second or 0.0
litellm.register_model(
{
f"{custom_llm_provider}/{model}": {
"input_cost_per_second": input_cost_per_second,
"output_cost_per_second": output_cost_per_second,
"litellm_provider": custom_llm_provider,
}
f"{custom_llm_provider}/{model}": _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
}
)

501
litellm/model_prices_and_context_window_backup.json
View file

@ -9779,6 +9779,122 @@
}
]
},
"dashscope/qwen3-max-2026-01-23": {
"litellm_provider": "dashscope",
"max_input_tokens": 258048,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"tiered_pricing": [
{
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 6e-06,
"range": [
0,
32000.0
]
},
{
"input_cost_per_token": 2.4e-06,
"output_cost_per_token": 1.2e-05,
"range": [
32000.0,
128000.0
]
},
{
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"range": [
128000.0,
252000.0
]
}
]
},
"dashscope/qwen3-next-80b-a3b-instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true
},
"dashscope/qwen3-next-80b-a3b-thinking": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"dashscope/qwen3-vl-235b-a22b-instruct": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-235b-a22b-thinking": {
"input_cost_per_token": 4e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-instruct": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.4e-07,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-32b-thinking": {
"input_cost_per_token": 1.6e-07,
"litellm_provider": "dashscope",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.87e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen3-vl-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 260096,
@ -10844,7 +10960,8 @@
"output_cost_per_token": 9e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/NousResearch/Hermes-3-Llama-3.1-405B": {
"max_tokens": 131072,
@ -10854,7 +10971,8 @@
"output_cost_per_token": 1e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/NousResearch/Hermes-3-Llama-3.1-70B": {
"max_tokens": 131072,
@ -10874,7 +10992,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen2.5-72B-Instruct": {
"max_tokens": 32768,
@ -10884,7 +11003,8 @@
"output_cost_per_token": 3.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen2.5-7B-Instruct": {
"max_tokens": 32768,
@ -10905,7 +11025,8 @@
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-14B": {
"max_tokens": 40960,
@ -10915,7 +11036,8 @@
"output_cost_per_token": 2.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-235B-A22B": {
"max_tokens": 40960,
@ -10925,7 +11047,8 @@
"output_cost_per_token": 5.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"max_tokens": 262144,
@ -10935,7 +11058,8 @@
"output_cost_per_token": 6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
@ -10945,7 +11069,8 @@
"output_cost_per_token": 2.9e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-30B-A3B": {
"max_tokens": 40960,
@ -10955,7 +11080,8 @@
"output_cost_per_token": 2.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-32B": {
"max_tokens": 40960,
@ -10965,7 +11091,8 @@
"output_cost_per_token": 2.8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"max_tokens": 262144,
@ -10975,7 +11102,8 @@
"output_cost_per_token": 1.6e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": {
"max_tokens": 262144,
@ -10985,7 +11113,8 @@
"output_cost_per_token": 1.2e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": {
"max_tokens": 262144,
@ -10995,7 +11124,8 @@
"output_cost_per_token": 1.4e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Qwen/Qwen3-Next-80B-A3B-Thinking": {
"max_tokens": 262144,
@ -11005,7 +11135,8 @@
"output_cost_per_token": 1.4e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/Sao10K/L3-8B-Lunaris-v1-Turbo": {
"max_tokens": 8192,
@ -11056,7 +11187,8 @@
"cache_read_input_token_cost": 3.3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/anthropic/claude-4-opus": {
"max_tokens": 200000,
@ -11066,7 +11198,8 @@
"output_cost_per_token": 8.25e-05,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/anthropic/claude-4-sonnet": {
"max_tokens": 200000,
@ -11076,7 +11209,8 @@
"output_cost_per_token": 1.65e-05,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-R1": {
"max_tokens": 163840,
@ -11086,7 +11220,8 @@
"output_cost_per_token": 2.4e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-R1-0528": {
"max_tokens": 163840,
@ -11097,7 +11232,8 @@
"cache_read_input_token_cost": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo": {
"max_tokens": 32768,
@ -11107,7 +11243,8 @@
"output_cost_per_token": 3e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": {
"max_tokens": 131072,
@ -11127,7 +11264,8 @@
"output_cost_per_token": 2.7e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-R1-Turbo": {
"max_tokens": 40960,
@ -11137,7 +11275,8 @@
"output_cost_per_token": 3e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-V3": {
"max_tokens": 163840,
@ -11147,7 +11286,8 @@
"output_cost_per_token": 8.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-V3-0324": {
"max_tokens": 163840,
@ -11157,7 +11297,8 @@
"output_cost_per_token": 8.8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 163840,
@ -11169,7 +11310,8 @@
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true,
"supports_reasoning": true
"supports_reasoning": true,
"supports_function_calling": true
},
"deepinfra/deepseek-ai/DeepSeek-V3.1-Terminus": {
"max_tokens": 163840,
@ -11180,7 +11322,8 @@
"cache_read_input_token_cost": 2.16e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemini-2.0-flash-001": {
"deprecation_date": "2026-06-01",
@ -11191,7 +11334,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemini-2.5-flash": {
"max_tokens": 1000000,
@ -11201,7 +11345,8 @@
"output_cost_per_token": 2.5e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemini-2.5-pro": {
"max_tokens": 1000000,
@ -11211,7 +11356,8 @@
"output_cost_per_token": 1e-05,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemma-3-12b-it": {
"max_tokens": 131072,
@ -11221,7 +11367,8 @@
"output_cost_per_token": 1e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemma-3-27b-it": {
"max_tokens": 131072,
@ -11231,7 +11378,8 @@
"output_cost_per_token": 1.6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/google/gemma-3-4b-it": {
"max_tokens": 131072,
@ -11241,7 +11389,8 @@
"output_cost_per_token": 8e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Llama-3.2-11B-Vision-Instruct": {
"max_tokens": 131072,
@ -11261,7 +11410,8 @@
"output_cost_per_token": 2e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Llama-3.3-70B-Instruct": {
"max_tokens": 131072,
@ -11271,7 +11421,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": {
"max_tokens": 131072,
@ -11281,6 +11432,7 @@
"output_cost_per_token": 3.9e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true
},
"deepinfra/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
@ -11291,7 +11443,8 @@
"output_cost_per_token": 6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 327680,
@ -11301,7 +11454,8 @@
"output_cost_per_token": 3e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Llama-Guard-3-8B": {
"max_tokens": 131072,
@ -11331,7 +11485,8 @@
"output_cost_per_token": 6e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct": {
"max_tokens": 131072,
@ -11341,7 +11496,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": {
"max_tokens": 131072,
@ -11351,7 +11507,8 @@
"output_cost_per_token": 2.8e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct": {
"max_tokens": 131072,
@ -11361,7 +11518,8 @@
"output_cost_per_token": 5e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": {
"max_tokens": 131072,
@ -11371,7 +11529,8 @@
"output_cost_per_token": 3e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/microsoft/WizardLM-2-8x22B": {
"max_tokens": 65536,
@ -11391,7 +11550,8 @@
"output_cost_per_token": 1.4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/mistralai/Mistral-Nemo-Instruct-2407": {
"max_tokens": 131072,
@ -11401,7 +11561,8 @@
"output_cost_per_token": 4e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/mistralai/Mistral-Small-24B-Instruct-2501": {
"max_tokens": 32768,
@ -11411,7 +11572,8 @@
"output_cost_per_token": 8e-08,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/mistralai/Mistral-Small-3.2-24B-Instruct-2506": {
"max_tokens": 128000,
@ -11421,7 +11583,8 @@
"output_cost_per_token": 2e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1": {
"max_tokens": 32768,
@ -11431,7 +11594,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/moonshotai/Kimi-K2-Instruct": {
"max_tokens": 131072,
@ -11441,7 +11605,8 @@
"output_cost_per_token": 2e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/moonshotai/Kimi-K2-Instruct-0905": {
"max_tokens": 262144,
@ -11452,7 +11617,8 @@
"cache_read_input_token_cost": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/nvidia/Llama-3.1-Nemotron-70B-Instruct": {
"max_tokens": 131072,
@ -11462,7 +11628,8 @@
"output_cost_per_token": 6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": {
"max_tokens": 131072,
@ -11472,7 +11639,8 @@
"output_cost_per_token": 4e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": {
"max_tokens": 131072,
@ -11482,7 +11650,8 @@
"output_cost_per_token": 1.6e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/openai/gpt-oss-120b": {
"max_tokens": 131072,
@ -11492,7 +11661,8 @@
"output_cost_per_token": 4.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/openai/gpt-oss-20b": {
"max_tokens": 131072,
@ -11502,7 +11672,8 @@
"output_cost_per_token": 1.5e-07,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepinfra/zai-org/GLM-4.5": {
"max_tokens": 131072,
@ -11512,7 +11683,8 @@
"output_cost_per_token": 1.6e-06,
"litellm_provider": "deepinfra",
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_function_calling": true
},
"deepseek/deepseek-chat": {
"cache_creation_input_token_cost": 0.0,
@ -14568,13 +14740,12 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "realtime",
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_token": 2e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions"
"/vertex_ai/live"
],
"supported_modalities": [
"text",
@ -14613,14 +14784,13 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "realtime",
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_token": 2e-06,
"rpm": 100000,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions"
"/v1/realtime"
],
"supported_modalities": [
"text",
@ -20544,6 +20714,40 @@
"supports_tool_choice": true,
"supports_vision": true
},
"gpt-5.3-chat-latest": {
"cache_read_input_token_cost": 1.75e-07,
"cache_read_input_token_cost_priority": 3.5e-07,
"input_cost_per_token": 1.75e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_cost_per_token_priority": 2.8e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"gpt-5.2-pro": {
"input_cost_per_token": 2.1e-05,
"litellm_provider": "openai",
@ -23005,6 +23209,19 @@
"max_input_tokens": 200000,
"max_output_tokens": 8192
},
"mistral.devstral-2-123b": {
"input_cost_per_token": 4e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 256000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2e-06,
"supports_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"mistral.magistral-small-2509": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock_converse",
@ -25806,6 +26023,30 @@
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-sonnet-4.6": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"source": "https://openrouter.ai/anthropic/claude-sonnet-4.6",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"openrouter/anthropic/claude-opus-4.5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
@ -26156,6 +26397,39 @@
"supports_web_search": true,
"tpm": 800000
},
"openrouter/google/gemini-3.1-pro-preview": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1048576,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_200k_tokens": 1.8e-05,
"source": "https://openrouter.ai/google/gemini-3.1-pro-preview",
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/gryphe/mythomax-l2-13b": {
"input_cost_per_token": 1.875e-06,
"litellm_provider": "openrouter",
@ -26533,6 +26807,29 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/openai/gpt-5.1-codex-max": {
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"source": "https://openrouter.ai/openai/gpt-5.1-codex-max",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/openai/gpt-5.2": {
"input_cost_per_image": 0,
"cache_read_input_token_cost": 1.75e-07,
@ -26687,6 +26984,19 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"openrouter/qwen/qwen3-coder-plus": {
"input_cost_per_token": 1e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 997952,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 5e-06,
"source": "https://openrouter.ai/qwen/qwen3-coder-plus",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3-235b-a22b-2507": {
"input_cost_per_token": 7.1e-08,
"litellm_provider": "openrouter",
@ -26822,6 +27132,19 @@
"supports_vision": true,
"supports_prompt_caching": false
},
"openrouter/z-ai/glm-5": {
"input_cost_per_token": 8e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 202752,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.56e-06,
"source": "https://openrouter.ai/z-ai/glm-5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/minimax/minimax-m2.1": {
"input_cost_per_token": 2.7e-07,
"output_cost_per_token": 1.2e-06,
@ -29736,6 +30059,18 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3.5-397B-A17B": {
"input_cost_per_token": 6e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"source": "https://www.together.ai/models/Qwen/Qwen3.5-397B-A17B",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"tts-1": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "openai",
@ -34315,6 +34650,50 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai.glm-4.7-flash": {
"input_cost_per_token": 7e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-5-code": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 5e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,

786
litellm/proxy/_experimental/mcp_server/byok_oauth_endpoints.py Normal file
View file

@ -0,0 +1,786 @@
"""
BYOK (Bring Your Own Key) OAuth 2.1 Authorization Server endpoints for MCP servers.
When an MCP client connects to a BYOK-enabled server and no stored credential exists,
LiteLLM runs a minimal OAuth 2.1 authorization code flow. The "authorization page" is
just a form that asks the user for their API key — not a full identity-provider OAuth.
Endpoints implemented here:
GET /.well-known/oauth-authorization-server — OAuth authorization server metadata
GET /.well-known/oauth-protected-resource — OAuth protected resource metadata
GET /v1/mcp/oauth/authorize — Shows HTML form to collect the API key
POST /v1/mcp/oauth/authorize — Stores temp auth code and redirects
POST /v1/mcp/oauth/token — Exchanges code for a bearer JWT token
"""
import base64
import hashlib
import html as _html_module
import time
import uuid
from typing import Dict, Optional, cast
from urllib.parse import urlencode, urlparse
import jwt
from fastapi import APIRouter, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from litellm._logging import verbose_proxy_logger
from litellm.proxy._experimental.mcp_server.db import store_user_credential
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
get_request_base_url,
)
# ---------------------------------------------------------------------------
# In-memory store for pending authorization codes.
# Each entry: {code: {api_key, server_id, code_challenge, redirect_uri, user_id, expires_at}}
# ---------------------------------------------------------------------------
_byok_auth_codes: Dict[str, dict] = {}
# Authorization codes expire after 5 minutes.
_AUTH_CODE_TTL_SECONDS = 300
# Hard cap to prevent memory exhaustion from incomplete OAuth flows.
_AUTH_CODES_MAX_SIZE = 1000
router = APIRouter(tags=["mcp"])
# ---------------------------------------------------------------------------
# PKCE helper
# ---------------------------------------------------------------------------
def _verify_pkce(code_verifier: str, code_challenge: str) -> bool:
"""Return True iff SHA-256(code_verifier) == code_challenge (base64url, no padding)."""
digest = hashlib.sha256(code_verifier.encode()).digest()
computed = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
return computed == code_challenge
# ---------------------------------------------------------------------------
# Cleanup of expired auth codes (called lazily on each request)
# ---------------------------------------------------------------------------
def _purge_expired_codes() -> None:
now = time.time()
expired = [k for k, v in _byok_auth_codes.items() if v["expires_at"] < now]
for k in expired:
del _byok_auth_codes[k]
def _build_authorize_html(
server_name: str,
server_initial: str,
client_id: str,
redirect_uri: str,
code_challenge: str,
code_challenge_method: str,
state: str,
server_id: str,
access_items: list,
help_url: str,
) -> str:
"""Build the 2-step BYOK OAuth authorization page HTML."""
# Escape all user-supplied / externally-derived values before interpolation
e = _html_module.escape
server_name = e(server_name)
server_initial = e(server_initial)
client_id = e(client_id)
redirect_uri = e(redirect_uri)
code_challenge = e(code_challenge)
code_challenge_method = e(code_challenge_method)
state = e(state)
server_id = e(server_id)
# Build access checklist rows
access_rows = "".join(
f'<div class="access-item"><span class="check">&#10003;</span>{e(item)}</div>'
for item in access_items
)
access_section = ""
if access_rows:
access_section = f"""
<div class="access-box">
<div class="access-header">
<span class="shield">&#9646;</span>
<span>Requested Access</span>
</div>
{access_rows}
</div>"""
# Help link for step 2
help_link_html = ""
if help_url:
help_link_html = f'<a class="help-link" href="{e(help_url)}" target="_blank">Where do I find my API key? &#8599;</a>'
return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Connect {server_name} &mdash; LiteLLM</title>
<style>
*, *::before, *::after {{ box-sizing: border-box; margin: 0; padding: 0; }}
body {{
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: #0f172a;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
padding: 24px;
}}
.modal {{
background: #ffffff;
border-radius: 20px;
padding: 36px 32px 32px;
width: 440px;
max-width: 100%;
position: relative;
box-shadow: 0 25px 60px rgba(0,0,0,0.35);
}}
/* Progress dots */
.dots {{
display: flex;
justify-content: center;
gap: 7px;
margin-bottom: 28px;
}}
.dot {{
width: 8px; height: 8px;
border-radius: 50%;
background: #e2e8f0;
}}
.dot.active {{ background: #38bdf8; }}
/* Close button */
.close-btn {{
position: absolute;
top: 16px; right: 16px;
background: none; border: none;
font-size: 16px; color: #94a3b8;
cursor: pointer; line-height: 1;
width: 28px; height: 28px;
border-radius: 6px;
display: flex; align-items: center; justify-content: center;
}}
.close-btn:hover {{ background: #f1f5f9; color: #475569; }}
/* Logo pair */
.logos {{
display: flex; align-items: center; justify-content: center;
gap: 12px; margin-bottom: 20px;
}}
.logo {{
width: 52px; height: 52px;
border-radius: 14px;
display: flex; align-items: center; justify-content: center;
font-size: 22px; font-weight: 800; color: white;
}}
.logo-img {{
width: 52px; height: 52px;
border-radius: 14px;
object-fit: cover;
border: 1.5px solid #e2e8f0;
}}
.logo-s {{ background: linear-gradient(135deg, #818cf8 0%, #4f46e5 100%); }}
.logo-arrow {{ color: #cbd5e1; font-size: 20px; font-weight: 300; }}
/* Headings */
.step-title {{
text-align: center;
font-size: 21px; font-weight: 700;
color: #0f172a; margin-bottom: 8px;
}}
.step-subtitle {{
text-align: center;
font-size: 14px; color: #64748b;
line-height: 1.55; margin-bottom: 22px;
}}
/* Info box */
.info-box {{
background: #f8fafc;
border-radius: 12px;
padding: 14px 16px;
display: flex; gap: 12px;
margin-bottom: 14px;
}}
.info-icon {{ font-size: 17px; flex-shrink: 0; margin-top: 1px; color: #38bdf8; }}
.info-box h4 {{ font-size: 13px; font-weight: 600; color: #1e293b; margin-bottom: 4px; }}
.info-box p {{ font-size: 13px; color: #64748b; line-height: 1.5; }}
/* Access checklist */
.access-box {{
background: #f8fafc;
border-radius: 12px;
padding: 14px 16px;
margin-bottom: 22px;
}}
.access-header {{
display: flex; align-items: center; gap: 8px;
margin-bottom: 10px;
}}
.shield {{ color: #22c55e; font-size: 15px; }}
.access-header > span:last-child {{
font-size: 11px; font-weight: 700;
letter-spacing: 0.07em;
text-transform: uppercase;
color: #475569;
}}
.access-item {{
display: flex; align-items: center; gap: 9px;
font-size: 13.5px; color: #374151;
padding: 3px 0;
}}
.check {{ color: #22c55e; font-weight: 700; font-size: 13px; }}
/* Primary CTA */
.btn-primary {{
width: 100%; padding: 15px;
background: #0f172a; color: white;
border: none; border-radius: 12px;
font-size: 15px; font-weight: 600;
cursor: pointer; margin-bottom: 10px;
}}
.btn-primary:hover {{ background: #1e293b; }}
.btn-cancel {{
width: 100%; padding: 8px;
background: none; border: none;
font-size: 13.5px; color: #94a3b8;
cursor: pointer;
}}
.btn-cancel:hover {{ color: #64748b; }}
/* Step 2 nav */
.step2-nav {{
display: flex; align-items: center;
justify-content: space-between;
margin-bottom: 24px;
}}
.back-btn {{
background: none; border: none;
font-size: 13.5px; color: #64748b;
cursor: pointer; display: flex; align-items: center; gap: 4px;
}}
.back-btn:hover {{ color: #374151; }}
/* Key icon */
.key-icon-wrap {{
width: 46px; height: 46px;
background: #e0f2fe;
border-radius: 12px;
display: flex; align-items: center; justify-content: center;
margin-bottom: 14px;
}}
.key-icon-wrap svg {{ width: 22px; height: 22px; color: #0284c7; }}
/* Form elements */
.field-label {{
font-size: 13.5px; font-weight: 600;
color: #1e293b; display: block;
margin-bottom: 7px;
}}
.key-input {{
width: 100%; padding: 11px 13px;
border: 1.5px solid #e2e8f0;
border-radius: 10px;
font-size: 14px; color: #0f172a;
outline: none; transition: border-color 0.15s, box-shadow 0.15s;
}}
.key-input:focus {{
border-color: #38bdf8;
box-shadow: 0 0 0 3px rgba(56,189,248,0.12);
}}
.help-link {{
display: inline-flex; align-items: center; gap: 4px;
color: #0ea5e9; font-size: 13px;
text-decoration: none; margin: 8px 0 16px;
}}
.help-link:hover {{ text-decoration: underline; }}
/* Save toggle card */
.save-card {{
border: 1.5px solid #e2e8f0;
border-radius: 12px;
padding: 13px 15px;
margin-bottom: 6px;
}}
.save-row {{
display: flex; align-items: center; gap: 10px;
}}
.save-icon {{ font-size: 16px; }}
.save-label {{
flex: 1;
font-size: 14px; font-weight: 500; color: #1e293b;
}}
/* Toggle switch */
.toggle {{ position: relative; width: 44px; height: 24px; flex-shrink: 0; }}
.toggle input {{ opacity: 0; width: 0; height: 0; }}
.slider {{
position: absolute; inset: 0;
background: #e2e8f0;
border-radius: 24px; cursor: pointer;
transition: background 0.18s;
}}
.slider::before {{
content: '';
position: absolute;
width: 18px; height: 18px;
left: 3px; bottom: 3px;
background: white;
border-radius: 50%;
transition: transform 0.18s;
box-shadow: 0 1px 3px rgba(0,0,0,0.18);
}}
input:checked + .slider {{ background: #38bdf8; }}
input:checked + .slider::before {{ transform: translateX(20px); }}
/* Duration pills */
.duration-section {{ margin-top: 14px; }}
.duration-label {{
font-size: 12px; font-weight: 600;
color: #64748b; margin-bottom: 8px;
text-transform: uppercase; letter-spacing: 0.05em;
}}
.pills {{ display: flex; flex-wrap: wrap; gap: 7px; }}
.pill {{
padding: 6px 13px;
border: 1.5px solid #e2e8f0;
border-radius: 20px;
font-size: 13px; color: #475569;
cursor: pointer; background: white;
transition: all 0.13s;
user-select: none;
}}
.pill:hover {{ border-color: #94a3b8; }}
.pill.sel {{
border-color: #38bdf8;
color: #0284c7;
background: #e0f2fe;
}}
/* Security note */
.sec-note {{
background: #f8fafc;
border-radius: 10px;
padding: 11px 14px;
display: flex; gap: 9px; align-items: flex-start;
margin: 16px 0;
}}
.sec-icon {{ font-size: 13px; color: #94a3b8; margin-top: 1px; flex-shrink: 0; }}
.sec-note p {{ font-size: 12.5px; color: #64748b; line-height: 1.5; }}
/* Connect button */
.btn-connect {{
width: 100%; padding: 15px;
border: none; border-radius: 12px;
font-size: 15px; font-weight: 600;
cursor: pointer;
background: #bae6fd; color: #0369a1;
transition: background 0.15s, color 0.15s;
}}
.btn-connect.ready {{
background: #0ea5e9; color: white;
}}
.btn-connect.ready:hover {{ background: #0284c7; }}
/* Step visibility */
.step {{ display: none; }}
.step.show {{ display: block; }}
</style>
</head>
<body>
<div class="modal">
<!-- ── STEP 1: Connect ─────────────────────────────────────── -->
<div id="s1" class="step show">
<div class="dots">
<div class="dot active"></div>
<div class="dot"></div>
</div>
<button class="close-btn" type="button" onclick="doCancel()" title="Close">&times;</button>
<div class="logos">
<img src="/ui/assets/logos/litellm_logo.jpg" class="logo-img" alt="LiteLLM">
<span class="logo-arrow">&#8594;</span>
<div class="logo logo-s">{server_initial}</div>
</div>
<h2 class="step-title">Connect {server_name} MCP</h2>
<p class="step-subtitle">LiteLLM needs access to {server_name} to complete your request.</p>
<div class="info-box">
<span class="info-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="12"/><line x1="12" y1="16" x2="12.01" y2="16"/></svg>
</span>
<div>
<h4>How it works</h4>
<p>LiteLLM acts as a secure bridge. Your requests are routed through our MCP client directly to {server_name}&rsquo;s API.</p>
</div>
</div>
{access_section}
<button class="btn-primary" type="button" onclick="goStep2()">
Continue to Authentication &rarr;
</button>
<button class="btn-cancel" type="button" onclick="doCancel()">Cancel</button>
</div>
<!-- ── STEP 2: Provide API Key ──────────────────────────────── -->
<div id="s2" class="step">
<div class="step2-nav">
<button class="back-btn" type="button" onclick="goStep1()">&#8592; Back</button>
<div class="dots">
<div class="dot active"></div>
<div class="dot active"></div>
</div>
<button class="close-btn" style="position:static;" type="button" onclick="doCancel()" title="Close">&times;</button>
</div>
<div class="key-icon-wrap">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="#0284c7" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M21 2l-2 2m-7.61 7.61a5.5 5.5 0 1 1-7.778 7.778 5.5 5.5 0 0 1 7.777-7.777zm0 0L15.5 7.5m0 0l3 3L22 7l-3-3m-3.5 3.5L19 4"/></svg>
</div>
<h2 class="step-title" style="text-align:left;">Provide API Key</h2>
<p class="step-subtitle" style="text-align:left;">Enter your {server_name} API key to authorize this connection.</p>
<form method="POST" id="authForm" onsubmit="prepareSubmit()">
<input type="hidden" name="client_id" value="{client_id}">
<input type="hidden" name="redirect_uri" value="{redirect_uri}">
<input type="hidden" name="code_challenge" value="{code_challenge}">
<input type="hidden" name="code_challenge_method" value="{code_challenge_method}">
<input type="hidden" name="state" value="{state}">
<input type="hidden" name="server_id" value="{server_id}">
<input type="hidden" name="duration" id="durInput" value="until_revoked">
<label class="field-label">{server_name} API Key</label>
<input
type="password"
name="api_key"
id="apiKey"
class="key-input"
placeholder="Enter your API key"
required
autofocus
oninput="syncBtn()"
>
{help_link_html}
<div class="save-card">
<div class="save-row">
<span class="save-label">Save key for future use</span>
<label class="toggle">
<input type="checkbox" id="saveToggle" onchange="toggleDur()">
<span class="slider"></span>
</label>
</div>
<div id="durSection" class="duration-section" style="display:none;">
<div class="duration-label">Duration</div>
<div class="pills">
<div class="pill" onclick="selDur('1h',this)">1 hour</div>
<div class="pill sel" onclick="selDur('24h',this)">24 hours</div>
<div class="pill" onclick="selDur('7d',this)">7 days</div>
<div class="pill" onclick="selDur('30d',this)">30 days</div>
<div class="pill" onclick="selDur('until_revoked',this)">Until I revoke</div>
</div>
</div>
</div>
<div class="sec-note">
<span class="sec-icon">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="11" width="18" height="11" rx="2" ry="2"/><path d="M7 11V7a5 5 0 0 1 10 0v4"/></svg>
</span>
<p>Your key is stored securely and transmitted over HTTPS. It is never shared with third parties.</p>
</div>
<button type="submit" class="btn-connect" id="connectBtn">
Connect &amp; Authorize
</button>
</form>
</div>
</div>
<script>
function goStep2() {{
document.getElementById('s1').classList.remove('show');
document.getElementById('s2').classList.add('show');
}}
function goStep1() {{
document.getElementById('s2').classList.remove('show');
document.getElementById('s1').classList.add('show');
}}
function doCancel() {{
if (window.opener) window.close();
else window.history.back();
}}
function toggleDur() {{
const on = document.getElementById('saveToggle').checked;
document.getElementById('durSection').style.display = on ? 'block' : 'none';
}}
function selDur(val, el) {{
document.querySelectorAll('.pill').forEach(p => p.classList.remove('sel'));
el.classList.add('sel');
document.getElementById('durInput').value = val;
}}
function syncBtn() {{
const btn = document.getElementById('connectBtn');
if (document.getElementById('apiKey').value.length > 0) {{
btn.classList.add('ready');
}} else {{
btn.classList.remove('ready');
}}
}}
function prepareSubmit() {{
// nothing extra needed — duration is already in the hidden input
}}
</script>
</body>
</html>"""
# ---------------------------------------------------------------------------
# OAuth metadata discovery endpoints
# ---------------------------------------------------------------------------
@router.get("/.well-known/oauth-authorization-server", include_in_schema=False)
async def oauth_authorization_server_metadata(request: Request) -> JSONResponse:
"""RFC 8414 Authorization Server Metadata for the BYOK OAuth flow."""
base_url = get_request_base_url(request)
return JSONResponse(
{
"issuer": base_url,
"authorization_endpoint": f"{base_url}/v1/mcp/oauth/authorize",
"token_endpoint": f"{base_url}/v1/mcp/oauth/token",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code"],
"code_challenge_methods_supported": ["S256"],
}
)
@router.get("/.well-known/oauth-protected-resource", include_in_schema=False)
async def oauth_protected_resource_metadata(request: Request) -> JSONResponse:
"""RFC 9728 Protected Resource Metadata pointing back at this server."""
base_url = get_request_base_url(request)
return JSONResponse(
{
"resource": base_url,
"authorization_servers": [base_url],
}
)
# ---------------------------------------------------------------------------
# Authorization endpoint — GET (show form) and POST (process form)
# ---------------------------------------------------------------------------
@router.get("/v1/mcp/oauth/authorize", include_in_schema=False)
async def byok_authorize_get(
request: Request,
client_id: Optional[str] = None,
redirect_uri: Optional[str] = None,
response_type: Optional[str] = None,
code_challenge: Optional[str] = None,
code_challenge_method: Optional[str] = None,
state: Optional[str] = None,
server_id: Optional[str] = None,
) -> HTMLResponse:
"""
Show the BYOK API-key entry form.
The MCP client navigates the user here; the user types their API key and
clicks "Connect & Authorize", which POSTs back to this same path.
"""
if response_type != "code":
raise HTTPException(status_code=400, detail="response_type must be 'code'")
if not redirect_uri:
raise HTTPException(status_code=400, detail="redirect_uri is required")
if not code_challenge:
raise HTTPException(status_code=400, detail="code_challenge is required")
# Resolve server metadata (name, description items, help URL).
server_name = "MCP Server"
access_items: list = []
help_url = ""
if server_id:
try:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
registry = global_mcp_server_manager.get_registry()
if server_id in registry:
srv = registry[server_id]
server_name = srv.server_name or srv.name
access_items = list(srv.byok_description or [])
help_url = srv.byok_api_key_help_url or ""
except Exception:
pass
server_initial = (server_name[0].upper()) if server_name else "S"
html = _build_authorize_html(
server_name=server_name,
server_initial=server_initial,
client_id=client_id or "",
redirect_uri=redirect_uri,
code_challenge=code_challenge,
code_challenge_method=code_challenge_method or "S256",
state=state or "",
server_id=server_id or "",
access_items=access_items,
help_url=help_url,
)
return HTMLResponse(content=html)
@router.post("/v1/mcp/oauth/authorize", include_in_schema=False)
async def byok_authorize_post(
request: Request,
client_id: str = Form(default=""),
redirect_uri: str = Form(...),
code_challenge: str = Form(...),
code_challenge_method: str = Form(default="S256"),
state: str = Form(default=""),
server_id: str = Form(default=""),
api_key: str = Form(...),
) -> RedirectResponse:
"""
Process the BYOK API-key form submission.
Stores a short-lived authorization code and redirects the client back to
redirect_uri with ?code=...&state=... query parameters.
"""
_purge_expired_codes()
# Validate redirect_uri scheme to prevent open redirect
parsed_uri = urlparse(redirect_uri)
if parsed_uri.scheme not in ("http", "https"):
raise HTTPException(status_code=400, detail="Invalid redirect_uri scheme")
# Reject new codes if the store is at capacity (prevents memory exhaustion
# from a burst of abandoned OAuth flows).
if len(_byok_auth_codes) >= _AUTH_CODES_MAX_SIZE:
raise HTTPException(status_code=503, detail="Too many pending authorization flows")
if code_challenge_method != "S256":
raise HTTPException(
status_code=400, detail="Only S256 code_challenge_method is supported"
)
auth_code = str(uuid.uuid4())
_byok_auth_codes[auth_code] = {
"api_key": api_key,
"server_id": server_id,
"code_challenge": code_challenge,
"redirect_uri": redirect_uri,
"user_id": client_id, # external client passes LiteLLM user-id as client_id
"expires_at": time.time() + _AUTH_CODE_TTL_SECONDS,
}
params = urlencode({"code": auth_code, "state": state})
separator = "&" if "?" in redirect_uri else "?"
location = f"{redirect_uri}{separator}{params}"
return RedirectResponse(url=location, status_code=302)
# ---------------------------------------------------------------------------
# Token endpoint
# ---------------------------------------------------------------------------
@router.post("/v1/mcp/oauth/token", include_in_schema=False)
async def byok_token(
request: Request,
grant_type: str = Form(...),
code: str = Form(...),
redirect_uri: str = Form(default=""),
code_verifier: str = Form(...),
client_id: str = Form(default=""),
) -> JSONResponse:
"""
Exchange an authorization code for a short-lived BYOK session JWT.
1. Validates the authorization code and PKCE challenge.
2. Stores the API key via store_user_credential().
3. Issues a signed JWT with type="byok_session".
"""
from litellm.proxy.proxy_server import master_key, prisma_client
_purge_expired_codes()
if grant_type != "authorization_code":
raise HTTPException(status_code=400, detail="unsupported_grant_type")
record = _byok_auth_codes.get(code)
if record is None:
raise HTTPException(status_code=400, detail="invalid_grant")
if time.time() > record["expires_at"]:
del _byok_auth_codes[code]
raise HTTPException(status_code=400, detail="invalid_grant")
# PKCE verification
if not _verify_pkce(code_verifier, record["code_challenge"]):
raise HTTPException(status_code=400, detail="invalid_grant")
# Consume the code (one-time use)
del _byok_auth_codes[code]
server_id: str = record["server_id"]
api_key_value: str = record["api_key"]
# Prefer the user_id that was stored when the code was issued; fall back to
# whatever client_id the token request supplies (they should match).
user_id: str = record.get("user_id") or client_id
if not user_id:
raise HTTPException(
status_code=400,
detail="Cannot determine user_id; pass LiteLLM user id as client_id",
)
# Persist the BYOK credential
if prisma_client is not None:
try:
await store_user_credential(
prisma_client=prisma_client,
user_id=user_id,
server_id=server_id,
credential=api_key_value,
)
# Invalidate any cached negative result so the user isn't blocked
# for up to the TTL period after completing the OAuth flow.
from litellm.proxy._experimental.mcp_server.server import (
_invalidate_byok_cred_cache,
)
_invalidate_byok_cred_cache(user_id, server_id)
except Exception as exc:
verbose_proxy_logger.error(
"byok_token: failed to store user credential for user=%s server=%s: %s",
user_id,
server_id,
exc,
)
raise HTTPException(status_code=500, detail="Failed to store credential")
else:
verbose_proxy_logger.warning(
"byok_token: prisma_client is None — credential not persisted"
)
if master_key is None:
raise HTTPException(
status_code=500, detail="Master key not configured; cannot issue token"
)
now = int(time.time())
payload = {
"user_id": user_id,
"server_id": server_id,
# "type" distinguishes this from regular proxy auth tokens.
# The proxy's SSO JWT path uses asymmetric keys (RS256/ES256), so an
# HS256 token signed with master_key cannot be accepted there.
"type": "byok_session",
"iat": now,
"exp": now + 3600,
}
access_token = jwt.encode(payload, cast(str, master_key), algorithm="HS256")
return JSONResponse(
{
"access_token": access_token,
"token_type": "bearer",
"expires_in": 3600,
}
)

82
litellm/proxy/_experimental/mcp_server/db.py
View file

@ -13,6 +13,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
_get_salt_key,
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.utils import PrismaClient
@ -60,8 +61,18 @@ def _prepare_mcp_server_data(
if data.env is not None:
data_dict["env"] = safe_dumps(data.env)
# Handle tool name override serialization
if data.tool_name_to_display_name is not None:
data_dict["tool_name_to_display_name"] = safe_dumps(data.tool_name_to_display_name)
if data.tool_name_to_description is not None:
data_dict["tool_name_to_description"] = safe_dumps(data.tool_name_to_description)
# mcp_access_groups is already List[str], no serialization needed
# Force include is_byok even when False (exclude_none=True would not drop it,
# but be explicit to ensure a False value is always written to the DB).
data_dict["is_byok"] = getattr(data, "is_byok", False)
return data_dict
@ -369,3 +380,74 @@ async def rotate_mcp_server_credentials_master_key(
"updated_by": touched_by,
},
)
async def store_user_credential(
prisma_client: PrismaClient,
user_id: str,
server_id: str,
credential: str,
) -> None:
"""Store a user credential for a BYOK MCP server."""
import base64
encoded = base64.urlsafe_b64encode(credential.encode()).decode()
await prisma_client.db.litellm_mcpusercredentials.upsert(
where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}},
data={
"create": {
"user_id": user_id,
"server_id": server_id,
"credential_b64": encoded,
},
"update": {"credential_b64": encoded},
},
)
async def get_user_credential(
prisma_client: PrismaClient,
user_id: str,
server_id: str,
) -> Optional[str]:
"""Return credential for a user+server pair, or None."""
import base64
row = await prisma_client.db.litellm_mcpusercredentials.find_unique(
where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}
)
if row is None:
return None
try:
return base64.urlsafe_b64decode(row.credential_b64).decode()
except Exception:
# Fall back to nacl decryption for credentials stored by older code
return decrypt_value_helper(
value=row.credential_b64,
key="byok_credential",
exception_type="debug",
return_original_value=False,
)
async def has_user_credential(
prisma_client: PrismaClient,
user_id: str,
server_id: str,
) -> bool:
"""Return True if the user has a stored credential for this server."""
row = await prisma_client.db.litellm_mcpusercredentials.find_unique(
where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}
)
return row is not None
async def delete_user_credential(
prisma_client: PrismaClient,
user_id: str,
server_id: str,
) -> None:
"""Delete the user's stored credential for a BYOK MCP server."""
await prisma_client.db.litellm_mcpusercredentials.delete(
where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}
)

12
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -644,6 +644,15 @@ class MCPServerManager:
),
created_at=getattr(mcp_server, "created_at", None),
updated_at=getattr(mcp_server, "updated_at", None),
tool_name_to_display_name=_deserialize_json_dict(
getattr(mcp_server, "tool_name_to_display_name", None)
),
tool_name_to_description=_deserialize_json_dict(
getattr(mcp_server, "tool_name_to_description", None)
),
is_byok=bool(getattr(mcp_server, "is_byok", False)),
byok_description=getattr(mcp_server, "byok_description", None) or [],
byok_api_key_help_url=getattr(mcp_server, "byok_api_key_help_url", None),
)
return new_server
@ -2651,6 +2660,9 @@ class MCPServerManager:
registration_url=server.registration_url,
allow_all_keys=server.allow_all_keys,
available_on_public_internet=server.available_on_public_internet,
is_byok=server.is_byok,
byok_description=server.byok_description,
byok_api_key_help_url=server.byok_api_key_help_url,
)
async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]:

27
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -3,6 +3,7 @@ This module is used to generate MCP tools from OpenAPI specs.
"""
import asyncio
import contextvars
import json
import os
from pathlib import PurePosixPath
@ -22,6 +23,13 @@ from litellm.proxy._experimental.mcp_server.tool_registry import (
BASE_URL = ""
HEADERS: Dict[str, str] = {}
# Per-request auth header override for BYOK servers.
# Set this ContextVar before calling a local tool handler to inject the user's
# stored credential into the HTTP request made by the tool function closure.
_request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar(
"_request_auth_header", default=None
)
def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
"""Ensure path params cannot introduce directory traversal."""
@ -211,6 +219,15 @@ def create_tool_function(
The function safely handles parameter names that aren't valid Python identifiers
by using **kwargs instead of named parameters.
"""
# Allow per-request auth override (e.g. BYOK credential set via ContextVar).
# The ContextVar holds the full Authorization header value, including the
# correct prefix (Bearer / ApiKey / Basic) formatted by the caller in
# server.py based on the server's configured auth_type.
effective_headers = dict(headers)
override_auth = _request_auth_header.get()
if override_auth:
effective_headers["Authorization"] = override_auth
# Build URL from base_url and path
url = base_url + path
@ -263,20 +280,20 @@ def create_tool_function(
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
if original_method == "get":
response = await client.get(url, params=params, headers=headers)
response = await client.get(url, params=params, headers=effective_headers)
elif original_method == "post":
response = await client.post(
url, params=params, json=json_body, headers=headers
url, params=params, json=json_body, headers=effective_headers
)
elif original_method == "put":
response = await client.put(
url, params=params, json=json_body, headers=headers
url, params=params, json=json_body, headers=effective_headers
)
elif original_method == "delete":
response = await client.delete(url, params=params, headers=headers)
response = await client.delete(url, params=params, headers=effective_headers)
elif original_method == "patch":
response = await client.patch(
url, params=params, json=json_body, headers=headers
url, params=params, json=json_body, headers=effective_headers
)
else:
return f"Unsupported HTTP method: {original_method}"

336
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -5,7 +5,7 @@ LiteLLM MCP Server Routes
import asyncio
import contextlib
import time
import traceback
import uuid
from datetime import datetime
@ -41,15 +41,46 @@ from litellm.proxy._experimental.mcp_server.utils import (
LITELLM_MCP_SERVER_DESCRIPTION,
LITELLM_MCP_SERVER_NAME,
LITELLM_MCP_SERVER_VERSION,
add_server_prefix_to_name,
get_server_prefix,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
get_chain_id_from_headers,
)
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall
from litellm.utils import Rules, client, function_setup
# Short-lived in-memory cache for BYOK credentials.
# Keyed by (user_id, server_id); value is (credential_or_None, monotonic_timestamp).
# Storing the credential value (not just a bool) means _get_byok_credential and
# _check_byok_credential share a single DB round-trip per TTL window.
_byok_cred_cache: Dict[Tuple[str, str], Tuple[Optional[str], float]] = {}
_BYOK_CRED_CACHE_TTL = 60 # seconds
_BYOK_CRED_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth
def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None:
"""Remove a (user_id, server_id) entry from the BYOK credential cache.
Call this after storing or deleting a credential so subsequent calls
see the fresh value rather than a stale cached result.
"""
_byok_cred_cache.pop((user_id, server_id), None)
def _write_byok_cred_cache(
user_id: str, server_id: str, credential: Optional[str]
) -> None:
"""Write a credential value to the cache, evicting all entries if at capacity."""
if len(_byok_cred_cache) >= _BYOK_CRED_CACHE_MAX_SIZE:
_byok_cred_cache.clear()
_byok_cred_cache[(user_id, server_id)] = (credential, time.monotonic())
# Check if MCP is available
# "mcp" requires python 3.10 or higher, but several litellm users use python 3.8
# We're making this conditional import to avoid breaking users who use python 3.8.
@ -114,6 +145,9 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
_request_auth_header,
)
from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
@ -331,6 +365,11 @@ if MCP_AVAILABLE:
try:
# Create a body date for logging
body_data = {"name": name, "arguments": arguments}
# Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A)
chain_id = get_chain_id_from_headers(raw_headers)
if chain_id:
body_data["litellm_trace_id"] = chain_id
body_data["litellm_session_id"] = chain_id
request = Request(
scope={
@ -730,6 +769,29 @@ if MCP_AVAILABLE:
return tools_to_return
def apply_tool_overrides(
tools: List[MCPTool],
mcp_server: MCPServer,
) -> List[MCPTool]:
"""Apply admin-configured display name/description overrides to tools.
Overrides are keyed by the unprefixed tool name, same convention as
allowed_tools configuration.
"""
display_name_map = mcp_server.tool_name_to_display_name or {}
description_map = mcp_server.tool_name_to_description or {}
if not display_name_map and not description_map:
return tools
for tool in tools:
unprefixed, _ = split_server_prefix_from_name(tool.name)
lookup_key = unprefixed or tool.name
if lookup_key in display_name_map:
tool.name = display_name_map[lookup_key]
if lookup_key in description_map:
tool.description = description_map[lookup_key]
return tools
def _get_client_ip_from_context() -> Optional[str]:
"""
Extract client_ip from auth context.
@ -884,6 +946,10 @@ if MCP_AVAILABLE:
# This is intentionally minimal: only async_success_handler / post_call_failure_hook
rules_obj = Rules()
list_tools_call_id = str(uuid.uuid4())
# Derive trace_id from raw_headers when not explicitly passed (same as A2A / MCP call_tool)
effective_litellm_trace_id = litellm_trace_id or get_chain_id_from_headers(
raw_headers
)
spend_logs_metadata: Dict[str, Any] = {
"mcp_operation": "list_tools",
}
@ -896,7 +962,7 @@ if MCP_AVAILABLE:
"model": "MCP: list_tools",
"call_type": CallTypes.list_mcp_tools.value,
"litellm_call_id": list_tools_call_id,
"litellm_trace_id": litellm_trace_id,
"litellm_trace_id": effective_litellm_trace_id,
"metadata": {
"spend_logs_metadata": spend_logs_metadata,
},
@ -980,6 +1046,10 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
)
# Apply display-name/description overrides last so that
# permission filtering always works against original names.
filtered_tools = apply_tool_overrides(filtered_tools, server)
verbose_logger.debug(
f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering"
)
@ -1438,6 +1508,142 @@ if MCP_AVAILABLE:
return managed_resource_templates
def _resolve_display_name_to_original(
name: str,
allowed_mcp_servers: List[MCPServer],
) -> str:
"""Translate a display-name override back to the original prefixed tool name.
When a client received a customised display name from tools/list (e.g.
"Get Pet") it will call tools/call with that same string. We need to
reverse-map it to the original prefixed name (e.g.
"petstore_mcp-getPetById") before any routing or permission logic runs.
"""
for server in allowed_mcp_servers:
display_map = server.tool_name_to_display_name or {}
for unprefixed_name, display_name in display_map.items():
if display_name == name:
return add_server_prefix_to_name(
unprefixed_name, get_server_prefix(server)
)
return name
async def _get_byok_credential(
mcp_server: MCPServer,
user_api_key_auth: Optional[UserAPIKeyAuth],
) -> Optional[str]:
"""Retrieve the stored BYOK credential for a user+server pair.
Uses the shared _byok_cred_cache to avoid a DB round-trip on every
tool call within the TTL window.
"""
if not mcp_server.is_byok:
return None
user_id = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
if not user_id:
return None
cache_key = (user_id, mcp_server.server_id)
cached = _byok_cred_cache.get(cache_key)
if cached is not None:
credential, ts = cached
if time.monotonic() - ts < _BYOK_CRED_CACHE_TTL:
return credential
from litellm.proxy._experimental.mcp_server.db import get_user_credential
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return None
credential = await get_user_credential(
prisma_client=prisma_client,
user_id=user_id,
server_id=mcp_server.server_id,
)
_write_byok_cred_cache(user_id, mcp_server.server_id, credential)
return credential
async def _check_byok_credential(
mcp_server: MCPServer,
user_api_key_auth: Optional[UserAPIKeyAuth],
) -> None:
"""
If the MCP server is BYOK-enabled, verify that the requesting user has a
stored credential. When no credential is found, raise an HTTP 401 with a
WWW-Authenticate header that points the MCP client to our OAuth metadata
endpoint so it can drive the authorization flow.
"""
if not mcp_server.is_byok:
return
user_id = (user_api_key_auth.user_id if user_api_key_auth else None) or ""
if not user_id:
raise HTTPException(
status_code=401,
detail={
"error": "byok_auth_required",
"server_id": mcp_server.server_id,
"server_name": mcp_server.server_name or mcp_server.name,
"message": "User identity is required for BYOK servers",
},
headers={
"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'
},
)
# Check shared credential cache before hitting the DB.
cache_key = (user_id, mcp_server.server_id)
cached = _byok_cred_cache.get(cache_key)
if cached is not None:
cached_cred, ts = cached
if time.monotonic() - ts < _BYOK_CRED_CACHE_TTL:
if cached_cred is None:
raise HTTPException(
status_code=401,
detail={
"error": "byok_auth_required",
"server_id": mcp_server.server_id,
"server_name": mcp_server.server_name or mcp_server.name,
"message": (
"No stored credential found for this BYOK server. "
"Complete the OAuth authorization flow to provide your API key."
),
},
headers={
"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'
},
)
return
from litellm.proxy._experimental.mcp_server.db import get_user_credential
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return
credential = await get_user_credential(
prisma_client=prisma_client,
user_id=user_id,
server_id=mcp_server.server_id,
)
_write_byok_cred_cache(user_id, mcp_server.server_id, credential)
if credential is None:
raise HTTPException(
status_code=401,
detail={
"error": "byok_auth_required",
"server_id": mcp_server.server_id,
"server_name": mcp_server.server_name or mcp_server.name,
"message": (
"No stored credential found for this BYOK server. "
"Complete the OAuth authorization flow to provide your API key."
),
},
headers={
"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'
},
)
async def execute_mcp_tool(
name: str,
arguments: Dict[str, Any],
@ -1474,6 +1680,10 @@ if MCP_AVAILABLE:
# Track resolved MCP server for both permission checks and dispatch
mcp_server: Optional[MCPServer] = None
# If the client called with a display-name override (e.g. "Get Pet"),
# translate it back to the original prefixed name before any routing.
name = _resolve_display_name_to_original(name, allowed_mcp_servers)
# Remove prefix from tool name for logging and processing
original_tool_name, server_name = split_server_prefix_from_name(name)
@ -1509,57 +1719,99 @@ if MCP_AVAILABLE:
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model = f"MCP: {name}"
# Resolve the MCP server early so BYOK checks and credential injection
# apply to ALL dispatch paths (local tool registry AND managed MCP server).
if mcp_server is None:
mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
if litellm_logging_obj:
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
# BYOK: retrieve the stored per-user credential. A single DB call
# both checks existence and fetches the value, avoiding a double query.
if mcp_server.is_byok and not mcp_auth_header:
byok_cred = await _get_byok_credential(mcp_server, user_api_key_auth)
if byok_cred is None:
raise HTTPException(
status_code=401,
detail={
"error": "byok_auth_required",
"server_id": mcp_server.server_id,
"server_name": mcp_server.server_name or mcp_server.name,
"message": (
"No stored credential found for this BYOK server. "
"Complete the OAuth authorization flow to provide your API key."
),
},
headers={
"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'
},
)
mcp_auth_header = byok_cred
elif mcp_server.is_byok:
# External auth header supplied; still enforce user-identity check.
await _check_byok_credential(mcp_server, user_api_key_auth)
# Check if tool exists in local registry first (for OpenAPI-based tools)
# These tools are registered with their prefixed names
#########################################################
local_tool = global_mcp_tool_registry.get_tool(name)
if local_tool:
verbose_logger.debug(f"Executing local registry tool: {name}")
local_content = await _handle_local_mcp_tool(name, arguments)
# For BYOK servers the credential must be injected via a ContextVar
# because the tool function has headers baked into its closure.
# Pre-format the full Authorization header value using the server's
# configured auth_type so the generator doesn't need to know the prefix.
auth_header_value: Optional[str] = None
if mcp_auth_header:
server_auth_type = getattr(mcp_server, "auth_type", None) if mcp_server else None
if server_auth_type == MCPAuth.api_key:
auth_header_value = f"ApiKey {mcp_auth_header}"
elif server_auth_type == MCPAuth.basic:
auth_header_value = f"Basic {mcp_auth_header}"
else:
auth_header_value = f"Bearer {mcp_auth_header}"
_auth_token = _request_auth_header.set(auth_header_value)
try:
local_content = await _handle_local_mcp_tool(name, arguments)
finally:
_request_auth_header.reset(_auth_token)
response = CallToolResult(content=cast(Any, local_content), isError=False)
# Try managed MCP server tool (pass the full prefixed name)
# Primary and recommended way to use external MCP servers
#########################################################
else:
# If we haven't already resolved the server, do it now for dispatch
if mcp_server is None:
mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(
name
)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
# Update model_call_details with the cost info
if litellm_logging_obj:
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
response = await _handle_managed_mcp_tool(
server_name=server_name,
name=original_tool_name, # Pass the full name (potentially prefixed)
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
litellm_logging_obj=litellm_logging_obj,
host_progress_callback=host_progress_callback,
)
elif mcp_server:
response = await _handle_managed_mcp_tool(
server_name=server_name,
name=original_tool_name, # Pass the full name (potentially prefixed)
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
litellm_logging_obj=litellm_logging_obj,
host_progress_callback=host_progress_callback,
)
# Fall back to local tool registry with original name (legacy support)
#########################################################
# Deprecated: Local MCP Server Tool
#########################################################
else:
local_content = await _handle_local_mcp_tool(
original_tool_name, arguments
)
response = CallToolResult(
content=cast(Any, local_content), isError=False
)
# Fall back to local tool registry with original name (legacy support)
#########################################################
# Deprecated: Local MCP Server Tool
#########################################################
else:
local_content = await _handle_local_mcp_tool(
original_tool_name, arguments
)
response = CallToolResult(
content=cast(Any, local_content), isError=False
)
return response

0
litellm/proxy/_experimental/out/404.html → litellm/proxy/_experimental/out/404/index.html
View file

0
litellm/proxy/_experimental/out/_not-found.html → litellm/proxy/_experimental/out/_not-found/index.html
View file

0
litellm/proxy/_experimental/out/api-reference.html → litellm/proxy/_experimental/out/api-reference/index.html
View file

0
litellm/proxy/_experimental/out/experimental/api-playground.html → litellm/proxy/_experimental/out/experimental/api-playground/index.html
View file

0
litellm/proxy/_experimental/out/experimental/budgets.html → litellm/proxy/_experimental/out/experimental/budgets/index.html
View file

0
litellm/proxy/_experimental/out/experimental/caching.html → litellm/proxy/_experimental/out/experimental/caching/index.html
View file

0
litellm/proxy/_experimental/out/experimental/claude-code-plugins.html → litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
View file

0
litellm/proxy/_experimental/out/experimental/old-usage.html → litellm/proxy/_experimental/out/experimental/old-usage/index.html
View file

0
litellm/proxy/_experimental/out/experimental/prompts.html → litellm/proxy/_experimental/out/experimental/prompts/index.html
View file

0
litellm/proxy/_experimental/out/experimental/tag-management.html → litellm/proxy/_experimental/out/experimental/tag-management/index.html
View file

0
litellm/proxy/_experimental/out/guardrails.html → litellm/proxy/_experimental/out/guardrails/index.html
View file

0
litellm/proxy/_experimental/out/login.html → litellm/proxy/_experimental/out/login/index.html
View file

0
litellm/proxy/_experimental/out/logs.html → litellm/proxy/_experimental/out/logs/index.html
View file

0
litellm/proxy/_experimental/out/mcp/oauth/callback.html → litellm/proxy/_experimental/out/mcp/oauth/callback/index.html
View file

0
litellm/proxy/_experimental/out/model-hub.html → litellm/proxy/_experimental/out/model-hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub.html → litellm/proxy/_experimental/out/model_hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View file

Some files were not shown because too many files have changed in this diff Show more