scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge remote-tracking branch 'origin' into litellm_add_model_fix_team_admin

Browse source
This commit is contained in:
yuneng-jiang 2025-11-21 16:31:53 -08:00
commit 38d9906029
464 changed files with 23270 additions and 6235 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
.circleci/config.yml
View file

@ -3339,7 +3339,7 @@ jobs:
python -m build
twine upload --verbose dist/*
e2e_ui_testing:
ui_build:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
@ -3366,6 +3366,48 @@ jobs:
# Now source the build script
source ./build_ui.sh
- persist_to_workspace:
root: .
paths:
- litellm/proxy/_experimental/out
ui_unit_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Run UI unit tests (Vitest)
command: |
# Use Node 20 (several deps require >=20)
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
--coverage.reporter=lcov \
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Upgrade Docker to v24.x (API 1.44+)
command: |
@ -3411,24 +3453,6 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Run UI unit tests (Vitest)
command: |
# Use Node 20 (several deps require >=20)
export NVM_DIR="/opt/circleci/.nvm"
source "$NVM_DIR/nvm.sh"
nvm install 20
nvm use 20
cd ui/litellm-dashboard
npm ci || npm install
# CI run, with both LCOV (Codecov) and HTML (artifact you can click)
CI=true npm run test -- --run --coverage \
--coverage.provider=v8 \
--coverage.reporter=lcov \
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
- run:
name: Build Docker image
@ -3633,6 +3657,20 @@ workflows:
only:
- main
- /litellm_.*/
- ui_build:
filters:
branches:
only:
- main
- /litellm_.*/
- ui_unit_tests:
requires:
- ui_build
filters:
branches:
only:
- main
- /litellm_.*/
- auth_ui_unit_tests:
filters:
branches:
@ -3640,6 +3678,8 @@ workflows:
- main
- /litellm_.*/
- e2e_ui_testing:
requires:
- ui_build
filters:
branches:
only:

6
Makefile
View file

@ -34,13 +34,13 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
pip install openai==1.99.5
pip install openai==2.8.0
poetry install --with dev
pip install openai==1.99.5
pip install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
pip install openai==1.99.5
pip install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"

12
cookbook/LiteLLM_CometAPI.ipynb vendored
View file

@ -28,7 +28,7 @@
"Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n",
"Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n",
"Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n",
"Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
"Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n",
"Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n",
"Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n",
@ -50,11 +50,11 @@
"Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n",
"Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n",
"Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n",
"Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n",
"Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n",
"Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n",
"Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n",

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.7
version: 0.4.8
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

3
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -22,6 +22,9 @@ spec:
metadata:
labels:
{{- include "litellm.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}

5
docker/Dockerfile.database
View file

@ -12,7 +12,10 @@ WORKDIR /app
USER root
# Install build dependencies
RUN apk add --no-cache gcc python3-dev openssl openssl-dev
RUN apk add --no-cache \
build-base \
python3-dev \
openssl-dev
RUN pip install --upgrade pip && \

24
docs/my-website/blog/authors.yml Normal file
View file

@ -0,0 +1,24 @@
litellm:
name: LiteLLM Team
title: LiteLLM Core Team
url: https://github.com/BerriAI/litellm
image_url: https://github.com/BerriAI.png
krrish:
name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
ishaan:
name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
# Alias for typo in name
ishaan-alt:
name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg

700
docs/my-website/blog/gemini_3/index.md Normal file
View file

@ -0,0 +1,700 @@
---
slug: gemini_3
title: "DAY 0 Support: Gemini 3 on LiteLLM"
date: 2025-11-19T10:00:00
authors:
- name: Sameer Kankute
title: "SWE @ LiteLLM (LLM Translation)"
url: https://in.linkedin.com/in/sameer-kankute
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
- name: Krrish Dholakia
title: "CEO, LiteLLM"
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: "CTO, LiteLLM"
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
tags: [gemini, day 0 support, llms]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::info
This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK.
:::
## Quick Start
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "Hello!"}],
reasoning_effort="low"
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Add to config.yaml:**
```yaml
model_list:
- model_name: gemini-3-pro-preview
litellm_params:
model: gemini/gemini-3-pro-preview
api_key: os.environ/GEMINI_API_KEY
```
**2. Start proxy:**
```bash
litellm --config /path/to/config.yaml
```
**3. Make request:**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "Hello!"}],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
## Supported Endpoints
LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on:
- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
Both endpoints support:
- Streaming and non-streaming responses
- Function calling with thought signatures
- Multi-turn conversations
- All Gemini 3-specific features
## Thought Signatures
#### What are Thought Signatures?
Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling.
#### How Thought Signatures Work
1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response
2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls
3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini
## Example: Multi-Turn Function Calling
#### Streaming with Thought Signatures
When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved:
<Tabs>
<TabItem value="streaming" label="Streaming SDK">
```python
import os
import litellm
from litellm import completion
os.environ["GEMINI_API_KEY"] = "your-api-key"
MODEL = "gemini/gemini-3-pro-preview"
messages = [
{"role": "system", "content": "You are a helpful assistant. Use the calculate tool."},
{"role": "user", "content": "What is 2+2?"},
]
tools = [{
"type": "function",
"function": {
"name": "calculate",
"description": "Calculate a mathematical expression",
"parameters": {
"type": "object",
"properties": {"expression": {"type": "string"}},
"required": ["expression"],
},
},
}]
print("Step 1: Sending request with stream=True...")
response = completion(
model=MODEL,
messages=messages,
stream=True,
tools=tools,
reasoning_effort="low"
)
# Collect all chunks
chunks = []
for part in response:
chunks.append(part)
# Reconstruct message using stream_chunk_builder
# Thought signatures are now preserved automatically!
full_response = litellm.stream_chunk_builder(chunks, messages=messages)
print(f"Full response: {full_response}")
assistant_msg = full_response.choices[0].message
# ✅ Thought signature is now preserved in provider_specific_fields
if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields:
thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature")
print(f"Thought signature preserved: {thought_sig is not None}")
# Append assistant message (includes thought signatures automatically)
messages.append(assistant_msg)
# Mock tool execution
messages.append({
"role": "tool",
"content": "4",
"tool_call_id": assistant_msg.tool_calls[0].id
})
print("\nStep 2: Sending tool result back to model...")
response_2 = completion(
model=MODEL,
messages=messages,
stream=True,
tools=tools,
reasoning_effort="low"
)
for part in response_2:
if part.choices[0].delta.content:
print(part.choices[0].delta.content, end="")
print() # New line
```
**Key Points:**
- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures
- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history
- ✅ Multi-turn conversations work seamlessly with streaming
</TabItem>
<TabItem value="sdk" label="Non-Streaming SDK">
```python
from openai import OpenAI
import json
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
# Define tools
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
# Step 1: Initial request
messages = [{"role": "user", "content": "What's the weather in Tokyo?"}]
response = client.chat.completions.create(
model="gemini-3-pro-preview",
messages=messages,
tools=tools,
reasoning_effort="low"
)
# Step 2: Append assistant message (thought signatures automatically preserved)
messages.append(response.choices[0].message)
# Step 3: Execute tool and append result
for tool_call in response.choices[0].message.tool_calls:
if tool_call.function.name == "get_weather":
result = {"temperature": 30, "unit": "celsius"}
messages.append({
"role": "tool",
"content": json.dumps(result),
"tool_call_id": tool_call.id
})
# Step 4: Follow-up request (thought signatures automatically included)
response2 = client.chat.completions.create(
model="gemini-3-pro-preview",
messages=messages,
tools=tools,
reasoning_effort="low"
)
print(response2.choices[0].message.content)
```
**Key Points:**
- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature`
- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved
- ✅ You don't need to manually extract or manage thought signatures
</TabItem>
<TabItem value="proxy" label="cURL">
```bash
# Step 1: Initial request
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [
{"role": "user", "content": "What'\''s the weather in Tokyo?"}
],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
],
"reasoning_effort": "low"
}'
```
**Response includes thought signature:**
```json
{
"choices": [{
"message": {
"role": "assistant",
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Tokyo\"}"
},
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
}
}]
}
}]
}
```
```bash
# Step 2: Follow-up request (include assistant message with thought signature)
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [
{"role": "user", "content": "What'\''s the weather in Tokyo?"},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Tokyo\"}"
},
"provider_specific_fields": {
"thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
}
}]
},
{
"role": "tool",
"content": "{\"temperature\": 30, \"unit\": \"celsius\"}",
"tool_call_id": "call_abc123"
}
],
"tools": [...],
"reasoning_effort": "low"
}'
```
</TabItem>
</Tabs>
#### Important Notes on Thought Signatures
1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them.
2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature.
3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved.
4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning.
## Conversation History: Switching from Non-Gemini-3 Models
#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history?
**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed.
#### How It Works
When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM:
1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures
2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility
3. **Maintains conversation flow**: Your conversation history continues to work seamlessly
#### Example: Switching Models Mid-Conversation
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from openai import OpenAI
client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
# Step 1: Start with gemini-2.5-flash (no thought signatures)
messages = [{"role": "user", "content": "What's the weather?"}]
response1 = client.chat.completions.create(
model="gemini-2.5-flash",
messages=messages,
tools=[...],
reasoning_effort="low"
)
# Append assistant message (no tool call thought signature from gemini-2.5-flash)
messages.append(response1.choices[0].message)
# Step 2: Switch to gemini-3-pro-preview
# LiteLLM automatically adds dummy thought signature to the previous assistant message
response2 = client.chat.completions.create(
model="gemini-3-pro-preview", # 👈 Switched model
messages=messages, # 👈 Same conversation history
tools=[...],
reasoning_effort="low"
)
# ✅ Works seamlessly! No errors, no breaking changes
print(response2.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="cURL">
```bash
# Step 1: Start with gemini-2.5-flash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-2.5-flash",
"messages": [{"role": "user", "content": "What'\''s the weather?"}],
"tools": [...],
"reasoning_effort": "low"
}'
# Step 2: Switch to gemini-3-pro-preview with same conversation history
# LiteLLM automatically handles the missing thought signature
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview", # 👈 Switched model
"messages": [
{"role": "user", "content": "What'\''s the weather?"},
{
"role": "assistant",
"tool_calls": [...] # 👈 No thought_signature from gemini-2.5-flash
}
],
"tools": [...],
"reasoning_effort": "low"
}'
# ✅ Works! LiteLLM adds dummy signature automatically
```
</TabItem>
</Tabs>
#### Dummy Signature Details
The dummy signature used is: `base64("skip_thought_signature_validator")`
This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to:
- Accept the conversation history without validation errors
- Continue the conversation seamlessly
- Maintain context across model switches
## Thinking Level Parameter
#### How `reasoning_effort` Maps to `thinking_level`
For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter:
| `reasoning_effort` | `thinking_level` | Notes |
|-------------------|------------------|-------|
| `"minimal"` | `"low"` | Maps to low thinking level |
| `"low"` | `"low"` | Default for most use cases |
| `"medium"` | `"high"` | Medium not available yet, maps to high |
| `"high"` | `"high"` | Maximum reasoning depth |
| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking |
| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking |
#### Default Behavior
If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs.
### Example Usage
<Tabs>
<TabItem value="sdk" label="Python SDK">
```python
from litellm import completion
# Low thinking level (faster, lower cost)
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "What's the weather?"}],
reasoning_effort="low" # Maps to thinking_level="low"
)
# High thinking level (deeper reasoning, higher cost)
response = completion(
model="gemini/gemini-3-pro-preview",
messages=[{"role": "user", "content": "Solve this complex math problem step by step."}],
reasoning_effort="high" # Maps to thinking_level="high"
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```bash
# Low thinking level
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "What'\''s the weather?"}],
"reasoning_effort": "low"
}'
# High thinking level
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-3-pro-preview",
"messages": [{"role": "user", "content": "Solve this complex problem."}],
"reasoning_effort": "high"
}'
```
</TabItem>
</Tabs>
## Important Notes
1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`.
2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause:
- Infinite loops
- Degraded reasoning performance
- Failure on complex tasks
3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance.
## Using with Claude Code CLI
You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows.
### Setup
**1. Add Gemini 3 Pro Preview to your `config.yaml`:**
```yaml
model_list:
- model_name: gemini-3-pro-preview
litellm_params:
model: gemini/gemini-3-pro-preview
api_key: os.environ/GEMINI_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
**2. Set environment variables:**
```bash
export GEMINI_API_KEY="your-gemini-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
**3. Start LiteLLM Proxy:**
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
**4. Configure Claude Code to use LiteLLM Proxy:**
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
**5. Use Gemini 3 Pro Preview with Claude Code:**
```bash
# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy
claude --model gemini-3-pro-preview
```
### Example Usage
Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface:
```bash
$ claude --model gemini-3-pro-preview
> Explain how thought signatures work in multi-turn conversations.
# Gemini 3 Pro Preview responds through Claude Code interface
```
### Benefits
- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface
- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy
- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging
- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models
- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code
### Troubleshooting
**Claude Code not finding the model:**
- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview`
- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy
**Authentication errors:**
- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
- Ensure `GEMINI_API_KEY` is set correctly
- Check LiteLLM proxy logs for detailed error messages
## Best Practices
#### 1. Always Include Thought Signatures in Conversation History
When building multi-turn conversations with function calling:
✅ **Do:**
```python
# Append the full assistant message (includes thought signatures)
messages.append(response.choices[0].message)
```
❌ **Don't:**
```python
# Don't manually construct assistant messages without thought signatures
messages.append({
"role": "assistant",
"tool_calls": [...] # Missing thought signatures!
})
```
#### 2. Use Appropriate Thinking Levels
- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization
- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning
#### 3. Keep Temperature at Default
For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues.
#### 4. Handle Model Switches Gracefully
When switching from non-Gemini-3 to Gemini-3:
- ✅ LiteLLM automatically handles missing thought signatures
- ✅ No manual intervention needed
- ✅ Conversation history continues seamlessly
## Troubleshooting
#### Issue: Missing Thought Signatures
**Symptom**: Error when including assistant messages in conversation history
**Solution**: Ensure you're appending the full assistant message from the response:
```python
messages.append(response.choices[0].message) # ✅ Includes thought signatures
```
#### Issue: Conversation Breaks When Switching Models
**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview
**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version.
#### Issue: Infinite Loops or Poor Performance
**Symptom**: Model gets stuck or produces poor results
**Solution**:
- Ensure `temperature=1.0` (default for Gemini 3)
- Check that `reasoning_effort` is set appropriately
- Verify you're using the correct model name: `gemini/gemini-3-pro-preview`
## Additional Resources
- [Gemini Provider Documentation](../gemini.md)
- [Thought Signatures Guide](../gemini.md#thought-signatures)
- [Reasoning Content Documentation](../../reasoning_content.md)
- [Function Calling Guide](../../function_calling.md)

98
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
@ -197,6 +197,53 @@ for idx, image_obj in enumerate(response.data):
f.write(base64.b64decode(image_obj.b64_json))
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
#### Basic Image Edit (Gemini)
```python showLineNumbers title="Vertex AI Gemini Image Edit"
import os
import litellm
# Set Vertex AI credentials
os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json"
response = litellm.image_edit(
model="vertex_ai/gemini-2.5-flash",
image=open("original_image.png", "rb"),
prompt="Add neon lights in the background",
size="1024x1024",
)
print(response)
```
#### Image Edit with Imagen (Supports Masks)
```python showLineNumbers title="Vertex AI Imagen Image Edit"
import os
import litellm
# Set Vertex AI credentials
os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service-account.json"
# Imagen supports mask for inpainting
response = litellm.image_edit(
model="vertex_ai/imagen-3.0-capability-001",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"), # Optional: for inpainting
prompt="Turn this into watercolor style scenery",
n=2, # Number of variations
size="1024x1024",
)
print(response)
```
</TabItem>
</Tabs>
@ -302,6 +349,55 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-F "size=1024x1024"
```
</TabItem>
<TabItem value="vertex_ai" label="Vertex AI">
1. Add Vertex AI image edit models to your `config.yaml`:
```yaml showLineNumbers title="Vertex AI Proxy Configuration"
model_list:
- model_name: vertex-gemini-image-edit
litellm_params:
model: vertex_ai/gemini-2.5-flash
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
- model_name: vertex-imagen-image-edit
litellm_params:
model: vertex_ai/imagen-3.0-capability-001
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: os.environ/VERTEXAI_LOCATION
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
```
2. Start the LiteLLM proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
```
3. Make an image edit request:
```bash showLineNumbers title="Vertex AI Gemini Proxy Image Edit"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=vertex-gemini-image-edit" \
-F "image=@original_image.png" \
-F "prompt=Add neon lights in the background" \
-F "size=1024x1024"
```
4. Imagen image edit with mask:
```bash showLineNumbers title="Vertex AI Imagen Proxy Image Edit with Mask"
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-F "model=vertex-imagen-image-edit" \
-F "image=@original_image.png" \
-F "mask=@mask_image.png" \
-F "prompt=Turn this into watercolor style scenery" \
-F "n=2" \
-F "size=1024x1024"
```
</TabItem>
</Tabs>

22
docs/my-website/docs/mcp.md
View file

@ -657,7 +657,7 @@ LiteLLM Proxy provides two methods for controlling access to specific MCP server
### Method 1: URL-based Namespacing
LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/mcp/<servers or access groups>`. This allows you to:
LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/<servers or access groups>/mcp`. This allows you to:
- **Direct URL Access**: Point MCP clients directly to specific servers or access groups via URL
- **Simplified Configuration**: Use URLs instead of headers for server selection
@ -666,14 +666,14 @@ LiteLLM Proxy supports URL-based namespacing for MCP servers using the format `/
#### URL Format
```
<your-litellm-proxy-base-url>/mcp/<server_alias_or_access_group>
<your-litellm-proxy-base-url>/<server_alias_or_access_group>/mcp
```
**Examples:**
- `/mcp/github` - Access tools from the "github" MCP server
- `/mcp/zapier` - Access tools from the "zapier" MCP server
- `/mcp/dev_group` - Access tools from all servers in the "dev_group" access group
- `/mcp/github,zapier` - Access tools from multiple specific servers
- `/github_mcp/mcp` - Access tools from the "github_mcp" MCP server
- `/zapier/mcp` - Access tools from the "zapier" MCP server
- `/dev_group/mcp` - Access tools from all servers in the "dev_group" access group
- `/github_mcp,zapier/mcp` - Access tools from multiple specific servers
#### Usage Examples
@ -690,7 +690,7 @@ curl --location 'https://api.openai.com/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/mcp/github",
"server_url": "<your-litellm-proxy-base-url>/github_mcp/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@ -718,7 +718,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
{
"type": "mcp",
"server_label": "litellm",
"server_url": "<your-litellm-proxy-base-url>/mcp/dev_group",
"server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
"require_approval": "never",
"headers": {
"x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@ -740,7 +740,7 @@ This example uses URL namespacing to access all servers in the "dev_group" acces
{
"mcpServers": {
"LiteLLM": {
"url": "<your-litellm-proxy-base-url>/mcp/github,zapier",
"url": "<your-litellm-proxy-base-url>/github_mcp,zapier/mcp",
"headers": {
"x-litellm-api-key": "Bearer $LITELLM_API_KEY"
}
@ -862,8 +862,8 @@ This configuration in Cursor IDE settings will limit tool access to only the spe
| Feature | Header Namespacing | URL Namespacing |
|---------|-------------------|-----------------|
| **Method** | Uses `x-mcp-servers` header | Uses URL path `/mcp/<servers>` |
| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `/mcp/<servers>` endpoint |
| **Method** | Uses `x-mcp-servers` header | Uses URL path `/<servers>/mcp` |
| **Endpoint** | Standard `litellm_proxy` endpoint | Custom `/<servers>/mcp` endpoint |
| **Configuration** | Requires additional header | Self-contained in URL |
| **Multiple Servers** | Comma-separated in header | Comma-separated in URL path |
| **Access Groups** | Supported via header | Supported via URL path |

277
docs/my-website/docs/providers/docker_model_runner.md Normal file
View file

@ -0,0 +1,277 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Docker Model Runner
## Overview
| Property | Details |
|-------|-------|
| Description | Docker Model Runner allows you to run large language models locally using Docker Desktop. |
| Provider Route on LiteLLM | `docker_model_runner/` |
| Link to Provider Doc | [Docker Model Runner ↗](https://docs.docker.com/ai/model-runner/) |
| Base URL | `http://localhost:22088` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
<br />
https://docs.docker.com/ai/model-runner/
**We support ALL Docker Model Runner models, just set `docker_model_runner/` as a prefix when sending completion requests**
## Quick Start
Docker Model Runner is a Docker Desktop feature that lets you run AI models locally. It provides better performance than other local solutions while maintaining OpenAI compatibility.
### Installation
1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/)
2. Enable Docker Model Runner in Docker Desktop settings
3. Download your preferred model through Docker Desktop
## Environment Variables
```python showLineNumbers title="Environment Variables"
os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp" # Optional - defaults to this
os.environ["DOCKER_MODEL_RUNNER_API_KEY"] = "dummy-key" # Optional - Docker Model Runner may not require auth for local instances
```
**Note:**
- Docker Model Runner typically runs locally and may not require authentication. LiteLLM will use a dummy key by default if no key is provided.
- The API base should include the engine path (e.g., `/engines/llama.cpp`)
## API Base Structure
Docker Model Runner uses a unique URL structure:
```
http://model-runner.docker.internal/engines/{engine}/v1/chat/completions
```
Where `{engine}` is the engine you want to use (typically `llama.cpp`).
**Important:** Specify the engine in your `api_base` URL, not in the model name:
- ✅ Correct: `api_base="http://localhost:22088/engines/llama.cpp"`, `model="docker_model_runner/llama-3.1"`
- ❌ Incorrect: `api_base="http://localhost:22088"`, `model="docker_model_runner/llama.cpp/llama-3.1"`
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Docker Model Runner Non-streaming Completion"
import os
import litellm
from litellm import completion
# Specify the engine in the api_base URL
os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Docker Model Runner call
response = completion(
model="docker_model_runner/llama-3.1",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Docker Model Runner Streaming Completion"
import os
import litellm
from litellm import completion
# Specify the engine in the api_base URL
os.environ["DOCKER_MODEL_RUNNER_API_BASE"] = "http://localhost:22088/engines/llama.cpp"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Docker Model Runner call with streaming
response = completion(
model="docker_model_runner/llama-3.1",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Custom API Base and Engine
```python showLineNumbers title="Custom API Base with Different Engine"
import litellm
from litellm import completion
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Specify the engine in the api_base URL
# Using a different host and engine
response = completion(
model="docker_model_runner/llama-3.1",
messages=messages,
api_base="http://model-runner.docker.internal/engines/llama.cpp"
)
print(response)
```
### Using Different Engines
```python showLineNumbers title="Using a Different Engine"
import litellm
from litellm import completion
messages = [{"content": "Hello, how are you?", "role": "user"}]
# To use a different engine, specify it in the api_base
# For example, if Docker Model Runner supports other engines:
response = completion(
model="docker_model_runner/mistral-7b",
messages=messages,
api_base="http://localhost:22088/engines/custom-engine"
)
print(response)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: llama-3.1
litellm_params:
model: docker_model_runner/llama-3.1
api_base: http://localhost:22088/engines/llama.cpp
- model_name: mistral-7b
litellm_params:
model: docker_model_runner/mistral-7b
api_base: http://localhost:22088/engines/llama.cpp
```
Start your LiteLLM Proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Docker Model Runner via Proxy - Non-streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Non-streaming response
response = client.chat.completions.create(
model="llama-3.1",
messages=[{"role": "user", "content": "hello from litellm"}]
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Docker Model Runner via Proxy - Streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Streaming response
response = client.chat.completions.create(
model="llama-3.1",
messages=[{"role": "user", "content": "hello from litellm"}],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK"
import litellm
# Configure LiteLLM to use your proxy
response = litellm.completion(
model="litellm_proxy/llama-3.1",
messages=[{"role": "user", "content": "hello from litellm"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Docker Model Runner via Proxy - LiteLLM SDK Streaming"
import litellm
# Configure LiteLLM to use your proxy with streaming
response = litellm.completion(
model="litellm_proxy/llama-3.1",
messages=[{"role": "user", "content": "hello from litellm"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key",
stream=True
)
for chunk in response:
if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Docker Model Runner via Proxy - cURL"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "llama-3.1",
"messages": [{"role": "user", "content": "hello from litellm"}]
}'
```
```bash showLineNumbers title="Docker Model Runner via Proxy - cURL Streaming"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "llama-3.1",
"messages": [{"role": "user", "content": "hello from litellm"}],
"stream": true
}'
```
</TabItem>
</Tabs>
For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
## API Reference
For detailed API information, see the [Docker Model Runner API Reference](https://docs.docker.com/ai/model-runner/api-reference/).

2
docs/my-website/docs/providers/gemini.md
View file

@ -1308,6 +1308,8 @@ curl --location 'http://localhost:4000/v1/chat/completions' \
5. **Format**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls in the response, and are automatically included when you append the assistant message to your conversation history.
6. **Chat Completions Clients**: With chat completions clients where you cannot control whether or not the previous assistant message is included as-is (ex langchain's ChatOpenAI), LiteLLM also preserves the thought signature by appending it to the tool call id (`call_123__thought__<thought-signature>`) and extracting it back out before sending the outbound request to Gemini.
## JSON Mode
<Tabs>

62
docs/my-website/docs/providers/xai.md
View file

@ -11,6 +11,68 @@ https://docs.x.ai/docs
:::
## Supported Models
**Latest Release** - Grok 4.1 Fast: Optimized for high-performance agentic tool calling with 2M context and prompt caching.
| Model | Context | Features |
|-------|---------|----------|
| `xai/grok-4-1-fast-reasoning` | 2M tokens | **Reasoning**, Function calling, Vision, Audio, Web search, Caching |
| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Function calling, Vision, Audio, Web search, Caching |
**When to use:**
- ✅ **Reasoning model**: Complex analysis, planning, multi-step reasoning problems
- ✅ **Non-reasoning model**: Simple queries, faster responses, lower token usage
**Example:**
```python
from litellm import completion
# With reasoning
response = completion(
model="xai/grok-4-1-fast-reasoning",
messages=[{"role": "user", "content": "Analyze this problem step by step..."}]
)
# Without reasoning
response = completion(
model="xai/grok-4-1-fast-non-reasoning",
messages=[{"role": "user", "content": "What's 2+2?"}]
)
```
---
### All Available Models
| Model Family | Model | Context | Features |
|--------------|-------|---------|----------|
| **Grok 4.1** | `xai/grok-4-1-fast-reasoning` | 2M | **Reasoning**, Tools, Vision, Audio, Web search, Caching |
| | `xai/grok-4-1-fast-non-reasoning` | 2M | Tools, Vision, Audio, Web search, Caching |
| **Grok 4** | `xai/grok-4` | 256K | Tools, Web search |
| | `xai/grok-4-0709` | 256K | Tools, Web search |
| | `xai/grok-4-fast-reasoning` | 2M | **Reasoning**, Tools, Web search |
| | `xai/grok-4-fast-non-reasoning` | 2M | Tools, Web search |
| **Grok 3** | `xai/grok-3` | 131K | Tools, Web search |
| | `xai/grok-3-mini` | 131K | Tools, Web search |
| | `xai/grok-3-fast-beta` | 131K | Tools, Web search |
| **Grok Code** | `xai/grok-code-fast` | 256K | **Reasoning**, Tools, Code generation, Caching |
| **Grok 2** | `xai/grok-2` | 131K | Tools, **Vision** |
| | `xai/grok-2-vision-latest` | 32K | Tools, **Vision** |
**Features:**
- **Reasoning** = Chain-of-thought reasoning with reasoning tokens
- **Tools** = Function calling / Tool use
- **Web search** = Live internet search
- **Vision** = Image understanding
- **Audio** = Audio input support
- **Caching** = Prompt caching for cost savings
- **Code generation** = Optimized for code tasks
**Pricing:** See [xAI's pricing page](https://docs.x.ai/docs/models) for current rates.
## API Key
```python
# env variable

51
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -380,3 +380,54 @@ If you need to inspect the JWT fields received from your SSO provider by LiteLLM
Once redirected, you should see a page called "SSO Debug Information". This page displays the JWT fields received from your SSO provider (as shown in the image above)
## Advanced
### Manage User Roles via Azure App Roles
Centralize role management by defining user permissions in Azure Entra ID. LiteLLM will automatically assign roles based on your Azure configuration when users sign in—no need to manually manage roles in LiteLLM.
#### Step 1: Create App Roles on Azure App Registration
1. Navigate to your App Registration on https://portal.azure.com/
2. Go to **App roles** > **Create app role**
3. Configure the app role using one of the [supported LiteLLM roles](./access_control.md#global-proxy-roles):
- **Display name**: Admin Viewer (or your preferred display name)
- **Value**: `proxy_admin_viewer` (must match one of the LiteLLM role values exactly)
4. Click **Apply** to save the role
5. Repeat for each LiteLLM role you want to use
**Supported LiteLLM role values** (see [full role documentation](./access_control.md#global-proxy-roles)):
- `proxy_admin` - Full admin access
- `proxy_admin_viewer` - Read-only admin access
- `internal_user` - Can create/view/delete own keys
- `internal_user_viewer` - Can view own keys (read-only)
<Image img={require('../../img/app_roles.png')} style={{ width: '900px', height: 'auto' }} />
---
#### Step 2: Assign Users to App Roles
1. Navigate to **Enterprise Applications** on https://portal.azure.com/
2. Select your LiteLLM application
3. Go to **Users and groups** > **Add user/group**
4. Select the user
5. Under **Select a role**, choose the app role you created (e.g., `proxy_admin_viewer`)
6. Click **Assign** to save
<Image img={require('../../img/app_role2.png')} style={{ width: '900px', height: 'auto' }} />
---
#### Step 3: Sign in and verify
1. Sign in to the LiteLLM UI via SSO
2. LiteLLM will automatically extract the app role from the JWT token
3. The user will be assigned the corresponding role (you can verify this in the UI by checking the user profile dropdown)
<Image img={require('../../img/app_role3.png')} style={{ width: '900px', height: 'auto' }} />
**Note:** The role from Entra ID will take precedence over any existing role in the LiteLLM database. This ensures your SSO provider is the authoritative source for user roles.

1
docs/my-website/docs/proxy/guardrails/ibm_guardrails.md
View file

@ -95,6 +95,7 @@ curl -i http://localhost:4000/v1/chat/completions \
These go under `optional_params`:
- `detector_params` - dict - Parameters to pass to your detector
- `extra_headers` - dict - Additional headers to inject into requests to IBM Guardrails, as a key-value dict.
- `score_threshold` - float - Only count detections above this score (0.0 to 1.0)
- `block_on_detection` - bool - Block the request when violations found. Default: `true`

43
docs/my-website/docs/proxy/guardrails/tool_permission.md
View file

@ -46,6 +46,43 @@ guardrails:
- `pre_call` Run **before** LLM call, on **input**
- `post_call` Run **after** LLM call, on **input & output**
### `on_disallowed_action` behavior
| Value | What happens |
| --- | --- |
| `block` | The request is immediately rejected. Pre-call checks raise a `400` HTTP error. Post-call checks raise `GuardrailRaisedException`, so the proxy responds with an error instead of the model output. Use when invoking the forbidden tool must halt the workflow. |
| `rewrite` | LiteLLM silently strips disallowed tools from the payload before it reaches the model (pre-call) or rewrites the model response/tool calls after the fact. The guardrail inserts error text into `message.content`/`tool_result` entries so the client learns the tool was blocked while the rest of the completion continues. Use when you want graceful degradation instead of hard failures. |
### Custom denial message
Set `violation_message_template` when you want the guardrail to return a branded error (e.g., “this violates our org policy…”). LiteLLM replaces placeholders from the denied tool:
- `{tool_name}` – the tool/function name (e.g., `Read`)
- `{rule_id}` – the matching rule ID (or `None` when the default action kicks in)
- `{default_message}` – the original LiteLLM message if you need to append it
Example:
```yaml
guardrails:
- guardrail_name: "tool-permission-guardrail"
litellm_params:
guardrail: tool_permission
mode: "post_call"
violation_message_template: "this violates our org policy, we don't support executing {tool_name} commands"
rules:
- id: "allow_bash"
tool_name: "Bash"
decision: "allow"
- id: "deny_read"
tool_name: "Read"
decision: "deny"
default_action: "deny"
on_disallowed_action: "block"
```
If a request tries to invoke `Read`, the proxy now returns “this violates our org policy, we don't support executing Read commands” instead of the stock error text. Omit the field to keep the default messaging.
### 2. Start the Proxy
```shell
@ -57,7 +94,7 @@ litellm --config config.yaml --port 4000
<Tabs>
<TabItem value="block" label="Block Request">
**Block requset**
**Block request (`on_disallowed_action: block`)**
```bash
# Test
@ -96,7 +133,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
</TabItem>
<TabItem value="rewrite" label="Rewrite Request">
**Rewrite requset**
**Rewrite request (`on_disallowed_action: rewrite`)**
```bash
# Test
@ -118,7 +155,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
}'
```
**Expected response:**
**Expected response (tool removed, completion continues):**
```json
{

2
docs/my-website/docs/proxy/litellm_managed_files.md
View file

@ -424,4 +424,4 @@ No, as of `v1.71.2` users can only view/edit/delete files they have created.
## See Also
- [Managed Files w/ Finetuning APIs](../../docs/proxy/managed_finetuning)
- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batch)
- [Managed Files w/ Batch APIs](../../docs/proxy/managed_batches)

54
docs/my-website/docs/secret_managers/aws_secret_manager.md
View file

@ -110,3 +110,57 @@ The `primary_secret_name` allows you to read multiple keys from a single AWS Sec
This reduces the number of AWS Secrets you need to manage.
## IAM Role Assumption
Use IAM roles instead of static AWS credentials for better security.
### Basic IAM Role
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMSecretManagerRole"
aws_session_name: "litellm-session"
```
### Cross-Account Access
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::999999999999:role/CrossAccountRole"
aws_external_id: "unique-external-id"
```
### EKS with IRSA
```yaml
general_settings:
key_management_system: "aws_secret_manager"
key_management_settings:
store_virtual_keys: true
aws_region_name: "us-east-1"
aws_role_name: "arn:aws:iam::123456789012:role/LiteLLMServiceAccountRole"
aws_web_identity_token: "os.environ/AWS_WEB_IDENTITY_TOKEN_FILE"
```
### Configuration Parameters
| Parameter | Description |
|-----------|-------------|
| `aws_region_name` | AWS region |
| `aws_role_name` | IAM role ARN to assume |
| `aws_session_name` | Session name (optional) |
| `aws_external_id` | External ID for cross-account |
| `aws_profile_name` | AWS profile from `~/.aws/credentials` |
| `aws_web_identity_token` | OIDC token path for IRSA |
| `aws_sts_endpoint` | Custom STS endpoint for VPC |

3
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -105,7 +105,7 @@ LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual ke
Alternatively, use the Anthropic pass-through endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
@ -221,7 +221,6 @@ You can also connect MCP servers to Claude Code via LiteLLM Proxy.
Limitations:
- Currently, only HTTP MCP servers are supported
- Does not work in Cursor IDE yet.
:::

17
docs/my-website/docusaurus.config.js
View file

@ -101,6 +101,21 @@ const config = {
include: ['**/*.{md,mdx}'],
},
],
[
'@docusaurus/plugin-content-blog',
{
id: 'blog',
path: './blog',
routeBasePath: 'blog',
blogTitle: 'Blog',
blogSidebarTitle: 'All Posts',
blogSidebarCount: 'ALL',
postsPerPage: 10,
showReadingTime: false,
sortPosts: 'descending',
include: ['**/index.{md,mdx}'],
},
],
() => ({
name: 'cripchat',
@ -129,6 +144,7 @@ const config = {
docs: {
sidebarPath: require.resolve('./sidebars.js'),
},
blog: false, // Disable the default blog plugin from preset-classic
theme: {
customCss: require.resolve('./src/css/custom.css'),
},
@ -177,6 +193,7 @@ const config = {
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Release Notes', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{
href: 'https://models.litellm.ai/',
label: '💸 LLM Model Cost Map',

BIN
docs/my-website/img/app_role2.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 256 KiB

BIN
docs/my-website/img/app_role3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 174 KiB

BIN
docs/my-website/img/app_roles.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 277 KiB

BIN
docs/my-website/img/favicon_converted.ico Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.2 KiB

18
docs/my-website/release_notes/authors.yml Normal file
View file

@ -0,0 +1,18 @@
krrish:
name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
ishaan:
name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
# Alias for typo in name
ishaan-alt:
name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg

119
docs/my-website/sidebars.js
View file

@ -146,13 +146,13 @@ const sidebars = {
type: "category",
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/custom_sso",
"proxy/ai_hub",
"proxy/public_teams",
"proxy/self_serve",
"proxy/ui",
"proxy/ui/bulk_edit_users",
"proxy/ui_credentials",
"tutorials/scim_litellm",
@ -530,13 +530,39 @@ const sidebars = {
"providers/bedrock_vector_store",
]
},
"providers/milvus_vector_stores",
"providers/litellm_proxy",
"providers/meta_llama",
"providers/mistral",
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/anyscale",
"providers/baseten",
"providers/bytez",
"providers/cerebras",
"providers/clarifai",
"providers/cloudflare_workers",
"providers/codestral",
"providers/cohere",
"providers/anyscale",
"providers/cometapi",
"providers/compactifai",
"providers/custom_llm_server",
"providers/dashscope",
"providers/databricks",
"providers/datarobot",
"providers/deepgram",
"providers/deepinfra",
"providers/deepseek",
"providers/docker_model_runner",
"providers/elevenlabs",
"providers/fal_ai",
"providers/featherless_ai",
"providers/fireworks_ai",
"providers/friendliai",
"providers/galadriel",
"providers/github",
"providers/github_copilot",
"providers/gradient_ai",
"providers/groq",
"providers/heroku",
{
type: "category",
label: "HuggingFace",
@ -546,10 +572,21 @@ const sidebars = {
]
},
"providers/hyperbolic",
"providers/databricks",
"providers/deepgram",
"providers/watsonx",
"providers/predibase",
"providers/infinity",
"providers/jina_ai",
"providers/lambda_ai",
"providers/lemonade",
"providers/llamafile",
"providers/lm_studio",
"providers/meta_llama",
"providers/milvus_vector_stores",
"providers/mistral",
"providers/moonshot",
"providers/morph",
"providers/nebius",
"providers/nlp_cloud",
"providers/novita",
{ type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" },
{
type: "category",
label: "Nvidia NIM",
@ -558,37 +595,13 @@ const sidebars = {
"providers/nvidia_nim_rerank",
]
},
{ type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" },
"providers/xai",
"providers/moonshot",
"providers/lm_studio",
"providers/cerebras",
"providers/volcano",
"providers/triton-inference-server",
"providers/oci",
"providers/ollama",
"providers/openrouter",
"providers/ovhcloud",
"providers/perplexity",
"providers/friendliai",
"providers/galadriel",
"providers/topaz",
"providers/groq",
"providers/deepseek",
"providers/elevenlabs",
"providers/fal_ai",
"providers/fireworks_ai",
"providers/clarifai",
"providers/compactifai",
"providers/lemonade",
"providers/vllm",
"providers/llamafile",
"providers/infinity",
"providers/xinference",
"providers/aiml",
"providers/cloudflare_workers",
"providers/deepinfra",
"providers/github",
"providers/github_copilot",
"providers/ai21",
"providers/nlp_cloud",
"providers/petals",
"providers/predibase",
"providers/recraft",
"providers/replicate",
{
@ -599,32 +612,20 @@ const sidebars = {
"providers/runwayml/videos",
]
},
"providers/sambanova",
"providers/snowflake",
"providers/togetherai",
"providers/topaz",
"providers/triton-inference-server",
"providers/v0",
"providers/vercel_ai_gateway",
"providers/morph",
"providers/lambda_ai",
"providers/novita",
"providers/vllm",
"providers/volcano",
"providers/voyage",
"providers/jina_ai",
"providers/aleph_alpha",
"providers/baseten",
"providers/openrouter",
"providers/sambanova",
"providers/custom_llm_server",
"providers/petals",
"providers/snowflake",
"providers/gradient_ai",
"providers/featherless_ai",
"providers/nebius",
"providers/dashscope",
"providers/bytez",
"providers/heroku",
"providers/oci",
"providers/datarobot",
"providers/ovhcloud",
"providers/wandb_inference",
"providers/cometapi",
"providers/watsonx",
"providers/xai",
"providers/xinference",
],
},
{

BIN
docs/my-website/static/img/favicon.ico
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 6.2 KiB

BIN
enterprise/dist/litellm_enterprise-0.1.22-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.22.tar.gz vendored Normal file
View file

Binary file not shown.

12
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -529,7 +529,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in file_ids:
## CHECK IF FILE ID IS MANAGED BY LITELM
is_base64_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
litellm_managed_file_ids.append(file_id)
@ -540,6 +539,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
unified_file_object = await self.get_unified_file_id(
file_id, litellm_parent_otel_span
)
if unified_file_object:
file_id_mapping[file_id] = unified_file_object.model_mappings
@ -815,18 +815,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Router,
**data: Dict,
) -> OpenAIFileObject:
file_id = convert_b64_uid_to_unified_uid(file_id)
# file_id = convert_b64_uid_to_unified_uid(file_id)
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span
)
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
for model_id, file_id in specific_model_file_id_mapping.items():
await llm_router.afile_delete(model=model_id, file_id=file_id, **data) # type: ignore
for model_id, model_file_id in specific_model_file_id_mapping.items():
await llm_router.afile_delete(model=model_id, file_id=model_file_id, **data) # type: ignore
stored_file_object = await self.delete_unified_file_id(
file_id, litellm_parent_otel_span
)
if stored_file_object:
return stored_file_object
else:
@ -847,6 +850,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_file_id_mapping
or await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
)
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.21"
version = "0.1.22"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.21"
version = "0.1.22"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.6.tar.gz vendored Normal file
View file

Binary file not shown.

12
litellm-proxy-extras/litellm_proxy_extras/migrations/20251119131227_add_prompt_versioning/migration.sql Normal file
View file

@ -0,0 +1,12 @@
-- DropIndex
DROP INDEX "LiteLLM_PromptTable_prompt_id_key";
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "version" INTEGER NOT NULL DEFAULT 1;
-- CreateIndex
CREATE INDEX "LiteLLM_PromptTable_prompt_id_idx" ON "LiteLLM_PromptTable"("prompt_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_key" ON "LiteLLM_PromptTable"("prompt_id", "version");

6
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -561,11 +561,15 @@ model LiteLLM_GuardrailsTable {
// Prompt table for storing prompt configurations
model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String @unique
prompt_id String
version Int @default(1)
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@index([prompt_id])
}
model LiteLLM_HealthCheckTable {

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.5"
version = "0.4.6"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.5"
version = "0.4.6"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

9
litellm/__init__.py
View file

@ -388,6 +388,7 @@ disable_add_prefix_to_prompt: bool = (
disable_copilot_system_to_assistant: bool = (
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
)
public_mcp_servers: Optional[List[str]] = None
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
@ -562,6 +563,7 @@ wandb_models: Set = set(WANDB_MODELS)
ovhcloud_models: Set = set()
ovhcloud_embedding_models: Set = set()
lemonade_models: Set = set()
docker_model_runner_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -796,6 +798,8 @@ def add_known_models():
ovhcloud_embedding_models.add(key)
elif value.get("litellm_provider") == "lemonade":
lemonade_models.add(key)
elif value.get("litellm_provider") == "docker_model_runner":
docker_model_runner_models.add(key)
add_known_models()
@ -899,6 +903,7 @@ model_list = list(
| wandb_models
| ovhcloud_models
| lemonade_models
| docker_model_runner_models
| set(clarifai_models)
)
@ -1342,10 +1347,14 @@ from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig
from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig
from .llms.github_copilot.responses.transformation import (
GithubCopilotResponsesAPIConfig,
)
from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig
from .llms.morph.chat.transformation import MorphChatConfig

75
litellm/batches/main.py
View file

@ -17,6 +17,7 @@ from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
from openai.types.batch import BatchRequestCounts
import litellm
from litellm._logging import verbose_logger
@ -223,10 +224,12 @@ def create_batch(
api_key=optional_params.api_key,
logging_obj=litellm_logging_obj,
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
model=model,
)
@ -609,10 +612,12 @@ def retrieve_batch(
function_id="batch_retrieve",
),
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
model=model,
)
@ -799,6 +804,7 @@ def list_batches(
async def acancel_batch(
batch_id: str,
model: Optional[str] = None,
custom_llm_provider: Literal["openai", "azure"] = "openai",
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
@ -813,11 +819,13 @@ async def acancel_batch(
try:
loop = asyncio.get_event_loop()
kwargs["acancel_batch"] = True
model = kwargs.pop("model", None)
# Use a partial function to pass your keyword arguments
func = partial(
cancel_batch,
batch_id,
model,
custom_llm_provider,
metadata,
extra_headers,
@ -840,7 +848,8 @@ async def acancel_batch(
def cancel_batch(
batch_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
metadata: Optional[Dict[str, str]] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
@ -852,6 +861,17 @@ def cancel_batch(
LiteLLM Equivalent of POST https://api.openai.com/v1/batches/{batch_id}/cancel
"""
try:
try:
if model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
)
except Exception as e:
verbose_logger.exception(
f"litellm.batches.main.py::cancel_batch() - Error inferring custom_llm_provider - {str(e)}"
)
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params = get_litellm_params(
custom_llm_provider=custom_llm_provider,
@ -1005,21 +1025,28 @@ def _handle_async_invoke_status(
created_at=status_response["submitTime"],
in_progress_at=status_response["lastModifiedTime"],
completed_at=status_response.get("endTime"),
failed_at=status_response.get("endTime")
if status_response["status"] == "failed"
else None,
request_counts={
"total": 1,
"completed": 1 if status_response["status"] == "completed" else 0,
"failed": 1 if status_response["status"] == "failed" else 0,
},
metadata={
"output_file_id": status_response["outputDataConfig"][
"s3OutputDataConfig"
]["s3Uri"],
"failure_message": status_response.get("failureMessage"),
"model_arn": status_response["modelArn"],
},
failed_at=(
status_response.get("endTime")
if status_response["status"] == "failed"
else None
),
request_counts=BatchRequestCounts(
total=1,
completed=1 if status_response["status"] == "completed" else 0,
failed=1 if status_response["status"] == "failed" else 0,
),
metadata=dict(
**{
"output_file_id": status_response["outputDataConfig"][
"s3OutputDataConfig"
]["s3Uri"],
"failure_message": status_response.get("failureMessage") or "",
"model_arn": status_response["modelArn"],
}
),
completion_window="24h",
endpoint="/v1/embeddings",
input_file_id="",
)
return result

50
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -26,7 +26,12 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
)
from litellm.types.llms.openai import ChatCompletionToolParamFunctionChunk, Reasoning
from litellm.types.llms.openai import (
ChatCompletionToolParamFunctionChunk,
Reasoning,
ResponsesAPIOptionalRequestParams,
ResponsesAPIStreamEvents,
)
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam
@ -165,13 +170,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
litellm_logging_obj: "LiteLLMLoggingObj",
client: Optional[Any] = None,
) -> dict:
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
(
input_items,
instructions,
) = self.convert_chat_completion_messages_to_responses_api(messages)
optional_params = self._extract_extra_body_params(optional_params)
# Build responses API request using the reverse transformation logic
responses_api_request = ResponsesAPIOptionalRequestParams()
@ -194,9 +199,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key in ("metadata"):
elif key == "metadata":
responses_api_request["metadata"] = value
elif key in ("previous_response_id"):
elif key == "previous_response_id":
responses_api_request["previous_response_id"] = value
elif key == "reasoning_effort":
responses_api_request["reasoning"] = self._map_reasoning_effort(value)
@ -538,6 +543,35 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return cast(List["ALL_RESPONSES_API_TOOL_PARAMS"], responses_tools)
def _extract_extra_body_params(self, optional_params: dict):
"""
Extract extra_body from optional_params and separate supported Responses API params
from unsupported ones. Supported params are moved to top-level optional_params,
unsupported params remain in extra_body.
"""
# Extract extra_body and separate supported params from unsupported ones
extra_body = optional_params.pop("extra_body", None) or {}
if not extra_body:
return optional_params
supported_responses_api_params = set(
ResponsesAPIOptionalRequestParams.__annotations__.keys()
)
# Also include params we handle specially
supported_responses_api_params.update({
"previous_response_id",
"reasoning_effort", # We map this to "reasoning"
})
# Extract supported params from extra_body and merge into optional_params
extra_body_copy = extra_body.copy()
for key, value in extra_body_copy.items():
if key in supported_responses_api_params:
# Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort)
optional_params[key] = extra_body.pop(key)
return optional_params
def _map_reasoning_effort(self, reasoning_effort: Union[str, Dict[str, Any]]) -> Optional[Reasoning]:
# If dict is passed, convert it directly to Reasoning object
if isinstance(reasoning_effort, dict):
@ -619,6 +653,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
# Handle different event types from responses API
event_type = parsed_chunk.get("type")
if isinstance(event_type, ResponsesAPIStreamEvents):
event_type = event_type.value
verbose_logger.debug(f"Chat provider: Processing event type: {event_type}")
if event_type == "response.created":
@ -638,7 +674,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
index=0,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=parsed_chunk.get("name", None),
name=output_item.get("name", None),
arguments=parsed_chunk.get("arguments", ""),
),
),
@ -684,7 +720,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
index=0,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=parsed_chunk.get("name", None),
name=output_item.get("name", None),
arguments="", # responses API sends everything again, we don't
),
),

3
litellm/constants.py
View file

@ -381,6 +381,7 @@ LITELLM_CHAT_PROVIDERS = [
"wandb",
"ovhcloud",
"lemonade",
"docker_model_runner",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@ -567,6 +568,7 @@ openai_compatible_providers: List = [
"wandb",
"cometapi",
"clarifai",
"docker_model_runner",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -1118,6 +1120,7 @@ SECRET_MANAGER_REFRESH_INTERVAL = int(
)
LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"default_internal_user_params",
"public_mcp_servers",
"public_agent_groups",
"public_model_groups",
"public_model_groups_links",

36
litellm/cost_calculator.py
View file

@ -133,6 +133,25 @@ def _cost_per_token_custom_pricing_helper(
return None
def _transcription_usage_has_token_details(
usage_block: Optional[Usage],
) -> bool:
if usage_block is None:
return False
prompt_tokens_val = getattr(usage_block, "prompt_tokens", 0) or 0
completion_tokens_val = getattr(usage_block, "completion_tokens", 0) or 0
prompt_details = getattr(usage_block, "prompt_tokens_details", None)
if prompt_details is not None:
audio_token_count = getattr(prompt_details, "audio_tokens", 0) or 0
text_token_count = getattr(prompt_details, "text_tokens", 0) or 0
if audio_token_count > 0 or text_token_count > 0:
return True
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def cost_per_token( # noqa: PLR0915
model: str = "",
prompt_tokens: int = 0,
@ -324,19 +343,18 @@ def cost_per_token( # noqa: PLR0915
usage=usage_block, model=model, custom_llm_provider=custom_llm_provider
)
elif call_type == "atranscription" or call_type == "transcription":
if model == "gpt-4o-mini-transcribe":
if _transcription_usage_has_token_details(usage_block):
return openai_cost_per_token(
model=model,
model=model_without_prefix,
usage=usage_block,
service_tier=service_tier,
)
else:
return openai_cost_per_second(
model=model,
custom_llm_provider=custom_llm_provider,
duration=audio_transcription_file_duration,
)
return openai_cost_per_second(
model=model_without_prefix,
custom_llm_provider=custom_llm_provider,
duration=audio_transcription_file_duration,
)
elif call_type == "search" or call_type == "asearch":
# Search providers use per-query pricing
from litellm.search import search_provider_cost_per_query

228
litellm/experimental_mcp_client/client.py
View file

@ -8,14 +8,21 @@ from datetime import timedelta
from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union
import httpx
from mcp import ClientSession, StdioServerParameters
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
from mcp.client.sse import sse_client
from mcp.client.stdio import stdio_client
from mcp.client.streamable_http import streamablehttp_client
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp.types import (
CallToolRequestParams as MCPCallToolRequestParams,
GetPromptRequestParams,
GetPromptResult,
Prompt,
ResourceTemplate,
)
from mcp.types import CallToolResult as MCPCallToolResult
from mcp.types import TextContent
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
@ -214,7 +221,7 @@ class MCPClient:
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
verbose_logger.exception(
f"MCP client list_tools failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
@ -289,3 +296,218 @@ class MCPClient:
], # Empty content for error case
isError=True,
)
async def list_prompts(self) -> List[Prompt]:
"""List available prompts from the server."""
verbose_logger.debug(
f"MCP client listing tools from {self.server_url or 'stdio'}"
)
async def _list_prompts_operation(session: ClientSession):
return await session.list_prompts()
try:
result = await self.run_with_session(_list_prompts_operation)
prompt_count = len(result.prompts)
prompt_names = [prompt.name for prompt in result.prompts]
verbose_logger.info(
f"MCP client listed {prompt_count} tools from {self.server_url or 'stdio'}: {prompt_names}"
)
return result.prompts
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_prompts was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_prompts failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_tools - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def get_prompt(
self, get_prompt_request_params: GetPromptRequestParams
) -> GetPromptResult:
"""Fetch a prompt definition from the MCP server."""
verbose_logger.info(
f"MCP client fetching prompt '{get_prompt_request_params.name}' with arguments: {get_prompt_request_params.arguments}"
)
async def _get_prompt_operation(session: ClientSession):
verbose_logger.debug("MCP client sending get_prompt request to session")
return await session.get_prompt(
name=get_prompt_request_params.name,
arguments=get_prompt_request_params.arguments,
)
try:
get_prompt_result = await self.run_with_session(_get_prompt_operation)
verbose_logger.info(
f"MCP client get_prompt '{get_prompt_request_params.name}' completed successfully"
)
return get_prompt_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client get_prompt was cancelled")
raise
except Exception as e:
import traceback
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
f"MCP client get_prompt failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Prompt: {get_prompt_request_params.name}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during get_prompt - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise
async def list_resources(self) -> list[Resource]:
"""List available resources from the server."""
verbose_logger.debug(
f"MCP client listing resources from {self.server_url or 'stdio'}"
)
async def _list_resources_operation(session: ClientSession):
return await session.list_resources()
try:
result = await self.run_with_session(_list_resources_operation)
resource_count = len(result.resources)
resource_names = [resource.name for resource in result.resources]
verbose_logger.info(
f"MCP client listed {resource_count} resources from {self.server_url or 'stdio'}: {resource_names}"
)
return result.resources
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resources was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_resources failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resources - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def list_resource_templates(self) -> list[ResourceTemplate]:
"""List available resource templates from the server."""
verbose_logger.debug(
f"MCP client listing resource templates from {self.server_url or 'stdio'}"
)
async def _list_resource_templates_operation(session: ClientSession):
return await session.list_resource_templates()
try:
result = await self.run_with_session(_list_resource_templates_operation)
resource_template_count = len(result.resourceTemplates)
resource_template_names = [
resourceTemplate.name for resourceTemplate in result.resourceTemplates
]
verbose_logger.info(
f"MCP client listed {resource_template_count} resource templates from {self.server_url or 'stdio'}: {resource_template_names}"
)
return result.resourceTemplates
except asyncio.CancelledError:
verbose_logger.warning("MCP client list_resource_templates was cancelled")
raise
except Exception as e:
error_type = type(e).__name__
verbose_logger.error(
f"MCP client list_resource_templates failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resource_templates - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
async def read_resource(self, url: AnyUrl) -> ReadResourceResult:
"""Fetch resource contents from the MCP server."""
verbose_logger.info(f"MCP client fetching resource '{url}'")
async def _read_resource_operation(session: ClientSession):
verbose_logger.debug("MCP client sending read_resource request to session")
return await session.read_resource(url)
try:
read_resource_result = await self.run_with_session(_read_resource_operation)
verbose_logger.info(
f"MCP client read_resource '{url}' completed successfully"
)
return read_resource_result
except asyncio.CancelledError:
verbose_logger.warning("MCP client read_resource was cancelled")
raise
except Exception as e:
import traceback
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
f"MCP client read_resource failed - "
f"Error Type: {error_type}, "
f"Error: {str(e)}, "
f"Url: {url}, "
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during read_resource - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise

28
litellm/files/main.py
View file

@ -95,7 +95,9 @@ async def acreate_file(
def create_file(
file: FileTypes,
purpose: Literal["assistants", "batch", "fine-tune"],
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock"]] = None,
custom_llm_provider: Optional[
Literal["openai", "azure", "vertex_ai", "bedrock"]
] = None,
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -155,10 +157,12 @@ def create_file(
api_key=optional_params.api_key,
logging_obj=logging_obj,
_is_async=_is_async,
client=client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None,
client=(
client
if client is not None
and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
else None
),
timeout=timeout,
)
elif custom_llm_provider == "openai":
@ -441,12 +445,14 @@ async def afile_delete(
"""
try:
loop = asyncio.get_event_loop()
model = kwargs.pop("model", None)
kwargs["is_async"] = True
# Use a partial function to pass your keyword arguments
func = partial(
file_delete,
file_id,
model,
custom_llm_provider,
extra_headers,
extra_body,
@ -470,7 +476,8 @@ async def afile_delete(
@client
def file_delete(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
model: Optional[str] = None,
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -481,6 +488,13 @@ def file_delete(
LiteLLM Equivalent of DELETE https://api.openai.com/v1/files
"""
try:
try:
if model is not None:
_, custom_llm_provider, _, _ = get_llm_provider(
model, custom_llm_provider
)
except Exception:
pass
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params_dict = get_litellm_params(**kwargs)
### TIMEOUT LOGIC ###
@ -566,7 +580,7 @@ def file_delete(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
message="LiteLLM doesn't support {} for 'delete_batch'. Only 'openai' is supported.".format(
custom_llm_provider
),
model="n/a",

3
litellm/integrations/arize/_utils.py
View file

@ -208,7 +208,10 @@ def set_attributes(
)
try:
# Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers)
optional_params = kwargs.get("optional_params", {})
if isinstance(optional_params, dict):
optional_params.pop("secret_fields", None)
litellm_params = kwargs.get("litellm_params", {})
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"

25
litellm/integrations/custom_guardrail.py
View file

@ -36,6 +36,7 @@ class CustomGuardrail(CustomLogger):
default_on: bool = False,
mask_request_content: bool = False,
mask_response_content: bool = False,
violation_message_template: Optional[str] = None,
**kwargs,
):
"""
@ -57,12 +58,34 @@ class CustomGuardrail(CustomLogger):
self.default_on: bool = default_on
self.mask_request_content: bool = mask_request_content
self.mask_response_content: bool = mask_response_content
self.violation_message_template: Optional[str] = violation_message_template
if supported_event_hooks:
## validate event_hook is in supported_event_hooks
self._validate_event_hook(event_hook, supported_event_hooks)
super().__init__(**kwargs)
def render_violation_message(
self, default: str, context: Optional[Dict[str, Any]] = None
) -> str:
"""Return a custom violation message if template is configured."""
if not self.violation_message_template:
return default
format_context: Dict[str, Any] = {"default_message": default}
if context:
format_context.update(context)
try:
return self.violation_message_template.format(**format_context)
except Exception as e:
verbose_logger.warning(
"Failed to format violation message template for guardrail %s: %s",
self.guardrail_name,
e,
)
return default
@staticmethod
def get_config_model() -> Optional[Type["GuardrailConfigModel"]]:
"""
@ -279,7 +302,7 @@ class CustomGuardrail(CustomLogger):
data, self.event_hook
)
if result is not None:
return result
return result
return True
def _event_hook_is_event_type(self, event_type: GuardrailEventHooks) -> bool:

22
litellm/integrations/dotprompt/__init__.py
View file

@ -25,6 +25,23 @@ def set_global_prompt_directory(directory: str) -> None:
litellm.global_prompt_directory = directory # type: ignore
def _get_prompt_data_from_dotprompt_content(dotprompt_content: str) -> dict:
"""
Get the prompt data from the dotprompt content.
The UI stores prompts under `dotprompt_content` in the database. This function parses the content and returns the prompt data in the format expected by the prompt manager.
"""
from .prompt_manager import PromptManager
# Parse the dotprompt content to extract frontmatter and content
temp_manager = PromptManager()
metadata, content = temp_manager._parse_frontmatter(dotprompt_content)
# Convert to prompt_data format
return {
"content": content.strip(),
"metadata": metadata
}
def prompt_initializer(
litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec"
@ -41,6 +58,11 @@ def prompt_initializer(
)
prompt_file = getattr(litellm_params, "prompt_file", None)
# Handle dotprompt_content from database
dotprompt_content = getattr(litellm_params, "dotprompt_content", None)
if dotprompt_content and not prompt_data and not prompt_file:
prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
try:
dot_prompt_manager = DotpromptManager(

21
litellm/integrations/dotprompt/dotprompt_manager.py
View file

@ -108,7 +108,7 @@ class DotpromptManager(CustomPromptManagement):
Compile a .prompt file into a PromptManagementClient structure.
This method:
1. Loads the prompt template from the .prompt file
1. Loads the prompt template from the .prompt file (with optional version)
2. Renders it with the provided variables
3. Converts the rendered text into chat messages
4. Extracts model and optional parameters from metadata
@ -116,13 +116,22 @@ class DotpromptManager(CustomPromptManagement):
try:
# Get the prompt template
template = self.prompt_manager.get_prompt(prompt_id)
# Get the prompt template (versioned or base)
template = self.prompt_manager.get_prompt(
prompt_id=prompt_id, version=prompt_version
)
if template is None:
raise ValueError(f"Prompt '{prompt_id}' not found in prompt directory")
version_str = f" (version {prompt_version})" if prompt_version else ""
raise ValueError(
f"Prompt '{prompt_id}'{version_str} not found in prompt directory"
)
# Render the template with variables
rendered_content = self.prompt_manager.render(prompt_id, prompt_variables)
# Render the template with variables (pass version for proper lookup)
rendered_content = self.prompt_manager.render(
prompt_id=prompt_id,
prompt_variables=prompt_variables,
version=prompt_version,
)
# Convert rendered content to chat messages
messages = self._convert_to_messages(rendered_content)

37
litellm/integrations/dotprompt/prompt_manager.py
View file

@ -183,7 +183,10 @@ class PromptManager:
return frontmatter, template_content
def render(
self, prompt_id: str, prompt_variables: Optional[Dict[str, Any]] = None
self,
prompt_id: str,
prompt_variables: Optional[Dict[str, Any]] = None,
version: Optional[int] = None,
) -> str:
"""
Render a prompt template with the given variables.
@ -191,6 +194,7 @@ class PromptManager:
Args:
prompt_id: The ID of the prompt template to render
prompt_variables: Variables to substitute in the template
version: Optional version number. If provided, looks for {prompt_id}.v{version}
Returns:
The rendered prompt string
@ -199,13 +203,16 @@ class PromptManager:
KeyError: If prompt_id is not found
ValueError: If template rendering fails
"""
if prompt_id not in self.prompts:
# Get the template (versioned or base)
template = self.get_prompt(prompt_id=prompt_id, version=version)
if template is None:
available_prompts = list(self.prompts.keys())
version_str = f" (version {version})" if version else ""
raise KeyError(
f"Prompt '{prompt_id}' not found. Available prompts: {available_prompts}"
f"Prompt '{prompt_id}'{version_str} not found. Available prompts: {available_prompts}"
)
template = self.prompts[prompt_id]
variables = prompt_variables or {}
# Validate input variables against schema if defined
@ -254,8 +261,26 @@ class PromptManager:
return type_mapping.get(schema_type.lower(), str) # type: ignore
def get_prompt(self, prompt_id: str) -> Optional[PromptTemplate]:
"""Get a prompt template by ID."""
def get_prompt(
self, prompt_id: str, version: Optional[int] = None
) -> Optional[PromptTemplate]:
"""
Get a prompt template by ID and optional version.
Args:
prompt_id: The base prompt ID
version: Optional version number. If provided, looks for {prompt_id}.v{version}
Returns:
The prompt template if found, None otherwise
"""
if version is not None:
# Try versioned prompt first: prompt_id.v{version}
versioned_id = f"{prompt_id}.v{version}"
if versioned_id in self.prompts:
return self.prompts[versioned_id]
# Fall back to base prompt_id
return self.prompts.get(prompt_id)
def list_prompts(self) -> List[str]:

2
litellm/integrations/langfuse/langfuse.py
View file

@ -228,6 +228,8 @@ class LangFuseLogger:
functions = optional_params.pop("functions", None)
tools = optional_params.pop("tools", None)
# Remove secret_fields to prevent leaking sensitive data (e.g., authorization headers)
optional_params.pop("secret_fields", None)
if functions is not None:
prompt["functions"] = functions
if tools is not None:

7
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -741,6 +741,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "docker_model_runner":
(
api_base,
dynamic_api_key,
) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "v0":
(
api_base,

74
litellm/litellm_core_utils/litellm_logging.py
View file

@ -585,7 +585,10 @@ class Logging(LiteLLMLoggingBaseClass):
custom_logger = (
prompt_management_logger
or self.get_custom_logger_for_prompt_management(
model=model, non_default_params=non_default_params
model=model,
non_default_params=non_default_params,
prompt_id=prompt_id,
dynamic_callback_params=self.standard_callback_dynamic_params,
)
)
@ -622,7 +625,11 @@ class Logging(LiteLLMLoggingBaseClass):
custom_logger = (
prompt_management_logger
or self.get_custom_logger_for_prompt_management(
model=model, tools=tools, non_default_params=non_default_params
model=model,
tools=tools,
non_default_params=non_default_params,
prompt_id=prompt_id,
dynamic_callback_params=self.standard_callback_dynamic_params,
)
)
@ -646,19 +653,69 @@ class Logging(LiteLLMLoggingBaseClass):
self.messages = messages
return model, messages, non_default_params
def _auto_detect_prompt_management_logger(
self,
prompt_id: str,
dynamic_callback_params: StandardCallbackDynamicParams,
) -> Optional[CustomLogger]:
"""
Auto-detect which prompt management system owns the given prompt_id.
This allows a user to just pass prompt_id in the completion call and it will be auto-detected which system owns this prompt.
Args:
prompt_id: The prompt ID to check
dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks
Returns:
A CustomLogger instance if a matching prompt management system is found, None otherwise
"""
prompt_management_loggers = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CustomPromptManagement
)
)
for logger in prompt_management_loggers:
if isinstance(logger, CustomPromptManagement):
try:
if logger.should_run_prompt_management(
prompt_id=prompt_id,
dynamic_callback_params=dynamic_callback_params,
):
self.model_call_details["prompt_integration"] = (
logger.__class__.__name__
)
return logger
except Exception:
# If check fails, continue to next logger
continue
return None
def get_custom_logger_for_prompt_management(
self, model: str, non_default_params: Dict, tools: Optional[List[Dict]] = None
self,
model: str,
non_default_params: Dict,
tools: Optional[List[Dict]] = None,
prompt_id: Optional[str] = None,
dynamic_callback_params: Optional[StandardCallbackDynamicParams] = None,
) -> Optional[CustomLogger]:
"""
Get a custom logger for prompt management based on model name or available callbacks.
Args:
model: The model name to check for prompt management integration
non_default_params: Non-default parameters passed to the completion call
tools: Optional tools passed to the completion call
prompt_id: Optional prompt ID to auto-detect which system owns this prompt
dynamic_callback_params: Dynamic callback parameters for should_run_prompt_management checks
Returns:
A CustomLogger instance if one is found, None otherwise
"""
# First check if model starts with a known custom logger compatible callback
# This takes precedence for backward compatibility
for callback_name in litellm._known_custom_logger_compatible_callbacks:
if model.startswith(callback_name):
custom_logger = _init_custom_logger_compatible_class(
@ -670,7 +727,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["prompt_integration"] = model.split("/")[0]
return custom_logger
# Then check for any registered CustomPromptManagement loggers
# If prompt_id is provided, try to auto-detect which system has this prompt
if prompt_id and dynamic_callback_params is not None:
auto_detected_logger = self._auto_detect_prompt_management_logger(
prompt_id=prompt_id,
dynamic_callback_params=dynamic_callback_params,
)
if auto_detected_logger is not None:
return auto_detected_logger
# Then check for any registered CustomPromptManagement loggers (fallback)
prompt_management_loggers = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CustomPromptManagement

186
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -58,6 +58,10 @@ def prompt_injection_detection_default_pt():
BAD_MESSAGE_ERROR_STR = "Invalid Message "
# Separator used to embed Gemini thought signatures in tool call IDs
# See: https://ai.google.dev/gemini-api/docs/thought-signatures
THOUGHT_SIGNATURE_SEPARATOR = "__thought__"
# used to interweave user messages, to ensure user/assistant alternating
DEFAULT_USER_CONTINUE_MESSAGE = {
"role": "user",
@ -1162,16 +1166,94 @@ def _gemini_tool_call_invoke_helper(
return function_call
def _get_thought_signature_from_tool(tool: dict) -> Optional[str]:
"""Extract thought signature from tool call's provider_specific_fields"""
def _encode_tool_call_id_with_signature(
tool_call_id: str, thought_signature: Optional[str]
) -> str:
"""
Embed thought signature into tool call ID for OpenAI client compatibility.
Args:
tool_call_id: The tool call ID (e.g., "call_abc123...")
thought_signature: Base64-encoded signature from Gemini response
Returns:
Tool call ID with embedded signature if present, otherwise original ID
Format: call_<uuid>__thought__<base64_signature>
See: https://ai.google.dev/gemini-api/docs/thought-signatures
"""
if thought_signature:
return f"{tool_call_id}{THOUGHT_SIGNATURE_SEPARATOR}{thought_signature}"
return tool_call_id
def _get_thought_signature_from_tool(
tool: dict, model: Optional[str] = None
) -> Optional[str]:
"""Extract thought signature from tool call's provider_specific_fields.
If not provided try to extract thought signature from tool call id
Checks both tool.provider_specific_fields and tool.function.provider_specific_fields.
If no signature is found and model is gemini-3, returns a dummy signature.
"""
# First check tool's provider_specific_fields
provider_fields = tool.get("provider_specific_fields") or {}
if isinstance(provider_fields, dict):
return provider_fields.get("thought_signature")
signature = provider_fields.get("thought_signature")
if signature:
return signature
# Then check function's provider_specific_fields
function = tool.get("function")
if function:
if isinstance(function, dict):
func_provider_fields = function.get("provider_specific_fields") or {}
if isinstance(func_provider_fields, dict):
signature = func_provider_fields.get("thought_signature")
if signature:
return signature
elif (
hasattr(function, "provider_specific_fields")
and function.provider_specific_fields
):
if isinstance(function.provider_specific_fields, dict):
signature = function.provider_specific_fields.get("thought_signature")
if signature:
return signature
# Check if thought signature is embedded in tool call ID
tool_call_id = tool.get("id")
if tool_call_id and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id:
parts = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
if len(parts) == 2:
_, signature = parts
return signature
# If no signature found and model is gemini-3, return dummy signature
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
return _get_dummy_thought_signature()
return None
def _get_dummy_thought_signature() -> str:
"""Generate a dummy thought signature for models that require it.
This is used when transferring conversation history from older models
(like gemini-2.5-flash) to gemini-3, which requires thought_signature
for strict validation.
"""
# Return a base64-encoded dummy signature string
# Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
dummy_data = b"skip_thought_signature_validator"
return base64.b64encode(dummy_data).decode("utf-8")
def convert_to_gemini_tool_call_invoke(
message: ChatCompletionAssistantMessage,
model: Optional[str] = None,
) -> List[VertexPartType]:
"""
OpenAI tool invokes:
@ -1216,23 +1298,25 @@ def convert_to_gemini_tool_call_invoke(
_parts_list: List[VertexPartType] = []
tool_calls = message.get("tool_calls", None)
function_call = message.get("function_call", None)
if tool_calls is not None:
for idx, tool in enumerate(tool_calls):
if "function" in tool:
gemini_function_call: Optional[VertexFunctionCall] = (
_gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
)
gemini_function_call: Optional[
VertexFunctionCall
] = _gemini_tool_call_invoke_helper(
function_call_params=tool["function"]
)
if gemini_function_call is not None:
part_dict: VertexPartType = {
"function_call": gemini_function_call
}
thought_signature = _get_thought_signature_from_tool(dict(tool))
thought_signature = _get_thought_signature_from_tool(
dict(tool), model=model
)
if thought_signature:
part_dict["thoughtSignature"] = thought_signature
_parts_list.append(part_dict)
else: # don't silently drop params. Make it clear to user what's happening.
raise Exception(
@ -1248,14 +1332,32 @@ def convert_to_gemini_tool_call_invoke(
part_dict_function: VertexPartType = {
"function_call": gemini_function_call
}
# Extract thought signature from function_call's provider_specific_fields
provider_fields = function_call.get("provider_specific_fields") if isinstance(function_call, dict) else {}
thought_signature = None
provider_fields = (
function_call.get("provider_specific_fields")
if isinstance(function_call, dict)
else {}
)
if isinstance(provider_fields, dict):
thought_signature = provider_fields.get("thought_signature")
if thought_signature:
part_dict_function["thoughtSignature"] = thought_signature
# If no signature found and model is gemini-3, use dummy signature
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
if (
not thought_signature
and model
and VertexGeminiConfig._is_gemini_3_or_newer(model)
):
thought_signature = _get_dummy_thought_signature()
if thought_signature:
part_dict_function["thoughtSignature"] = thought_signature
_parts_list.append(part_dict_function)
else: # don't silently drop params. Make it clear to user what's happening.
raise Exception(
@ -1758,9 +1860,9 @@ def anthropic_messages_pt( # noqa: PLR0915
)
if "cache_control" in _content_element:
_anthropic_content_element["cache_control"] = (
_content_element["cache_control"]
)
_anthropic_content_element[
"cache_control"
] = _content_element["cache_control"]
user_content.append(_anthropic_content_element)
elif m.get("type", "") == "text":
m = cast(ChatCompletionTextObject, m)
@ -1798,9 +1900,9 @@ def anthropic_messages_pt( # noqa: PLR0915
)
if "cache_control" in _content_element:
_anthropic_content_text_element["cache_control"] = (
_content_element["cache_control"]
)
_anthropic_content_text_element[
"cache_control"
] = _content_element["cache_control"]
user_content.append(_anthropic_content_text_element)
@ -2566,17 +2668,19 @@ class BedrockImageProcessor:
"""Handles both sync and async image processing for Bedrock conversations."""
@staticmethod
def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> Tuple[str, str]:
def _post_call_image_processing(
response: httpx.Response, image_url: str = ""
) -> Tuple[str, str]:
# Check the response's content type to ensure it is an image
content_type = response.headers.get("content-type")
# Use helper function to infer content type with fallback logic
content_type = infer_content_type_from_url_and_content(
url=image_url,
content=response.content,
current_content_type=content_type,
)
content_type = _parse_content_type(content_type)
# Convert the image content to base64 bytes
@ -2595,7 +2699,9 @@ class BedrockImageProcessor:
response = await client.get(image_url, follow_redirects=True)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
return BedrockImageProcessor._post_call_image_processing(
response, image_url
)
except Exception as e:
raise e
@ -2608,7 +2714,9 @@ class BedrockImageProcessor:
response = client.get(image_url, follow_redirects=True)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
return BedrockImageProcessor._post_call_image_processing(
response, image_url
)
except Exception as e:
raise e
@ -2939,21 +3047,33 @@ def _convert_to_bedrock_tool_call_result(
"""
-
"""
content_str: str = ""
tool_result_content_blocks:List[BedrockToolResultContentBlock] = []
if isinstance(message["content"], str):
content_str = message["content"]
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"]))
elif isinstance(message["content"], List):
content_list = message["content"]
for content in content_list:
if content["type"] == "text":
content_str += content["text"]
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"]))
elif content["type"] == "image_url":
format: Optional[str] = None
if isinstance(content["image_url"], dict):
image_url = content["image_url"]["url"]
format = content["image_url"].get("format")
else:
image_url = content["image_url"]
_block:BedrockContentBlock = BedrockImageProcessor.process_image_sync(
image_url=image_url,
format=format,
)
if "image" in _block:
tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"]))
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))
tool_result_content_block = BedrockToolResultContentBlock(text=content_str)
tool_result = BedrockToolResultBlock(
content=[tool_result_content_block],
content=tool_result_content_blocks,
toolUseId=id,
)
@ -3865,7 +3985,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
)
elif element["type"] == "text":
# AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings
text_content = element["text"] if element["text"].strip() else "."
text_content = (
element["text"] if element["text"].strip() else "."
)
assistants_part = BedrockContentBlock(text=text_content)
assistants_parts.append(assistants_part)
elif element["type"] == "image_url":

45
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -137,6 +137,7 @@ class ChunkProcessor:
"name": None,
"type": None,
"arguments": [],
"provider_specific_fields": None,
}
if hasattr(tool_call, "id") and tool_call.id:
@ -156,22 +157,48 @@ class ChunkProcessor:
tool_call_map[index]["arguments"].append(
tool_call.function.arguments
)
# Preserve provider_specific_fields from streaming chunks
provider_fields = None
if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields:
provider_fields = tool_call.provider_specific_fields
elif hasattr(tool_call, "function") and hasattr(tool_call.function, "provider_specific_fields") and tool_call.function.provider_specific_fields:
provider_fields = tool_call.function.provider_specific_fields
if provider_fields:
# Merge provider_specific_fields if multiple chunks have them
if tool_call_map[index]["provider_specific_fields"] is None:
tool_call_map[index]["provider_specific_fields"] = {}
if isinstance(provider_fields, dict):
tool_call_map[index]["provider_specific_fields"].update(
provider_fields
)
# Convert the map to a list of tool calls
for index in sorted(tool_call_map.keys()):
tool_call_data = tool_call_map[index]
if tool_call_data["id"] and tool_call_data["name"]:
combined_arguments = "".join(tool_call_data["arguments"]) or "{}"
tool_calls_list.append(
ChatCompletionMessageToolCall(
id=tool_call_data["id"],
function=Function(
arguments=combined_arguments,
name=tool_call_data["name"],
),
type=tool_call_data["type"] or "function",
)
# Build function - provider_specific_fields should be on tool_call level, not function level
function = Function(
arguments=combined_arguments,
name=tool_call_data["name"],
)
# Prepare params for ChatCompletionMessageToolCall
tool_call_params = {
"id": tool_call_data["id"],
"function": function,
"type": tool_call_data["type"] or "function",
}
# Add provider_specific_fields if present (for thought signatures in Gemini 3)
if tool_call_data.get("provider_specific_fields"):
tool_call_params["provider_specific_fields"] = tool_call_data["provider_specific_fields"]
tool_call = ChatCompletionMessageToolCall(**tool_call_params)
tool_calls_list.append(tool_call)
return tool_calls_list

81
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -3,6 +3,7 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Dict,
List,
Literal,
Optional,
@ -129,6 +130,39 @@ class LiteLLMAnthropicMessagesAdapter:
### FOR [BETA] `/v1/messages` endpoint support
def _extract_signature_from_tool_call(
self, tool_call: Any
) -> Optional[str]:
"""
Extract signature from a tool call's provider_specific_fields.
Only checks provider_specific_fields, not thinking blocks.
"""
signature = None
if hasattr(tool_call, "provider_specific_fields") and tool_call.provider_specific_fields:
if "thought_signature" in tool_call.provider_specific_fields:
signature = tool_call.provider_specific_fields["thought_signature"]
elif (
hasattr(tool_call.function, "provider_specific_fields")
and tool_call.function.provider_specific_fields
):
if "thought_signature" in tool_call.function.provider_specific_fields:
signature = tool_call.function.provider_specific_fields["thought_signature"]
return signature
def _extract_signature_from_tool_use_content(
self, content: Dict[str, Any]
) -> Optional[str]:
"""
Extract signature from a tool_use content block's provider_specific_fields.
"""
provider_specific_fields = content.get("provider_specific_fields", {})
if provider_specific_fields:
return provider_specific_fields.get("signature")
return None
def translatable_anthropic_params(self) -> List:
"""
Which anthropic params, we need to translate to the openai format.
@ -263,10 +297,18 @@ class LiteLLMAnthropicMessagesAdapter:
else:
assistant_message_str += content.get("text", "")
elif content.get("type") == "tool_use":
function_chunk = ChatCompletionToolCallFunctionChunk(
name=content.get("name", ""),
arguments=json.dumps(content.get("input", {})),
)
function_chunk: ChatCompletionToolCallFunctionChunk = {
"name": content.get("name", ""),
"arguments": json.dumps(content.get("input", {})),
}
signature = self._extract_signature_from_tool_use_content(content)
if signature:
provider_specific_fields: Dict[str, Any] = (
function_chunk.get("provider_specific_fields") or {}
)
provider_specific_fields["thought_signature"] = signature
function_chunk["provider_specific_fields"] = provider_specific_fields
tool_calls.append(
ChatCompletionAssistantToolCall(
@ -512,18 +554,27 @@ class LiteLLMAnthropicMessagesAdapter:
and len(choice.message.tool_calls) > 0
):
for tool_call in choice.message.tool_calls:
new_content.append(
AnthropicResponseContentBlockToolUse(
type="tool_use",
id=tool_call.id,
name=tool_call.function.name or "",
input=(
json.loads(tool_call.function.arguments)
if tool_call.function.arguments
else {}
),
)
# Extract signature from provider_specific_fields only
signature = self._extract_signature_from_tool_call(tool_call)
provider_specific_fields = {}
if signature:
provider_specific_fields["signature"] = signature
tool_use_block = AnthropicResponseContentBlockToolUse(
type="tool_use",
id=tool_call.id,
name=tool_call.function.name or "",
input=(
json.loads(tool_call.function.arguments)
if tool_call.function.arguments
else {}
),
)
# Add provider_specific_fields if signature is present
if provider_specific_fields:
tool_use_block.provider_specific_fields = provider_specific_fields
new_content.append(tool_use_block)
# Handle text content
elif choice.message.content is not None:
new_content.append(

272
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -51,7 +51,11 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallFunctionChunk,
ChatCompletionUsageBlock,
)
from litellm.types.utils import ChatCompletionMessageToolCall, Choices, Delta
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Delta,
)
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import (
ModelResponse,
@ -493,9 +497,9 @@ class BedrockLLM(BaseAWSLLM):
content=None,
)
model_response.choices[0].message = _message # type: ignore
model_response._hidden_params["original_response"] = (
outputText # allow user to access raw anthropic tool calling response
)
model_response._hidden_params[
"original_response"
] = outputText # allow user to access raw anthropic tool calling response
if (
_is_function_call is True
and stream is not None
@ -793,9 +797,9 @@ class BedrockLLM(BaseAWSLLM):
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
inference_params[k] = v
if stream is True:
inference_params["stream"] = (
True # cohere requires stream = True in inference params
)
inference_params[
"stream"
] = True # cohere requires stream = True in inference params
data = json.dumps({"prompt": prompt, **inference_params})
elif provider == "anthropic":
if model.startswith("anthropic.claude-3"):
@ -1184,6 +1188,7 @@ class AWSEventStreamDecoder:
self.parser = EventStreamJSONParser()
self.content_blocks: List[ContentBlockDeltaEvent] = []
self.tool_calls_index: Optional[int] = None
self.response_id: Optional[str] = None
def check_empty_tool_call_args(self) -> bool:
"""
@ -1245,8 +1250,169 @@ class AWSEventStreamDecoder:
thinking_blocks_list.append(_thinking_block)
return thinking_blocks_list
def _initialize_converse_response_id(self, chunk_data: dict):
"""Initialize response_id from chunk data if not already set."""
if self.response_id is None:
if "messageStart" in chunk_data:
conversation_id = chunk_data["messageStart"].get("conversationId")
if conversation_id:
self.response_id = f"chatcmpl-{conversation_id}"
else:
# Fallback to generating a UUID if the first chunk is not messageStart
self.response_id = f"chatcmpl-{uuid.uuid4()}"
def _handle_converse_start_event(
self,
start_obj: ContentBlockStartEvent,
) -> Tuple[
Optional[ChatCompletionToolCallChunk],
dict,
Optional[
List[
Union[
ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
]
]
],
]:
"""Handle 'start' event in converse chunk parsing."""
tool_use: Optional[ChatCompletionToolCallChunk] = None
provider_specific_fields: dict = {}
thinking_blocks: Optional[
List[
Union[
ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
]
]
] = None
self.content_blocks = [] # reset
if start_obj is not None:
if "toolUse" in start_obj and start_obj["toolUse"] is not None:
## check tool name was formatted by litellm
_response_tool_name = start_obj["toolUse"]["name"]
response_tool_name = get_bedrock_tool_name(
response_tool_name=_response_tool_name
)
self.tool_calls_index = (
0
if self.tool_calls_index is None
else self.tool_calls_index + 1
)
tool_use = {
"id": start_obj["toolUse"]["toolUseId"],
"type": "function",
"function": {
"name": response_tool_name,
"arguments": "",
},
"index": self.tool_calls_index,
}
elif (
"reasoningContent" in start_obj
and start_obj["reasoningContent"] is not None
): # redacted thinking can be in start object
thinking_blocks = self.translate_thinking_blocks(
start_obj["reasoningContent"]
)
provider_specific_fields = {
"reasoningContent": start_obj["reasoningContent"],
}
return tool_use, provider_specific_fields, thinking_blocks
def _handle_converse_delta_event(
self,
delta_obj: ContentBlockDeltaEvent,
index: int,
) -> Tuple[
str,
Optional[ChatCompletionToolCallChunk],
dict,
Optional[str],
Optional[
List[
Union[
ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
]
]
],
]:
"""Handle 'delta' event in converse chunk parsing."""
text = ""
tool_use: Optional[ChatCompletionToolCallChunk] = None
provider_specific_fields: dict = {}
reasoning_content: Optional[str] = None
thinking_blocks: Optional[
List[
Union[
ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock
]
]
] = None
self.content_blocks.append(delta_obj)
if "text" in delta_obj:
text = delta_obj["text"]
elif "toolUse" in delta_obj:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "reasoningContent" in delta_obj:
provider_specific_fields = {
"reasoningContent": delta_obj["reasoningContent"],
}
reasoning_content = self.extract_reasoning_content_str(
delta_obj["reasoningContent"]
)
thinking_blocks = self.translate_thinking_blocks(
delta_obj["reasoningContent"]
)
if (
thinking_blocks
and len(thinking_blocks) > 0
and reasoning_content is None
):
reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic
return text, tool_use, provider_specific_fields, reasoning_content, thinking_blocks
def _handle_converse_stop_event(
self, index: int
) -> Optional[ChatCompletionToolCallChunk]:
"""Handle stop/contentBlockIndex event in converse chunk parsing."""
tool_use: Optional[ChatCompletionToolCallChunk] = None
is_empty = self.check_empty_tool_call_args()
if is_empty:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": "{}",
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
return tool_use
def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream:
try:
# Capture the conversationId from the first messageStart event
# and use it as the consistent ID for all subsequent chunks.
self._initialize_converse_response_id(chunk_data)
verbose_logger.debug("\n\nRaw Chunk: {}\n\n".format(chunk_data))
text = ""
tool_use: Optional[ChatCompletionToolCallChunk] = None
@ -1265,91 +1431,22 @@ class AWSEventStreamDecoder:
index = int(chunk_data.get("contentBlockIndex", 0))
if "start" in chunk_data:
start_obj = ContentBlockStartEvent(**chunk_data["start"])
self.content_blocks = [] # reset
if start_obj is not None:
if "toolUse" in start_obj and start_obj["toolUse"] is not None:
## check tool name was formatted by litellm
_response_tool_name = start_obj["toolUse"]["name"]
response_tool_name = get_bedrock_tool_name(
response_tool_name=_response_tool_name
)
self.tool_calls_index = (
0
if self.tool_calls_index is None
else self.tool_calls_index + 1
)
tool_use = {
"id": start_obj["toolUse"]["toolUseId"],
"type": "function",
"function": {
"name": response_tool_name,
"arguments": "",
},
"index": self.tool_calls_index,
}
elif (
"reasoningContent" in start_obj
and start_obj["reasoningContent"] is not None
): # redacted thinking can be in start object
thinking_blocks = self.translate_thinking_blocks(
start_obj["reasoningContent"]
)
provider_specific_fields = {
"reasoningContent": start_obj["reasoningContent"],
}
tool_use, provider_specific_fields, thinking_blocks = (
self._handle_converse_start_event(start_obj)
)
elif "delta" in chunk_data:
delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"])
self.content_blocks.append(delta_obj)
if "text" in delta_obj:
text = delta_obj["text"]
elif "toolUse" in delta_obj:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "reasoningContent" in delta_obj:
provider_specific_fields = {
"reasoningContent": delta_obj["reasoningContent"],
}
reasoning_content = self.extract_reasoning_content_str(
delta_obj["reasoningContent"]
)
thinking_blocks = self.translate_thinking_blocks(
delta_obj["reasoningContent"]
)
if (
thinking_blocks
and len(thinking_blocks) > 0
and reasoning_content is None
):
reasoning_content = "" # set to non-empty string to ensure consistency with Anthropic
(
text,
tool_use,
provider_specific_fields,
reasoning_content,
thinking_blocks,
) = self._handle_converse_delta_event(delta_obj, index)
elif (
"contentBlockIndex" in chunk_data
): # stop block, no 'start' or 'delta' object
is_empty = self.check_empty_tool_call_args()
if is_empty:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": "{}",
},
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
tool_use = self._handle_converse_stop_event(index)
elif "stopReason" in chunk_data:
finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
elif "usage" in chunk_data:
@ -1378,6 +1475,7 @@ class AWSEventStreamDecoder:
),
)
],
id=self.response_id,
usage=usage,
provider_specific_fields=model_response_provider_specific_fields,
)

4
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1140,6 +1140,7 @@ class BaseLLMHTTPHandler:
atranscription: bool = False,
headers: Optional[Dict[str, Any]] = None,
provider_config: Optional[BaseAudioTranscriptionConfig] = None,
shared_session: Optional["ClientSession"] = None,
) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]:
if provider_config is None:
raise ValueError(
@ -1162,6 +1163,7 @@ class BaseLLMHTTPHandler:
client=client,
headers=headers,
provider_config=provider_config,
shared_session=shared_session,
)
# Prepare the request
@ -1226,6 +1228,7 @@ class BaseLLMHTTPHandler:
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
headers: Optional[Dict[str, Any]] = None,
provider_config: Optional[BaseAudioTranscriptionConfig] = None,
shared_session: Optional["ClientSession"] = None,
) -> TranscriptionResponse:
if provider_config is None:
raise ValueError(
@ -1254,6 +1257,7 @@ class BaseLLMHTTPHandler:
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client

144
litellm/llms/docker_model_runner/chat/transformation.py Normal file
View file

@ -0,0 +1,144 @@
"""
Translates from OpenAI's `/v1/chat/completions` to Docker Model Runner's `/engines/{engine}/v1/chat/completions`
Docker Model Runner API Reference: https://docs.docker.com/ai/model-runner/api-reference/
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class DockerModelRunnerChatConfig(OpenAIGPTConfig):
"""
Configuration for Docker Model Runner API.
Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions
The engine name (e.g., "llama.cpp") is part of the API endpoint path.
"""
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
def _transform_messages(
self,
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
"""
Docker Model Runner is OpenAI-compatible, so we use standard message transformation.
"""
messages = handle_messages_with_content_list_to_str_conversion(messages)
if is_async:
return super()._transform_messages(
messages=messages, model=model, is_async=True
)
else:
return super()._transform_messages(
messages=messages, model=model, is_async=False
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
"""
Get API base and key for Docker Model Runner.
Default API base: http://localhost:22088/engines/llama.cpp
The engine path should be included in the api_base.
"""
api_base = (
api_base
or get_secret_str("DOCKER_MODEL_RUNNER_API_BASE")
or "http://localhost:22088/engines/llama.cpp"
) # type: ignore
# Docker Model Runner may not require authentication for local instances
dynamic_api_key = api_key or get_secret_str("DOCKER_MODEL_RUNNER_API_KEY") or "dummy-key"
return api_base, dynamic_api_key
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Build the complete URL for Docker Model Runner API.
Docker Model Runner uses URLs in the format: /engines/{engine}/v1/chat/completions
The engine name should be specified in the api_base:
- api_base="http://model-runner.docker.internal/engines/llama.cpp"
- Default: "http://localhost:22088/engines/llama.cpp"
Args:
api_base: Base URL for the Docker Model Runner instance including engine path
api_key: API key (may not be required for local instances)
model: Model name (e.g., "llama-3.1")
optional_params: Optional parameters
litellm_params: LiteLLM parameters
stream: Whether streaming is enabled
Returns:
Complete URL for the API call
"""
if not api_base:
api_base = "http://localhost:22088/engines/llama.cpp"
# Remove trailing slashes from api_base
api_base = api_base.rstrip("/")
# Build the URL: {api_base}/v1/chat/completions
# api_base is expected to already contain the engine path
complete_url = f"{api_base}/v1/chat/completions"
return complete_url
def get_supported_openai_params(self, model: str) -> list:
"""
Get the supported OpenAI params for Docker Model Runner.
Docker Model Runner is OpenAI-compatible and supports standard parameters.
"""
return super().get_supported_openai_params(model=model)
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to Docker Model Runner parameters.
Docker Model Runner is OpenAI-compatible, so most parameters map directly.
"""
supported_openai_params = self.get_supported_openai_params(model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params

2
litellm/llms/gemini/chat/transformation.py
View file

@ -140,4 +140,4 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
except Exception:
# If conversion fails, leave as is and let the API handle it
pass
return _gemini_convert_messages_with_history(messages=messages)
return _gemini_convert_messages_with_history(messages=messages, model=model)

5
litellm/llms/gemini/videos/transformation.py
View file

@ -15,17 +15,16 @@ from litellm.images.utils import ImageEditRequestUtils
import litellm
from litellm.types.llms.gemini import GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest
from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig
from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException
LiteLLMLoggingObj = _LiteLLMLoggingObj
BaseVideoConfig = _BaseVideoConfig
BaseLLMException = _BaseLLMException
else:
LiteLLMLoggingObj = Any
BaseVideoConfig = Any
BaseLLMException = Any

317
litellm/llms/github_copilot/responses/transformation.py Normal file
View file

@ -0,0 +1,317 @@
"""
GitHub Copilot Responses API Configuration.
This module provides the configuration for GitHub Copilot's Responses API,
which is required for models like gpt-5.1-codex that only support the /responses endpoint.
Implementation based on analysis of the copilot-api project by caozhiyuan:
https://github.com/caozhiyuan/copilot-api
"""
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from uuid import uuid4
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.exceptions import AuthenticationError
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..authenticator import Authenticator
from ..common_utils import GetAPIKeyError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
# GitHub Copilot API Constants (from copilot-api)
COPILOT_VERSION = "0.26.7"
EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}"
USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}"
API_VERSION = "2025-04-01"
class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
Configuration for GitHub Copilot's Responses API.
Inherits from OpenAIResponsesAPIConfig since GitHub Copilot's Responses API
is compatible with OpenAI's Responses API specification.
Key differences from OpenAI:
- Uses OAuth Device Flow authentication (handled by Authenticator)
- Uses api.githubcopilot.com as the API base
- Requires specific headers for VSCode/Copilot integration
- Supports vision requests with special header
- Requires X-Initiator header based on input analysis
Reference: https://api.githubcopilot.com/
"""
GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com"
def __init__(self) -> None:
super().__init__()
self.authenticator = Authenticator()
@property
def custom_llm_provider(self) -> LlmProviders:
"""Return the GitHub Copilot provider identifier."""
return LlmProviders.GITHUB_COPILOT
def get_supported_openai_params(self, model: str) -> list:
"""
Get supported parameters for GitHub Copilot Responses API.
GitHub Copilot supports all standard OpenAI Responses API parameters.
"""
return super().get_supported_openai_params(model)
def map_openai_params(
self,
response_api_optional_params: ResponsesAPIOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""
Map parameters for GitHub Copilot Responses API.
GitHub Copilot uses the same parameter format as OpenAI,
so no transformation is needed.
"""
return dict(response_api_optional_params)
def validate_environment(
self,
headers: dict,
model: str,
litellm_params: Optional[GenericLiteLLMParams],
) -> dict:
"""
Validate environment and set up headers for GitHub Copilot API.
Uses the Authenticator to obtain GitHub Copilot API key via OAuth Device Flow,
then configures all required headers for the Responses API.
Headers include:
- Authorization with API key
- Standard GitHub Copilot headers (editor-version, user-agent, etc.)
- X-Initiator based on input analysis
- copilot-vision-request if vision content detected
- User-provided extra_headers (merged with priority)
"""
try:
# Get GitHub Copilot API key via OAuth
api_key = self.authenticator.get_api_key()
if not api_key:
raise AuthenticationError(
model=model,
llm_provider="github_copilot",
message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.",
)
# Get default headers (from copilot-api configuration)
default_headers = self._get_default_headers(api_key)
# Merge with existing headers (user's extra_headers take priority)
merged_headers = {**default_headers, **headers}
# Analyze input to determine additional headers
input_param = self._get_input_from_params(litellm_params)
# Add X-Initiator header based on input analysis
if input_param is not None:
initiator = self._get_initiator(input_param)
merged_headers["X-Initiator"] = initiator
verbose_logger.debug(
f"GitHub Copilot Responses API: Set X-Initiator={initiator}"
)
# Add vision header if input contains images
if self._has_vision_input(input_param):
merged_headers["copilot-vision-request"] = "true"
verbose_logger.debug(
"GitHub Copilot Responses API: Enabled vision request"
)
verbose_logger.debug(
f"GitHub Copilot Responses API: Successfully configured headers for model {model}"
)
return merged_headers
except GetAPIKeyError as e:
raise AuthenticationError(
model=model,
llm_provider="github_copilot",
message=str(e),
)
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for GitHub Copilot Responses API endpoint.
Returns: https://api.githubcopilot.com/responses
Note: Currently only supports individual accounts.
Business/enterprise accounts (api.business.githubcopilot.com) can be
added in the future by detecting account type.
"""
# Use provided api_base or fall back to authenticator's base or default
api_base = (
api_base
or self.authenticator.get_api_base()
or self.GITHUB_COPILOT_API_BASE
)
# Remove trailing slashes
api_base = api_base.rstrip("/")
# Return the responses endpoint
return f"{api_base}/responses"
# ==================== Helper Methods ====================
def _get_default_headers(self, api_key: str) -> Dict[str, str]:
"""
Get default headers for GitHub Copilot Responses API.
Based on copilot-api's header configuration.
"""
return {
"Authorization": f"Bearer {api_key}",
"content-type": "application/json",
"copilot-integration-id": "vscode-chat",
"editor-version": "vscode/1.95.0", # Fixed version for stability
"editor-plugin-version": EDITOR_PLUGIN_VERSION,
"user-agent": USER_AGENT,
"openai-intent": "conversation-panel",
"x-github-api-version": API_VERSION,
"x-request-id": str(uuid4()),
"x-vscode-user-agent-library-version": "electron-fetch",
}
def _get_input_from_params(
self, litellm_params: Optional[GenericLiteLLMParams]
) -> Optional[Union[str, ResponseInputParam]]:
"""
Extract input parameter from litellm_params.
The input parameter contains the conversation history and is needed
for vision detection and initiator determination.
"""
if litellm_params is None:
return None
# Try to get input from litellm_params
# This might be in different locations depending on how LiteLLM structures it
if hasattr(litellm_params, "input"):
return litellm_params.input
# If not found, return None and let the API handle it
return None
def _get_initiator(self, input_param: Union[str, ResponseInputParam]) -> str:
"""
Determine X-Initiator header value based on input analysis.
Based on copilot-api's hasAgentInitiator logic:
- Returns "agent" if input contains assistant role or items without role
- Returns "user" otherwise
Args:
input_param: The input parameter (string or list of input items)
Returns:
"agent" or "user"
"""
# If input is a string, it's user-initiated
if isinstance(input_param, str):
return "user"
# If input is a list, analyze items
if isinstance(input_param, list):
for item in input_param:
if not isinstance(item, dict):
continue
# Check if item has no role (agent-initiated)
if "role" not in item or not item.get("role"):
return "agent"
# Check if role is assistant (agent-initiated)
role = item.get("role")
if isinstance(role, str) and role.lower() == "assistant":
return "agent"
# Default to user-initiated
return "user"
def _has_vision_input(self, input_param: Union[str, ResponseInputParam]) -> bool:
"""
Check if input contains vision content (images).
Based on copilot-api's hasVisionInput and containsVisionContent logic.
Recursively searches for input_image type in the input structure.
Args:
input_param: The input parameter to analyze
Returns:
True if input contains image content, False otherwise
"""
return self._contains_vision_content(input_param)
def _contains_vision_content(
self, value: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH
) -> bool:
"""
Recursively check if a value contains vision content.
Looks for items with type="input_image" in the structure.
"""
if depth > max_depth:
verbose_logger.warning(
f"[GitHub Copilot] Max recursion depth {max_depth} reached while checking for vision content"
)
return False
if value is None:
return False
# Check arrays
if isinstance(value, list):
return any(
self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth)
for item in value
)
# Only check dict/object types
if not isinstance(value, dict):
return False
# Check if this item is an input_image
item_type = value.get("type")
if isinstance(item_type, str) and item_type.lower() == "input_image":
return True
# Check content field recursively
if "content" in value and isinstance(value["content"], list):
return any(
self._contains_vision_content(item, depth=depth + 1, max_depth=max_depth)
for item in value["content"]
)
return False

9
litellm/llms/openai/transcriptions/handler.py
View file

@ -1,10 +1,13 @@
from typing import Optional, Union, cast
from typing import TYPE_CHECKING, Optional, Union, cast
import httpx
from openai import AsyncOpenAI, OpenAI
from pydantic import BaseModel
import litellm
if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.audio_transcription.transformation import (
@ -89,6 +92,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
client=None,
atranscription: bool = False,
provider_config: Optional[BaseAudioTranscriptionConfig] = None,
shared_session: Optional["ClientSession"] = None,
) -> TranscriptionResponse:
"""
Handle audio transcription request
@ -116,6 +120,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
client=client,
max_retries=max_retries,
logging_obj=logging_obj,
shared_session=shared_session,
)
openai_client: OpenAI = self._get_openai_client( # type: ignore
@ -170,6 +175,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
api_base: Optional[str] = None,
client=None,
max_retries=None,
shared_session: Optional["ClientSession"] = None,
):
try:
openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore
@ -179,6 +185,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
timeout=timeout,
max_retries=max_retries,
client=client,
shared_session=shared_session,
)
## LOGGING

2
litellm/llms/vertex_ai/context_caching/transformation.py
View file

@ -173,7 +173,7 @@ def transform_openai_messages_to_gemini_context_caching(
supports_system_message=supports_system_message, messages=messages
)
transformed_messages = _gemini_convert_messages_with_history(messages=new_messages)
transformed_messages = _gemini_convert_messages_with_history(messages=new_messages, model=model)
model_name = "models/{}".format(model)

3
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -195,6 +195,7 @@ def check_if_part_exists_in_parts(
def _gemini_convert_messages_with_history( # noqa: PLR0915
messages: List[AllMessageValues],
model: Optional[str] = None,
) -> List[ContentType]:
"""
Converts given messages from OpenAI format to Gemini format
@ -379,7 +380,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
or assistant_msg.get("function_call") is not None
): # support assistant tool invoke conversion
gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
assistant_msg
assistant_msg, model=model
)
## check if gemini_tool_call already exists in assistant_content
for gemini_tool_call_part in gemini_tool_call_parts:

200
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -81,6 +81,9 @@ from litellm.types.utils import (
TopLogprob,
Usage,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
_encode_tool_call_id_with_signature,
)
from litellm.utils import (
CustomStreamWrapper,
ModelResponse,
@ -217,12 +220,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
@classmethod
def get_config(cls):
return super().get_config()
@staticmethod
def _is_gemini_3_or_newer(model: str) -> bool:
"""
Check if the model is Gemini 3 Pro or newer.
Gemini 3 models include:
- gemini-3-pro-preview
- Any future Gemini 3.x models
@ -230,7 +233,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# Check for Gemini 3 models
if "gemini-3" in model:
return True
return False
def _supports_penalty_parameters(self, model: str) -> bool:
@ -260,11 +263,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"parallel_tool_calls",
"web_search_options",
]
# Add penalty parameters only for non-preview models
if self._supports_penalty_parameters(model):
supported_params.extend(["frequency_penalty", "presence_penalty"])
if supports_reasoning(model):
supported_params.append("reasoning_effort")
supported_params.append("thinking")
@ -308,14 +311,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
) -> Tuple[dict, Optional[dict]]:
"""
Extract location configuration from googleMaps tool for Vertex AI toolConfig.
Supports two interface styles:
1. Nested (recommended): {"enableWidget": "...", "retrievalConfig": {"latitude": ..., "longitude": ...}}
2. Flat (backward compat): {"enableWidget": "...", "latitude": ..., "longitude": ...}
Args:
google_maps_config: The googleMaps tool configuration from LiteLLM
Returns:
Tuple of (cleaned_google_maps_config, retrieval_config):
- cleaned_google_maps_config: googleMaps config without location fields
@ -325,7 +328,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
latitude = google_maps_config.get("latitude")
longitude = google_maps_config.get("longitude")
language_code = google_maps_config.get("languageCode")
if latitude is not None and longitude is not None:
retrieval_config = {
"latLng": {
@ -335,21 +338,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
}
if language_code is not None:
retrieval_config["languageCode"] = language_code
# Remove location fields from tool definition
cleaned_config = {
k: v
for k, v in google_maps_config.items()
if k not in ["latitude", "longitude", "languageCode"]
}
return cleaned_config, retrieval_config
def get_tool_value(
self,
tool: dict,
tool_name: str
) -> Optional[dict]:
def get_tool_value(self, tool: dict, tool_name: str) -> Optional[dict]:
"""
Helper function to get tool value handling both camelCase and underscore_case variants
@ -373,19 +372,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
return None
def _map_function( # noqa: PLR0915
def _map_function( # noqa: PLR0915
self, value: List[dict], optional_params: dict
) -> List[Tools]:
"""
Map OpenAI-style tools/functions to Vertex AI format.
Args:
value: List of tool definitions
optional_params: Request-scoped parameters to store retrieval config
Returns:
List of mapped tools in Vertex AI format
Side effects:
May add 'toolConfig' with 'retrievalConfig' to optional_params if
googleMaps tools contain location data
@ -432,25 +431,43 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
tool_name = list(tool.keys())[0] if len(tool.keys()) == 1 else None
if tool_name and (
tool_name == "codeExecution" or tool_name == VertexToolName.CODE_EXECUTION.value
tool_name == "codeExecution"
or tool_name == VertexToolName.CODE_EXECUTION.value
): # code_execution maintained for backwards compatibility
code_execution = self.get_tool_value(tool, "codeExecution")
elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH.value:
googleSearch = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH.value)
elif tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value:
googleSearchRetrieval = self.get_tool_value(tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value)
googleSearch = self.get_tool_value(
tool, VertexToolName.GOOGLE_SEARCH.value
)
elif (
tool_name and tool_name == VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value
):
googleSearchRetrieval = self.get_tool_value(
tool, VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value
)
elif tool_name and tool_name == VertexToolName.ENTERPRISE_WEB_SEARCH.value:
enterpriseWebSearch = self.get_tool_value(tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value)
elif tool_name and (tool_name == VertexToolName.URL_CONTEXT.value or tool_name == "urlContext"):
enterpriseWebSearch = self.get_tool_value(
tool, VertexToolName.ENTERPRISE_WEB_SEARCH.value
)
elif tool_name and (
tool_name == VertexToolName.URL_CONTEXT.value
or tool_name == "urlContext"
):
urlContext = self.get_tool_value(tool, tool_name)
elif tool_name and (
tool_name == VertexToolName.GOOGLE_MAPS.value or tool_name == "google_maps"
tool_name == VertexToolName.GOOGLE_MAPS.value
or tool_name == "google_maps"
):
google_maps_value = self.get_tool_value(tool, VertexToolName.GOOGLE_MAPS.value)
google_maps_value = self.get_tool_value(
tool, VertexToolName.GOOGLE_MAPS.value
)
# Extract and transform location configuration for toolConfig
if google_maps_value is not None:
googleMaps, google_maps_retrieval_config = self._extract_google_maps_retrieval_config(
(
googleMaps,
google_maps_retrieval_config,
) = self._extract_google_maps_retrieval_config(
google_maps_config=google_maps_value
)
elif openai_function_object is not None:
@ -490,13 +507,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
_tools[VertexToolName.URL_CONTEXT.value] = urlContext
if googleMaps is not None:
_tools[VertexToolName.GOOGLE_MAPS.value] = googleMaps
# Add retrieval config to toolConfig if googleMaps has location data
if google_maps_retrieval_config is not None:
if "toolConfig" not in optional_params:
optional_params["toolConfig"] = {}
optional_params["toolConfig"]["retrievalConfig"] = google_maps_retrieval_config
optional_params["toolConfig"][
"retrievalConfig"
] = google_maps_retrieval_config
return [_tools]
def _map_response_schema(self, value: dict) -> dict:
@ -599,23 +618,27 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Map reasoning_effort to thinking_level for Gemini 3+ models.
Args:
reasoning_effort: The reasoning effort value
model: The model name (for validation, currently unused but kept for consistency)
model: The model name
Returns:
GeminiThinkingConfig with thinkingLevel set
GeminiThinkingConfig with thinkingLevel and includeThoughts
"""
if reasoning_effort == "minimal":
return {"thinkingLevel": "low"}
return {"thinkingLevel": "low", "includeThoughts": True}
elif reasoning_effort == "low":
return {"thinkingLevel": "low"}
return {"thinkingLevel": "low", "includeThoughts": True}
elif reasoning_effort == "medium":
return {"thinkingLevel": "high"} # medium is not out yet
return {
"thinkingLevel": "high",
"includeThoughts": True,
} # medium is not out yet
elif reasoning_effort == "high":
return {"thinkingLevel": "high"}
return {"thinkingLevel": "high", "includeThoughts": True}
elif reasoning_effort == "disable":
return {"thinkingLevel": "low"} # gemini 3 cannot fully disable thinking, so we use "low"
# Gemini 3 cannot fully disable thinking, so we use "low" but hide thoughts
return {"thinkingLevel": "low", "includeThoughts": False}
elif reasoning_effort == "none":
return {"thinkingLevel": "low"} # gemini 3 cannot fully disable thinking, so we use "low"
return {"thinkingLevel": "low", "includeThoughts": False}
else:
raise ValueError(f"Invalid reasoning effort: {reasoning_effort}")
@ -663,7 +686,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
status_code=400,
)
@staticmethod
def _map_thinking_param(
thinking_param: AnthropicThinkingParam,
@ -835,9 +857,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if VertexGeminiConfig._is_gemini_3_or_newer(model):
optional_params[
"thinkingConfig"
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
value, model
)
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
value, model
)
else:
optional_params[
"thinkingConfig"
@ -879,8 +901,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if VertexGeminiConfig._is_gemini_3_or_newer(model):
if "temperature" not in optional_params:
optional_params["temperature"] = 1.0
if "thinkingConfig" not in optional_params or "thinkingLevel" not in optional_params.get("thinkingConfig", {}):
thinking_config = optional_params.get("thinkingConfig", {})
thinking_config = optional_params.get("thinkingConfig", {})
if (
"thinkingLevel" not in thinking_config
and "thinkingBudget" not in thinking_config
):
thinking_config["thinkingLevel"] = "low"
optional_params["thinkingConfig"] = thinking_config
@ -1147,17 +1172,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if "functionCall" in part:
_function_chunk: ChatCompletionToolCallFunctionChunk = {
"name": part["functionCall"]["name"],
"arguments": json.dumps(part["functionCall"]["args"], ensure_ascii=False),
"arguments": json.dumps(
part["functionCall"]["args"], ensure_ascii=False
),
}
# Extract thought signature if present
thought_signature = part.get("thoughtSignature")
if is_function_call is True:
function_dict: Dict[str, Any] = dict(_function_chunk)
if thought_signature:
if "provider_specific_fields" not in function_dict:
function_dict["provider_specific_fields"] = {}
function_dict["provider_specific_fields"]["thought_signature"] = thought_signature
function_dict["provider_specific_fields"][
"thought_signature"
] = thought_signature
function = cast(ChatCompletionToolCallFunctionChunk, function_dict)
else:
_tool_response_chunk: ChatCompletionToolCallChunk = {
@ -1166,7 +1195,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"function": _function_chunk,
"index": cumulative_tool_call_idx,
}
# Embed thought signature in ID for OpenAI client compatibility
if thought_signature:
_tool_response_chunk[
"id"
] = _encode_tool_call_id_with_signature(
_tool_response_chunk["id"], thought_signature
)
_tool_response_chunk["provider_specific_fields"] = { # type: ignore
"thought_signature": thought_signature
}
@ -1506,7 +1541,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
annotations: List[ChatCompletionAnnotation] = []
for metadata in grounding_metadata:
# Extract groundingSupports - these map text segments to sources
grounding_supports = metadata.get("groundingSupports", [])
@ -1527,23 +1561,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
segment = support.get("segment", {})
start_index = segment.get("startIndex")
end_index = segment.get("endIndex")
# Get the chunk indices for this support
chunk_indices = support.get("groundingChunkIndices", [])
if start_index is not None and end_index is not None and chunk_indices:
# Use the first chunk's URL for the annotation
first_chunk_idx = chunk_indices[0]
if first_chunk_idx in chunk_to_uri_map:
uri_info = chunk_to_uri_map[first_chunk_idx]
url_citation: ChatCompletionAnnotationURLCitation = {
"start_index": start_index,
"end_index": end_index,
"url": uri_info["url"],
"title": uri_info["title"],
}
annotation: ChatCompletionAnnotation = {
"type": "url_citation",
"url_citation": url_citation,
@ -1643,9 +1677,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
chat_completion_message["reasoning_content"] = reasoning_content
if candidate_grounding_metadata:
annotations = VertexGeminiConfig._convert_grounding_metadata_to_annotations(
grounding_metadata=candidate_grounding_metadata,
content_text=content,
annotations = (
VertexGeminiConfig._convert_grounding_metadata_to_annotations(
grounding_metadata=candidate_grounding_metadata,
content_text=content,
)
)
if annotations:
chat_completion_message["annotations"] = annotations # type: ignore
@ -1673,6 +1709,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if thinking_blocks is not None:
chat_completion_message["thinking_blocks"] = thinking_blocks # type: ignore
# Convert thinking_blocks to reasoning_content for streaming
# This ensures reasoning_content is available in streaming responses
if (
isinstance(model_response, ModelResponseStream)
and reasoning_content is None
):
reasoning_content_parts = []
for block in thinking_blocks:
thinking_text = block.get("thinking")
if thinking_text:
reasoning_content_parts.append(thinking_text)
if reasoning_content_parts:
reasoning_content = "\n".join(reasoning_content_parts)
chat_completion_message["reasoning_content"] = reasoning_content
if isinstance(model_response, ModelResponseStream):
choice = VertexGeminiConfig._create_streaming_choice(
chat_completion_message=chat_completion_message,
@ -1852,7 +1904,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
def _transform_messages(
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> List[ContentType]:
return _gemini_convert_messages_with_history(messages=messages)
return _gemini_convert_messages_with_history(messages=messages, model=model)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
@ -1911,7 +1963,9 @@ async def make_call(
)
try:
response = await client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj)
response = await client.post(
api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj
)
response.raise_for_status()
except httpx.HTTPStatusError as e:
exception_string = str(await e.response.aread())
@ -1958,7 +2012,9 @@ def make_sync_call(
if client is None:
client = HTTPHandler() # Create a new client if none provided
response = client.post(api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj)
response = client.post(
api_base, headers=headers, data=data, stream=True, logging_obj=logging_obj
)
if response.status_code != 200 and response.status_code != 201:
raise VertexAIError(
@ -2013,7 +2069,6 @@ class VertexLLM(VertexBase):
gemini_api_key: Optional[str] = None,
extra_headers: Optional[dict] = None,
) -> CustomStreamWrapper:
should_use_v1beta1_features = self.is_using_v1beta1_features(
optional_params=optional_params
)
@ -2050,8 +2105,8 @@ class VertexLLM(VertexBase):
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header) # type: ignore
vertex_auth_header=auth_header,
) # type: ignore
## LOGGING
logging_obj.pre_call(
@ -2144,7 +2199,8 @@ class VertexLLM(VertexBase):
**data,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header) # type: ignore
vertex_auth_header=auth_header,
) # type: ignore
_async_client_params = {}
if timeout:
@ -2168,7 +2224,10 @@ class VertexLLM(VertexBase):
try:
response = await client.post(
api_base, headers=headers, json=cast(dict, request_body), logging_obj=logging_obj
api_base,
headers=headers,
json=cast(dict, request_body),
logging_obj=logging_obj,
) # type: ignore
response.raise_for_status()
except httpx.HTTPStatusError as err:
@ -2322,9 +2381,10 @@ class VertexLLM(VertexBase):
## TRANSFORMATION ##
data = sync_transform_request_body(
**transform_request_params,
vertex_project=vertex_project,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_auth_header=auth_header)
vertex_auth_header=auth_header,
)
## LOGGING
logging_obj.pre_call(

39
litellm/llms/vertex_ai/image_edit/__init__.py Normal file
View file

@ -0,0 +1,39 @@
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.vertex_ai.common_utils import VertexAIModelRoute, get_vertex_ai_model_route
from .cost_calculator import cost_calculator
from .vertex_gemini_transformation import VertexAIGeminiImageEditConfig
from .vertex_imagen_transformation import VertexAIImagenImageEditConfig
__all__ = [
"VertexAIGeminiImageEditConfig",
"VertexAIImagenImageEditConfig",
"get_vertex_ai_image_edit_config",
"cost_calculator"
]
def get_vertex_ai_image_edit_config(model: str) -> BaseImageEditConfig:
"""
Get the appropriate image edit config for a Vertex AI model.
Routes to the correct transformation class based on the model type:
- Gemini models use generateContent API (VertexAIGeminiImageEditConfig)
- Imagen models use predict API (VertexAIImagenImageEditConfig)
Args:
model: The model name (e.g., "gemini-2.5-flash", "imagegeneration@006")
Returns:
BaseImageEditConfig: The appropriate configuration class
"""
# Determine the model route
model_route = get_vertex_ai_model_route(model)
if model_route == VertexAIModelRoute.GEMINI:
# Gemini models use generateContent API
return VertexAIGeminiImageEditConfig()
else:
# Default to Imagen for other models (imagegeneration, etc.)
# This includes NON_GEMINI models like imagegeneration@006
return VertexAIImagenImageEditConfig()

34
litellm/llms/vertex_ai/image_edit/cost_calculator.py Normal file
View file

@ -0,0 +1,34 @@
"""
Vertex AI Image Edit Cost Calculator
"""
from typing import Any
import litellm
from litellm.types.utils import ImageResponse
def cost_calculator(
model: str,
image_response: Any,
) -> float:
"""
Vertex AI image edit cost calculator.
Mirrors image generation pricing: charge per returned image based on
model metadata (`output_cost_per_image`).
"""
model_info = litellm.get_model_info(
model=model,
custom_llm_provider="vertex_ai",
)
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
if not isinstance(image_response, ImageResponse):
raise ValueError(
f"image_response must be of type ImageResponse got type={type(image_response)}"
)
num_images = len(image_response.data or [])
return output_cost_per_image * num_images

263
litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py Normal file
View file

@ -0,0 +1,263 @@
import base64
import json
import os
from io import BufferedReader, BytesIO
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
"""
Vertex AI Gemini Image Edit Configuration
Uses generateContent API for Gemini models on Vertex AI
"""
SUPPORTED_PARAMS: List[str] = ["size"]
def __init__(self) -> None:
BaseImageEditConfig.__init__(self)
VertexLLM.__init__(self)
def get_supported_openai_params(self, model: str) -> List[str]:
return list(self.SUPPORTED_PARAMS)
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict[str, Any]:
supported_params = self.get_supported_openai_params(model)
filtered_params = {
key: value
for key, value in image_edit_optional_params.items()
if key in supported_params
}
mapped_params: Dict[str, Any] = {}
if "size" in filtered_params:
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(
filtered_params["size"] # type: ignore[arg-type]
)
return mapped_params
def _resolve_vertex_project(self) -> Optional[str]:
return (
getattr(self, "_vertex_project", None)
or os.environ.get("VERTEXAI_PROJECT")
or getattr(litellm, "vertex_project", None)
or get_secret_str("VERTEXAI_PROJECT")
)
def _resolve_vertex_location(self) -> Optional[str]:
return (
getattr(self, "_vertex_location", None)
or os.environ.get("VERTEXAI_LOCATION")
or os.environ.get("VERTEX_LOCATION")
or getattr(litellm, "vertex_location", None)
or get_secret_str("VERTEXAI_LOCATION")
or get_secret_str("VERTEX_LOCATION")
)
def _resolve_vertex_credentials(self) -> Optional[str]:
return (
getattr(self, "_vertex_credentials", None)
or os.environ.get("VERTEXAI_CREDENTIALS")
or getattr(litellm, "vertex_credentials", None)
or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
or get_secret_str("VERTEXAI_CREDENTIALS")
)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
headers = headers or {}
vertex_project = self._resolve_vertex_project()
vertex_credentials = self._resolve_vertex_credentials()
access_token, _ = self._ensure_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
)
return self.set_headers(access_token, headers)
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for Vertex AI Gemini generateContent API
"""
vertex_project = self._resolve_vertex_project()
vertex_location = self._resolve_vertex_location()
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Use the model name as provided, handling vertex_ai prefix
model_name = model
if model.startswith("vertex_ai/"):
model_name = model.replace("vertex_ai/", "")
if api_base:
base_url = api_base.rstrip("/")
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent"
def transform_image_edit_request( # type: ignore[override]
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict[str, Any],
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict[str, Any], Optional[RequestFiles]]:
inline_parts = self._prepare_inline_image_parts(image)
if not inline_parts:
raise ValueError("Vertex AI Gemini image edit requires at least one image.")
# Correct format for Vertex AI Gemini image editing
contents = {
"role": "USER",
"parts": inline_parts + [{"text": prompt}]
}
request_body: Dict[str, Any] = {"contents": contents}
# Generation config with proper structure for image editing
generation_config: Dict[str, Any] = {
"response_modalities": ["IMAGE"]
}
# Add image-specific configuration
image_config: Dict[str, Any] = {}
if "aspectRatio" in image_edit_optional_request_params:
image_config["aspect_ratio"] = image_edit_optional_request_params["aspectRatio"]
if image_config:
generation_config["image_config"] = image_config
request_body["generationConfig"] = generation_config
payload: Any = json.dumps(request_body)
empty_files = cast(RequestFiles, [])
return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files))
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: Any,
) -> ImageResponse:
model_response = ImageResponse()
try:
response_json = raw_response.json()
except Exception as exc:
raise self.get_error_class(
error_message=f"Error transforming image edit response: {exc}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
candidates = response_json.get("candidates", [])
data_list: List[ImageObject] = []
for candidate in candidates:
content = candidate.get("content", {})
parts = content.get("parts", [])
for part in parts:
inline_data = part.get("inlineData")
if inline_data and inline_data.get("data"):
data_list.append(
ImageObject(
b64_json=inline_data["data"],
url=None,
)
)
model_response.data = cast(List[OpenAIImage], data_list)
return model_response
def _map_size_to_aspect_ratio(self, size: str) -> str:
"""Map OpenAI size format to Gemini aspect ratio format"""
aspect_ratio_map = {
"1024x1024": "1:1",
"1792x1024": "16:9",
"1024x1792": "9:16",
"1280x896": "4:3",
"896x1280": "3:4",
}
return aspect_ratio_map.get(size, "1:1")
def _prepare_inline_image_parts(
self, image: Union[FileTypes, List[FileTypes]]
) -> List[Dict[str, Any]]:
images: List[FileTypes]
if isinstance(image, list):
images = image
else:
images = [image]
inline_parts: List[Dict[str, Any]] = []
for img in images:
if img is None:
continue
mime_type = ImageEditRequestUtils.get_image_content_type(img)
image_bytes = self._read_all_bytes(img)
inline_parts.append(
{
"inlineData": {
"mimeType": mime_type,
"data": base64.b64encode(image_bytes).decode("utf-8"),
}
}
)
return inline_parts
def _read_all_bytes(self, image: FileTypes) -> bytes:
if isinstance(image, bytes):
return image
if isinstance(image, BytesIO):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
if isinstance(image, BufferedReader):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
raise ValueError("Unsupported image type for Vertex AI Gemini image edit.")

353
litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py Normal file
View file

@ -0,0 +1,353 @@
import base64
import json
import os
from io import BufferedRandom, BufferedReader, BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM):
"""
Vertex AI Imagen Image Edit Configuration
Uses predict API for Imagen models on Vertex AI
"""
SUPPORTED_PARAMS: List[str] = ["n", "size", "mask"]
def __init__(self) -> None:
BaseImageEditConfig.__init__(self)
VertexLLM.__init__(self)
def get_supported_openai_params(self, model: str) -> List[str]:
return list(self.SUPPORTED_PARAMS)
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict[str, Any]:
supported_params = self.get_supported_openai_params(model)
filtered_params = {
key: value
for key, value in image_edit_optional_params.items()
if key in supported_params
}
mapped_params: Dict[str, Any] = {}
# Map OpenAI parameters to Imagen format
if "n" in filtered_params:
mapped_params["sampleCount"] = filtered_params["n"]
if "size" in filtered_params:
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(
filtered_params["size"] # type: ignore[arg-type]
)
if "mask" in filtered_params:
mapped_params["mask"] = filtered_params["mask"]
return mapped_params
def _resolve_vertex_project(self) -> Optional[str]:
return (
getattr(self, "_vertex_project", None)
or os.environ.get("VERTEXAI_PROJECT")
or getattr(litellm, "vertex_project", None)
or get_secret_str("VERTEXAI_PROJECT")
)
def _resolve_vertex_location(self) -> Optional[str]:
return (
getattr(self, "_vertex_location", None)
or os.environ.get("VERTEXAI_LOCATION")
or os.environ.get("VERTEX_LOCATION")
or getattr(litellm, "vertex_location", None)
or get_secret_str("VERTEXAI_LOCATION")
or get_secret_str("VERTEX_LOCATION")
)
def _resolve_vertex_credentials(self) -> Optional[str]:
return (
getattr(self, "_vertex_credentials", None)
or os.environ.get("VERTEXAI_CREDENTIALS")
or getattr(litellm, "vertex_credentials", None)
or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
or get_secret_str("VERTEXAI_CREDENTIALS")
)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
headers = headers or {}
vertex_project = self._resolve_vertex_project()
vertex_credentials = self._resolve_vertex_credentials()
access_token, _ = self._ensure_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
)
return self.set_headers(access_token, headers)
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for Vertex AI Imagen predict API
"""
vertex_project = self._resolve_vertex_project()
vertex_location = self._resolve_vertex_location()
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Use the model name as provided, handling vertex_ai prefix
model_name = model
if model.startswith("vertex_ai/"):
model_name = model.replace("vertex_ai/", "")
if api_base:
base_url = api_base.rstrip("/")
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict"
def transform_image_edit_request( # type: ignore[override]
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict[str, Any],
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict[str, Any], Optional[RequestFiles]]:
# Prepare reference images in the correct Imagen format
reference_images = self._prepare_reference_images(image, image_edit_optional_request_params)
if not reference_images:
raise ValueError("Vertex AI Imagen image edit requires at least one reference image.")
# Correct Imagen instances format
instances = [
{
"prompt": prompt,
"referenceImages": reference_images
}
]
# Extract OpenAI parameters and set sensible defaults for Vertex AI-specific parameters
sample_count = image_edit_optional_request_params.get("sampleCount", 1)
# Use sensible defaults for Vertex AI-specific parameters (not exposed to users)
edit_mode = "EDIT_MODE_INPAINT_INSERTION" # Default edit mode
base_steps = 50 # Default number of steps
# Imagen parameters with correct structure
parameters = {
"sampleCount": sample_count,
"editMode": edit_mode,
"editConfig": {
"baseSteps": base_steps
}
}
# Set default values for Vertex AI-specific parameters (not configurable by users via OpenAI API)
parameters["guidanceScale"] = 7.5 # Default guidance scale
parameters["seed"] = None # Let Vertex AI choose random seed
request_body: Dict[str, Any] = {
"instances": instances,
"parameters": parameters
}
payload: Any = json.dumps(request_body)
empty_files = cast(RequestFiles, [])
return cast(Tuple[Dict[str, Any], Optional[RequestFiles]], (payload, empty_files))
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: Any,
) -> ImageResponse:
model_response = ImageResponse()
try:
response_json = raw_response.json()
except Exception as exc:
raise self.get_error_class(
error_message=f"Error transforming image edit response: {exc}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
predictions = response_json.get("predictions", [])
data_list: List[ImageObject] = []
for prediction in predictions:
# Imagen returns images as bytesBase64Encoded
if "bytesBase64Encoded" in prediction:
data_list.append(
ImageObject(
b64_json=prediction["bytesBase64Encoded"],
url=None,
)
)
model_response.data = cast(List[OpenAIImage], data_list)
return model_response
def _map_size_to_aspect_ratio(self, size: str) -> str:
"""Map OpenAI size format to Imagen aspect ratio format"""
aspect_ratio_map = {
"1024x1024": "1:1",
"1792x1024": "16:9",
"1024x1792": "9:16",
"1280x896": "4:3",
"896x1280": "3:4",
}
return aspect_ratio_map.get(size, "1:1")
def _prepare_reference_images(
self, image: Union[FileTypes, List[FileTypes]],
image_edit_optional_request_params: Dict[str, Any]
) -> List[Dict[str, Any]]:
"""
Prepare reference images in the correct Imagen API format
"""
images: List[FileTypes]
if isinstance(image, list):
images = image
else:
images = [image]
reference_images: List[Dict[str, Any]] = []
for idx, img in enumerate(images):
if img is None:
continue
image_bytes = self._read_all_bytes(img)
base64_data = base64.b64encode(image_bytes).decode("utf-8")
# Create reference image structure
reference_image = {
"referenceType": "REFERENCE_TYPE_RAW",
"referenceId": idx + 1,
"referenceImage": {
"bytesBase64Encoded": base64_data
}
}
reference_images.append(reference_image)
# Handle mask image if provided (for inpainting)
mask_image = image_edit_optional_request_params.get("mask")
if mask_image is not None:
mask_bytes = self._read_all_bytes(mask_image)
mask_base64 = base64.b64encode(mask_bytes).decode("utf-8")
mask_reference = {
"referenceType": "REFERENCE_TYPE_MASK",
"referenceId": len(reference_images) + 1,
"referenceImage": {
"bytesBase64Encoded": mask_base64
},
"maskImageConfig": {
"maskMode": "MASK_MODE_USER_PROVIDED",
"dilation": 0.03 # Default dilation value (not configurable via OpenAI API)
}
}
reference_images.append(mask_reference)
return reference_images
def _read_all_bytes(
self, image: Any, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH
) -> bytes:
if depth > max_depth:
raise ValueError(
f"Max recursion depth {max_depth} reached while reading image bytes for Vertex AI Imagen image edit."
)
if isinstance(image, (list, tuple)):
for item in image:
if item is not None:
return self._read_all_bytes(item, depth=depth + 1, max_depth=max_depth)
raise ValueError("Unsupported image type for Vertex AI Imagen image edit.")
if isinstance(image, dict):
for key in ("data", "bytes", "content"):
if key in image and image[key] is not None:
value = image[key]
if isinstance(value, str):
try:
return base64.b64decode(value)
except Exception:
continue
return self._read_all_bytes(value, depth=depth + 1, max_depth=max_depth)
if "path" in image:
return self._read_all_bytes(image["path"], depth=depth + 1, max_depth=max_depth)
if isinstance(image, bytes):
return image
if isinstance(image, bytearray):
return bytes(image)
if isinstance(image, BytesIO):
current_pos = image.tell()
image.seek(0)
data = image.read()
image.seek(current_pos)
return data
if isinstance(image, (BufferedReader, BufferedRandom)):
stream_pos: Optional[int] = None
try:
stream_pos = image.tell()
except Exception:
stream_pos = None
if stream_pos is not None:
image.seek(0)
data = image.read()
if stream_pos is not None:
image.seek(stream_pos)
return data
if isinstance(image, (str, Path)):
path_obj = Path(image)
if not path_obj.exists():
raise ValueError(
f"Mask/image path does not exist for Vertex AI Imagen image edit: {path_obj}"
)
return path_obj.read_bytes()
if hasattr(image, "read"):
data = image.read()
if isinstance(data, str):
data = data.encode("utf-8")
return data
raise ValueError(
f"Unsupported image type for Vertex AI Imagen image edit. Got type={type(image)}"
)

3
litellm/main.py
View file

@ -5516,6 +5516,7 @@ def transcription(
atranscription = kwargs.pop("atranscription", False)
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
extra_headers = kwargs.get("extra_headers", None)
shared_session = kwargs.get("shared_session", None)
kwargs.pop("tags", [])
non_default_params = get_non_default_transcription_params(kwargs)
@ -5653,6 +5654,7 @@ def transcription(
api_key=api_key,
provider_config=provider_config,
litellm_params=litellm_params_dict,
shared_session=shared_session,
)
elif provider_config is not None:
response = base_llm_http_handler.audio_transcriptions(
@ -5679,6 +5681,7 @@ def transcription(
custom_llm_provider=custom_llm_provider,
headers={},
provider_config=provider_config,
shared_session=shared_session,
)
# Calculate and add duration if response is missing it

1098
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

317
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -16,10 +16,20 @@ from urllib.parse import urlparse
from fastapi import HTTPException
from httpx import HTTPStatusError
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
from mcp import ReadResourceResult, Resource
from mcp.types import (
CallToolRequestParams as MCPCallToolRequestParams,
GetPromptRequestParams,
GetPromptResult,
Prompt,
ResourceTemplate,
)
from mcp.types import CallToolResult
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.experimental_mcp_client.client import MCPClient
@ -28,11 +38,11 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.utils import (
add_server_prefix_to_tool_name,
get_server_name_prefix_tool_mcp,
add_server_prefix_to_name,
get_server_prefix,
is_tool_name_prefixed,
normalize_server_name,
split_server_prefix_from_name,
validate_mcp_server_name,
)
from litellm.proxy._types import (
@ -356,7 +366,7 @@ class MCPServerManager:
base_tool_name = operation_id.replace(" ", "_").lower()
# Add server prefix to tool name
prefixed_tool_name = add_server_prefix_to_tool_name(
prefixed_tool_name = add_server_prefix_to_name(
base_tool_name, server_prefix
)
@ -385,12 +395,12 @@ class MCPServerManager:
)
# Update tool name to server name mapping (for both prefixed and base names)
self.tool_name_to_mcp_server_name_mapping[
base_tool_name
] = server_prefix
self.tool_name_to_mcp_server_name_mapping[
prefixed_tool_name
] = server_prefix
self.tool_name_to_mcp_server_name_mapping[base_tool_name] = (
server_prefix
)
self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = (
server_prefix
)
registered_count += 1
verbose_logger.debug(
@ -715,6 +725,190 @@ class MCPServerManager:
)
return []
async def get_prompts_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
) -> List[Prompt]:
"""
Helper method to get prompts from a single MCP server with prefixed names.
Args:
server (MCPServer): The server to query prompts from
mcp_auth_header: Optional auth header for MCP server
Returns:
List[Prompt]: List of prompts available on the server with prefixed names
"""
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"get_prompts_from_server for {server.name}...")
client = None
try:
if server.static_headers:
if extra_headers is None:
extra_headers = {}
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
prompts = await client.list_prompts()
prefixed_or_original_prompts = self._create_prefixed_prompts(
prompts, server, add_prefix=add_prefix
)
return prefixed_or_original_prompts
except Exception as e:
verbose_logger.warning(
f"Failed to get prompts from server {server.name}: {str(e)}"
)
return []
async def get_resources_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
) -> List[Resource]:
"""Fetch available resources from a single MCP server."""
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"get_resources_from_server for {server.name}...")
client = None
try:
if server.static_headers:
if extra_headers is None:
extra_headers = {}
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
resources = await client.list_resources()
prefixed_resources = self._create_prefixed_resources(
resources, server, add_prefix=add_prefix
)
return prefixed_resources
except Exception as e:
verbose_logger.warning(
f"Failed to get resources from server {server.name}: {str(e)}"
)
return []
async def get_resource_templates_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
) -> List[ResourceTemplate]:
"""Fetch available resource templates from a single MCP server."""
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"get_resource_templates_from_server for {server.name}...")
client = None
try:
if server.static_headers:
if extra_headers is None:
extra_headers = {}
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
resource_templates = await client.list_resource_templates()
prefixed_templates = self._create_prefixed_resource_templates(
resource_templates, server, add_prefix=add_prefix
)
return prefixed_templates
except Exception as e:
verbose_logger.warning(
f"Failed to get resource templates from server {server.name}: {str(e)}"
)
return []
async def read_resource_from_server(
self,
server: MCPServer,
url: AnyUrl,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
) -> ReadResourceResult:
"""Read resource contents from a specific MCP server."""
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"read_resource_from_server for {server.name}...")
if server.static_headers:
if extra_headers is None:
extra_headers = {}
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
return await client.read_resource(url)
async def get_prompt_from_server(
self,
server: MCPServer,
prompt_name: str,
arguments: Optional[Dict[str, Any]] = None,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
) -> GetPromptResult:
"""Fetch a specific prompt definition from a single MCP server."""
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"get_prompt_from_server for {server.name}...")
if server.static_headers:
if extra_headers is None:
extra_headers = {}
extra_headers.update(server.static_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
get_prompt_request_params = GetPromptRequestParams(
name=prompt_name,
arguments=arguments,
)
return await client.get_prompt(get_prompt_request_params)
async def _descovery_metadata(
self,
server_url: str,
@ -1025,7 +1219,7 @@ class MCPServerManager:
prefix = get_server_prefix(server)
for tool in tools:
prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix)
prefixed_name = add_server_prefix_to_name(tool.name, prefix)
name_to_use = prefixed_name if add_prefix else tool.name
@ -1045,6 +1239,82 @@ class MCPServerManager:
)
return prefixed_tools
def _create_prefixed_prompts(
self, prompts: List[Prompt], server: MCPServer, add_prefix: bool = True
) -> List[Prompt]:
"""
Create prefixed prompts and update prompt mapping.
Args:
prompts: List of original prompts from server
server: Server instance
Returns:
List of prompts with prefixed names
"""
prefixed_prompts = []
prefix = get_server_prefix(server)
for prompt in prompts:
prefixed_name = add_server_prefix_to_name(prompt.name, prefix)
name_to_use = prefixed_name if add_prefix else prompt.name
prompt.name = name_to_use
prefixed_prompts.append(prompt)
verbose_logger.info(
f"Successfully fetched {len(prefixed_prompts)} prompts from server {server.name}"
)
return prefixed_prompts
def _create_prefixed_resources(
self, resources: List[Resource], server: MCPServer, add_prefix: bool = True
) -> List[Resource]:
"""Prefix resource names and track origin server for read requests."""
prefixed_resources: List[Resource] = []
prefix = get_server_prefix(server)
for resource in resources:
name_to_use = (
add_server_prefix_to_name(resource.name, prefix)
if add_prefix
else resource.name
)
resource.name = name_to_use
prefixed_resources.append(resource)
verbose_logger.info(
f"Successfully fetched {len(prefixed_resources)} resources from server {server.name}"
)
return prefixed_resources
def _create_prefixed_resource_templates(
self,
resource_templates: List[ResourceTemplate],
server: MCPServer,
add_prefix: bool = True,
) -> List[ResourceTemplate]:
"""Prefix resource template names for multi-server scenarios."""
prefixed_templates: List[ResourceTemplate] = []
prefix = get_server_prefix(server)
for resource_template in resource_templates:
name_to_use = (
add_server_prefix_to_name(resource_template.name, prefix)
if add_prefix
else resource_template.name
)
resource_template.name = name_to_use
prefixed_templates.append(resource_template)
verbose_logger.info(
f"Successfully fetched {len(prefixed_templates)} resource templates from server {server.name}"
)
return prefixed_templates
def check_allowed_or_banned_tools(self, tool_name: str, server: MCPServer) -> bool:
"""
Check if the tool is allowed or banned for the given server
@ -1079,7 +1349,7 @@ class MCPServerManager:
HTTPException: If allowed_params is configured for this tool but arguments contain disallowed params
"""
from litellm.proxy._experimental.mcp_server.utils import (
get_server_name_prefix_tool_mcp,
split_server_prefix_from_name,
)
# If no allowed_params configured, return all arguments
@ -1087,7 +1357,7 @@ class MCPServerManager:
return
# Get the unprefixed tool name to match against config
unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(tool_name)
unprefixed_tool_name, _ = split_server_prefix_from_name(tool_name)
# Check both prefixed and unprefixed tool names
allowed_params_list = server.allowed_params.get(
@ -1488,7 +1758,7 @@ class MCPServerManager:
start_time = datetime.datetime.now()
# Get the MCP server
prefixed_tool_name = add_server_prefix_to_tool_name(name, server_name)
prefixed_tool_name = add_server_prefix_to_name(name, server_name)
mcp_server = self._get_mcp_server_from_tool_name(prefixed_tool_name)
if mcp_server is None:
raise ValueError(f"Tool {name} not found")
@ -1594,7 +1864,7 @@ class MCPServerManager:
for tool in tools:
# The tool.name here is already prefixed from _get_tools_from_server
# Extract original name for mapping
original_name, _ = get_server_name_prefix_tool_mcp(tool.name)
original_name, _ = split_server_prefix_from_name(tool.name)
self.tool_name_to_mcp_server_name_mapping[original_name] = server.name
self.tool_name_to_mcp_server_name_mapping[tool.name] = server.name
@ -1622,7 +1892,7 @@ class MCPServerManager:
(
original_tool_name,
server_name_from_prefix,
) = get_server_name_prefix_tool_mcp(tool_name)
) = split_server_prefix_from_name(tool_name)
if original_tool_name in self.tool_name_to_mcp_server_name_mapping:
for server in self.get_registry().values():
if normalize_server_name(server.name) == normalize_server_name(
@ -1668,11 +1938,16 @@ class MCPServerManager:
return server
return None
def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]:
servers = []
registry = self.get_registry()
for server in registry.values():
if server.server_id in server_ids:
def get_public_mcp_servers(self) -> List[MCPServer]:
"""
Get the public MCP servers
"""
servers: List[MCPServer] = []
if litellm.public_mcp_servers is None:
return servers
for server_id in litellm.public_mcp_servers:
server = self.get_mcp_server_by_id(server_id)
if server:
servers.append(server)
return servers

723
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -8,7 +8,15 @@ from datetime import datetime
from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union
from fastapi import FastAPI, HTTPException
from pydantic import ConfigDict
from mcp import ReadResourceResult, Resource
from mcp.server.lowlevel.helper_types import ReadResourceContents
from mcp.types import (
BlobResourceContents,
GetPromptResult,
ResourceTemplate,
TextResourceContents,
)
from pydantic import AnyUrl, ConfigDict
from starlette.types import Receive, Scope, Send
from litellm._logging import verbose_logger
@ -54,6 +62,7 @@ if MCP_AVAILABLE:
from mcp.server.streamable_http_manager import StreamableHTTPSessionManager
from mcp.types import EmbeddedResource, ImageContent, TextContent
from mcp.types import Tool as MCPTool
from mcp.types import Prompt
from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import (
MCPAuthenticatedUser,
@ -66,7 +75,7 @@ if MCP_AVAILABLE:
global_mcp_tool_registry,
)
from litellm.proxy._experimental.mcp_server.utils import (
get_server_name_prefix_tool_mcp,
split_server_prefix_from_name,
)
######################################################
@ -303,6 +312,208 @@ if MCP_AVAILABLE:
return response
@server.list_prompts()
async def list_prompts() -> List[Prompt]:
"""
List all available prompts
"""
try:
# Get user authentication from context variable
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
raw_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_prompts - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(
f"MCP list_prompts - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
# Get mcp_servers from context variable
verbose_logger.debug("MCP list_prompts - Calling _list_prompts")
prompts = await _list_mcp_prompts(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.info(
f"MCP list_prompts - Successfully returned {len(prompts)} prompts"
)
return prompts
except Exception as e:
verbose_logger.exception(f"Error in list_prompts endpoint: {str(e)}")
# Return empty list instead of failing completely
# This prevents the HTTP stream from failing and allows the client to get a response
return []
@server.get_prompt()
async def get_prompt(
name: str, arguments: dict[str, str] | None
) -> GetPromptResult:
"""
Get a specific prompt with the provided arguments
Args:
name (str): Name of the prompt to get
arguments (Dict[str, Any] | None): Arguments to pass to the prompt
Returns:
GetPromptResult: Getting prompt execution results
"""
# Validate arguments
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
raw_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}"
)
return await mcp_get_prompt(
name=name,
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
@server.list_resources()
async def list_resources() -> List[Resource]:
"""List all available resources."""
try:
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
raw_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_resources - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(
f"MCP list_resources - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
resources = await _list_mcp_resources(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.info(
f"MCP list_resources - Successfully returned {len(resources)} resources"
)
return resources
except Exception as e:
verbose_logger.exception(f"Error in list_resources endpoint: {str(e)}")
return []
@server.list_resource_templates()
async def list_resource_templates() -> List[ResourceTemplate]:
"""List all available resource templates."""
try:
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
raw_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_resource_templates - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(
f"MCP list_resource_templates - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
resource_templates = await _list_mcp_resource_templates(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.info(
"MCP list_resource_templates - Successfully returned "
f"{len(resource_templates)} resource templates"
)
return resource_templates
except Exception as e:
verbose_logger.exception(
f"Error in list_resource_templates endpoint: {str(e)}"
)
return []
@server.read_resource()
async def read_resource(url: AnyUrl) -> list[ReadResourceContents]:
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
raw_headers,
) = get_auth_context()
read_resource_result = await mcp_read_resource(
url=url,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
normalized_contents: List[ReadResourceContents] = []
for content in read_resource_result.contents:
if isinstance(content, TextResourceContents):
normalized_contents.append(
ReadResourceContents(
content=content.text,
mime_type=content.mimeType,
)
)
elif isinstance(content, BlobResourceContents):
normalized_contents.append(
ReadResourceContents(
content=content.blob,
mime_type=None,
)
)
return normalized_contents
########################################################
############ End of MCP Server Routes ##################
########################################################
@ -379,7 +590,7 @@ if MCP_AVAILABLE:
True if the tool name (prefixed or unprefixed) is in the filter list
"""
from litellm.proxy._experimental.mcp_server.utils import (
get_server_name_prefix_tool_mcp,
split_server_prefix_from_name,
)
# Check if the full name is in the list
@ -387,7 +598,7 @@ if MCP_AVAILABLE:
return True
# Check if the unprefixed name is in the list
unprefixed_name, _ = get_server_name_prefix_tool_mcp(tool_name)
unprefixed_name, _ = split_server_prefix_from_name(tool_name)
return unprefixed_name in filter_list
def filter_tools_by_allowed_tools(
@ -428,6 +639,56 @@ if MCP_AVAILABLE:
return tools_to_return
async def _get_allowed_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_servers: Optional[List[str]],
) -> List[MCPServer]:
"""Return allowed MCP servers for a request after applying filters."""
allowed_mcp_server_ids = (
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
)
allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( # type: ignore[attr-defined]
allowed_mcp_server_ids
)
if mcp_servers is not None:
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
mcp_servers=mcp_servers,
allowed_mcp_servers=allowed_mcp_servers,
)
return allowed_mcp_servers
def _prepare_mcp_server_headers(
server: MCPServer,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
mcp_auth_header: Optional[str],
oauth2_headers: Optional[Dict[str, str]],
raw_headers: Optional[Dict[str, str]],
) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]:
"""Build auth and extra headers for a server."""
server_auth_header: Optional[Union[Dict[str, str], str]] = None
if mcp_server_auth_headers and server.alias is not None:
server_auth_header = mcp_server_auth_headers.get(server.alias)
elif mcp_server_auth_headers and server.server_name is not None:
server_auth_header = mcp_server_auth_headers.get(server.server_name)
extra_headers: Optional[Dict[str, str]] = None
if server.auth_type == MCPAuth.oauth2:
extra_headers = oauth2_headers
if server.extra_headers and raw_headers:
if extra_headers is None:
extra_headers = {}
for header in server.extra_headers:
if header in raw_headers:
extra_headers[header] = raw_headers[header]
if server_auth_header is None:
server_auth_header = mcp_auth_header
return server_auth_header, extra_headers
async def _get_tools_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
@ -452,19 +713,10 @@ if MCP_AVAILABLE:
if not MCP_AVAILABLE:
return []
# Get allowed MCP servers based on user permissions
allowed_mcp_server_ids = (
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids(
allowed_mcp_server_ids
)
if mcp_servers is not None:
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
mcp_servers=mcp_servers,
allowed_mcp_servers=allowed_mcp_servers,
)
# Decide whether to add prefix based on number of allowed servers
add_prefix = not (len(allowed_mcp_servers) == 1)
@ -475,27 +727,13 @@ if MCP_AVAILABLE:
if server is None:
continue
# Get server-specific auth header if available
server_auth_header: Optional[Union[Dict[str, str], str]] = None
if mcp_server_auth_headers and server.alias is not None:
server_auth_header = mcp_server_auth_headers.get(server.alias)
elif mcp_server_auth_headers and server.server_name is not None:
server_auth_header = mcp_server_auth_headers.get(server.server_name)
extra_headers: Optional[Dict[str, str]] = None
if server.auth_type == MCPAuth.oauth2:
extra_headers = oauth2_headers
if server.extra_headers and raw_headers:
if extra_headers is None:
extra_headers = {}
for header in server.extra_headers:
if header in raw_headers:
extra_headers[header] = raw_headers[header]
# Fall back to deprecated mcp_auth_header if no server-specific header found
if server_auth_header is None:
server_auth_header = mcp_auth_header
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
try:
tools = await global_mcp_server_manager._get_tools_from_server(
@ -530,6 +768,195 @@ if MCP_AVAILABLE:
return all_tools
async def _get_prompts_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Prompt]:
"""
Helper method to fetch prompt from MCP servers based on server filtering criteria.
Args:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional auth header for MCP server (deprecated)
mcp_servers: Optional list of server names/aliases to filter by
mcp_server_auth_headers: Optional dict of server-specific auth headers
oauth2_headers: Optional dict of oauth2 headers
Returns:
List[Prompt]: Combined list of prompts from filtered servers
"""
if not MCP_AVAILABLE:
return []
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
# Decide whether to add prefix based on number of allowed servers
add_prefix = not (len(allowed_mcp_servers) == 1)
# Get prompts from each allowed server
all_prompts = []
for server in allowed_mcp_servers:
if server is None:
continue
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
try:
prompts = await global_mcp_server_manager.get_prompts_from_server(
server=server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
)
all_prompts.extend(prompts)
verbose_logger.debug(
f"Successfully fetched {len(prompts)} prompts from server {server.name}"
)
except Exception as e:
verbose_logger.exception(
f"Error getting prompts from server {server.name}: {str(e)}"
)
# Continue with other servers instead of failing completely
verbose_logger.info(
f"Successfully fetched {len(all_prompts)} prompts total from all MCP servers"
)
return all_prompts
async def _get_resources_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Resource]:
"""Fetch resources from allowed MCP servers."""
if not MCP_AVAILABLE:
return []
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
add_prefix = not (len(allowed_mcp_servers) == 1)
all_resources: List[Resource] = []
for server in allowed_mcp_servers:
if server is None:
continue
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
try:
resources = await global_mcp_server_manager.get_resources_from_server(
server=server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
)
all_resources.extend(resources)
verbose_logger.debug(
f"Successfully fetched {len(resources)} resources from server {server.name}"
)
except Exception as e:
verbose_logger.exception(
f"Error getting resources from server {server.name}: {str(e)}"
)
verbose_logger.info(
f"Successfully fetched {len(all_resources)} resources total from all MCP servers"
)
return all_resources
async def _get_resource_templates_from_mcp_servers(
user_api_key_auth: Optional[UserAPIKeyAuth],
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[ResourceTemplate]:
"""Fetch resource templates from allowed MCP servers."""
if not MCP_AVAILABLE:
return []
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
add_prefix = not (len(allowed_mcp_servers) == 1)
all_resource_templates: List[ResourceTemplate] = []
for server in allowed_mcp_servers:
if server is None:
continue
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
try:
resource_templates = (
await global_mcp_server_manager.get_resource_templates_from_server(
server=server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
)
)
all_resource_templates.extend(resource_templates)
verbose_logger.debug(
"Successfully fetched %s resource templates from server %s",
len(resource_templates),
server.name,
)
except Exception as e:
verbose_logger.exception(
"Error getting resource templates from server %s: %s",
server.name,
str(e),
)
verbose_logger.info(
"Successfully fetched %s resource templates total from all MCP servers",
len(all_resource_templates),
)
return all_resource_templates
async def filter_tools_by_key_team_permissions(
tools: List[MCPTool],
server_id: str,
@ -553,7 +980,7 @@ if MCP_AVAILABLE:
filtered_tools = []
for t in tools:
# Get tool name without server prefix
unprefixed_tool_name, _ = get_server_name_prefix_tool_mcp(t.name)
unprefixed_tool_name, _ = split_server_prefix_from_name(t.name)
if unprefixed_tool_name in allowed_tool_names:
filtered_tools.append(t)
else:
@ -606,6 +1033,118 @@ if MCP_AVAILABLE:
return managed_tools
async def _list_mcp_prompts(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Prompt]:
"""
List all available MCP prompts.
Args:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional auth header for MCP server (deprecated)
mcp_servers: Optional list of server names/aliases to filter by
mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
Returns:
List[Prompt]: Combined list of tools from all accessible servers
"""
if not MCP_AVAILABLE:
return []
# Get tools from managed MCP servers with error handling
managed_prompts = []
try:
managed_prompts = await _get_prompts_from_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.debug(
f"Successfully fetched {len(managed_prompts)} prompts from managed MCP servers"
)
except Exception as e:
verbose_logger.exception(
f"Error getting tools from managed MCP servers: {str(e)}"
)
# Continue with empty managed tools list instead of failing completely
return managed_prompts
async def _list_mcp_resources(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Resource]:
"""List all available MCP resources."""
if not MCP_AVAILABLE:
return []
managed_resources: List[Resource] = []
try:
managed_resources = await _get_resources_from_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.debug(
f"Successfully fetched {len(managed_resources)} resources from managed MCP servers"
)
except Exception as e:
verbose_logger.exception(
f"Error getting resources from managed MCP servers: {str(e)}"
)
return managed_resources
async def _list_mcp_resource_templates(
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[ResourceTemplate]:
"""List all available MCP resource templates."""
if not MCP_AVAILABLE:
return []
managed_resource_templates: List[ResourceTemplate] = []
try:
managed_resource_templates = await _get_resource_templates_from_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
verbose_logger.debug(
"Successfully fetched %s resource templates from managed MCP servers",
len(managed_resource_templates),
)
except Exception as e:
verbose_logger.exception(
"Error getting resource templates from managed MCP servers: %s",
str(e),
)
return managed_resource_templates
@client
async def call_mcp_tool(
name: str,
@ -634,7 +1173,7 @@ if MCP_AVAILABLE:
)
)
allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids(
allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( # type: ignore[attr-defined]
allowed_mcp_server_ids
)
@ -647,7 +1186,7 @@ if MCP_AVAILABLE:
mcp_server: Optional[MCPServer] = None
# Remove prefix from tool name for logging and processing
original_tool_name, server_name = get_server_name_prefix_tool_mcp(name)
original_tool_name, server_name = split_server_prefix_from_name(name)
# If tool name is unprefixed, resolve its server so we can enforce permissions
if not server_name:
@ -735,6 +1274,110 @@ if MCP_AVAILABLE:
)
return response
async def mcp_get_prompt(
name: str,
arguments: Optional[Dict[str, Any]] = None,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> GetPromptResult:
"""
Fetch a specific MCP prompt, handling both prefixed and unprefixed names.
"""
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
if not allowed_mcp_servers:
raise HTTPException(
status_code=403,
detail="User not allowed to get this prompt.",
)
# Decide whether to add prefix based on number of allowed servers
add_prefix = not (len(allowed_mcp_servers) == 1)
if add_prefix:
original_prompt_name, server_name = split_server_prefix_from_name(name)
else:
original_prompt_name = name
server_name = allowed_mcp_servers[0].name
server = next((s for s in allowed_mcp_servers if s.name == server_name), None)
if server is None:
raise HTTPException(
status_code=403,
detail="User not allowed to get this prompt.",
)
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
return await global_mcp_server_manager.get_prompt_from_server(
server=server,
prompt_name=original_prompt_name,
arguments=arguments,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
)
async def mcp_read_resource(
url: AnyUrl,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> ReadResourceResult:
"""Read resource contents from upstream MCP servers."""
allowed_mcp_servers = await _get_allowed_mcp_servers(
user_api_key_auth=user_api_key_auth,
mcp_servers=mcp_servers,
)
if not allowed_mcp_servers:
raise HTTPException(
status_code=403,
detail="User not allowed to read this resource.",
)
if len(allowed_mcp_servers) != 1:
raise HTTPException(
status_code=400,
detail=(
"Multiple MCP servers configured; read_resource currently "
"supports exactly one allowed server."
),
)
server = allowed_mcp_servers[0]
server_auth_header, extra_headers = _prepare_mcp_server_headers(
server=server,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=raw_headers,
)
return await global_mcp_server_manager.read_resource_from_server(
server=server,
url=url,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
)
def _get_standard_logging_mcp_tool_call(
name: str,
arguments: Dict[str, Any],

81
litellm/proxy/_experimental/mcp_server/utils.py
View file

@ -13,6 +13,7 @@ LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM"
MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-")
MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}"
def is_mcp_available() -> bool:
"""
Returns True if the MCP module is available, False otherwise
@ -23,92 +24,81 @@ def is_mcp_available() -> bool:
except ImportError:
return False
def normalize_server_name(server_name: str) -> str:
"""
Normalize server name by replacing spaces with underscores
"""
return server_name.replace(" ", "_")
def validate_and_normalize_mcp_server_payload(payload: Any) -> None:
"""
Validate and normalize MCP server payload fields (server_name and alias).
This function:
1. Validates that server_name and alias don't contain the MCP_TOOL_PREFIX_SEPARATOR
2. Normalizes alias by replacing spaces with underscores
3. Sets default alias if not provided (using server_name as base)
Args:
payload: The payload object containing server_name and alias fields
Raises:
HTTPException: If validation fails
"""
# Server name validation: disallow '-'
if hasattr(payload, 'server_name') and payload.server_name:
if hasattr(payload, "server_name") and payload.server_name:
validate_mcp_server_name(payload.server_name, raise_http_exception=True)
# Alias validation: disallow '-'
if hasattr(payload, 'alias') and payload.alias:
if hasattr(payload, "alias") and payload.alias:
validate_mcp_server_name(payload.alias, raise_http_exception=True)
# Alias normalization and defaulting
alias = getattr(payload, 'alias', None)
server_name = getattr(payload, 'server_name', None)
alias = getattr(payload, "alias", None)
server_name = getattr(payload, "server_name", None)
if not alias and server_name:
alias = normalize_server_name(server_name)
elif alias:
alias = normalize_server_name(alias)
# Update the payload with normalized alias
if hasattr(payload, 'alias'):
if hasattr(payload, "alias"):
payload.alias = alias
def add_server_prefix_to_tool_name(tool_name: str, server_name: str) -> str:
"""
Add server name prefix to tool name
Args:
tool_name: Original tool name
server_name: MCP server name
Returns:
Prefixed tool name in format: server_name::tool_name
"""
def add_server_prefix_to_name(name: str, server_name: str) -> str:
"""Add server name prefix to any MCP resource name."""
formatted_server_name = normalize_server_name(server_name)
return MCP_TOOL_PREFIX_FORMAT.format(
server_name=formatted_server_name,
separator=MCP_TOOL_PREFIX_SEPARATOR,
tool_name=tool_name
tool_name=name,
)
def get_server_prefix(server: Any) -> str:
"""Return the prefix for a server: alias if present, else server_name, else server_id"""
if hasattr(server, 'alias') and server.alias:
if hasattr(server, "alias") and server.alias:
return server.alias
if hasattr(server, 'server_name') and server.server_name:
if hasattr(server, "server_name") and server.server_name:
return server.server_name
if hasattr(server, 'server_id'):
if hasattr(server, "server_id"):
return server.server_id
return ""
def get_server_name_prefix_tool_mcp(prefixed_tool_name: str) -> Tuple[str, str]:
"""
Remove server name prefix from tool name
Args:
prefixed_tool_name: Tool name with server prefix
Returns:
Tuple of (original_tool_name, server_name)
"""
if MCP_TOOL_PREFIX_SEPARATOR in prefixed_tool_name:
parts = prefixed_tool_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1)
def split_server_prefix_from_name(prefixed_name: str) -> Tuple[str, str]:
"""Return the unprefixed name plus the server name used as prefix."""
if MCP_TOOL_PREFIX_SEPARATOR in prefixed_name:
parts = prefixed_name.split(MCP_TOOL_PREFIX_SEPARATOR, 1)
if len(parts) == 2:
return parts[1], parts[0] # tool_name, server_name
return prefixed_tool_name, "" # No prefix found, return original name
return parts[1], parts[0]
return prefixed_name, ""
def is_tool_name_prefixed(tool_name: str) -> bool:
"""
@ -122,14 +112,17 @@ def is_tool_name_prefixed(tool_name: str) -> bool:
"""
return MCP_TOOL_PREFIX_SEPARATOR in tool_name
def validate_mcp_server_name(server_name: str, raise_http_exception: bool = False) -> None:
def validate_mcp_server_name(
server_name: str, raise_http_exception: bool = False
) -> None:
"""
Validate that MCP server name does not contain 'MCP_TOOL_PREFIX_SEPARATOR'.
Args:
server_name: The server name to validate
raise_http_exception: If True, raises HTTPException instead of generic Exception
Raises:
Exception or HTTPException: If server name contains 'MCP_TOOL_PREFIX_SEPARATOR'
"""
@ -138,9 +131,9 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals
if raise_http_exception:
from fastapi import HTTPException
from starlette import status
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"error": error_message}
status_code=status.HTTP_400_BAD_REQUEST, detail={"error": error_message}
)
else:
raise Exception(error_message)

0
litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/zzKcMfj4Db-ZZ7hcspdhR/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/RP6qFLO9Sa0mS2DSQFL5i/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/1116-2d5ec30ef7d86f0e.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1160-08491effeedbaae3.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/131-d505623ce13e3958.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1491-80dbf1ebc561e9b6.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1491-d6d75a98cb22e323.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1518-708cd3bb943546cf.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/1567-5650c3aa172cd633.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/1598-b4dd64df9427ab1c.js → litellm/proxy/_experimental/out/_next/static/chunks/1598-a69df0ba9a12c4f3.js
View file

File diff suppressed because one or more lines are too long

0
litellm/proxy/_experimental/out/_next/static/chunks/1739-1495570659e854dc.js → litellm/proxy/_experimental/out/_next/static/chunks/1739-b555a2473b0946bb.js
View file

2
litellm/proxy/_experimental/out/_next/static/chunks/7220-6db32c86493cbcd6.js → litellm/proxy/_experimental/out/_next/static/chunks/1960-95ee080c178a4e18.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/2284-6840f6cabd9dbd7e.js → litellm/proxy/_experimental/out/_next/static/chunks/1994-2289bfd6e7d65533.js
View file

File diff suppressed because one or more lines are too long

0
litellm/proxy/_experimental/out/_next/static/chunks/2004-2acee05add5250f4.js → litellm/proxy/_experimental/out/_next/static/chunks/2004-a8136543e13a4de3.js
View file

0
litellm/proxy/_experimental/out/_next/static/chunks/2012-0f598b370e2766b0.js → litellm/proxy/_experimental/out/_next/static/chunks/2012-db5a85de324150e9.js
View file

0
litellm/proxy/_experimental/out/_next/static/chunks/2019-02887e9caea1e319.js → litellm/proxy/_experimental/out/_next/static/chunks/2019-2005428b5853a7a6.js
View file

0
litellm/proxy/_experimental/out/_next/static/chunks/2202-404db85e26800c7a.js → litellm/proxy/_experimental/out/_next/static/chunks/2202-c4a45b6cf77ca4a9.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/2249-b6e8bcded6cfd197.js Normal file
View file

File diff suppressed because one or more lines are too long

0
litellm/proxy/_experimental/out/_next/static/chunks/2273-40f31894fc8c0e24.js → litellm/proxy/_experimental/out/_next/static/chunks/2273-4649f34440d8d399.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/2312-02f166690919a2cc.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2358-ff0b1979f8905d3f.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2409-33ed6e753f8afe9b.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2409-7de3355d049b247b.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/7519-c017e0c3b119c340.js → litellm/proxy/_experimental/out/_next/static/chunks/2837-753bc846fbc41ba6.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/2924-98771dd2cd6a8a3d.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/3135-fbf13217ca7d2de9.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/3603-dd19ac8e31e4bc25.js
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more