scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

Merge remote-tracking branch 'origin' into litellm_scim_v2_fix

Browse source
This commit is contained in:
yuneng-jiang 2025-12-06 20:01:05 -08:00
commit a560db56d5
502 changed files with 14065 additions and 1911 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -274,8 +274,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
docker compose up
```

2
ci_cd/security_scans.sh
View file

@ -76,6 +76,8 @@ run_grype_scans() {
"GHSA-4xh5-x5gv-qwph"
"CVE-2025-8291" # no fix available as of Oct 11, 2025
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
)
# Build JSON array of allowlisted CVE IDs for jq

87
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
View file

@ -404,6 +404,93 @@ This release has a known issue...
- **New Providers** - Provider name, supported endpoints, description
- **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link
- Only include major new provider integrations, not minor provider updates
- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13)
### 12. Section Header Counts
**Always include counts in section headers for:**
- **New Providers** - Add count in parentheses: `### New Providers (X new providers)`
- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)`
- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)`
**Format:**
```markdown
### New Providers (4 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | --------------------------- | ----------- |
...
### New LLM API Endpoints (2 new endpoints)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
...
#### New Model Support (32 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
...
```
**Counting Rules:**
- Count each row in the table (excluding the header row)
- For models, count each model entry in the pricing table
- For providers, count each new provider added
- For endpoints, count each new API endpoint added
### 13. Update provider_endpoints_support.json
**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.**
This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation.
**Required Steps:**
1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json`
2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers
**Provider Entry Format:**
```json
"provider_slug": {
"display_name": "Provider Name (`provider_slug`)",
"url": "https://docs.litellm.ai/docs/providers/provider_slug",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": true
}
}
```
**Available Endpoint Types:**
- `chat_completions` - `/chat/completions` endpoint
- `messages` - `/messages` endpoint (Anthropic format)
- `responses` - `/responses` endpoint (OpenAI/Anthropic unified)
- `embeddings` - `/embeddings` endpoint
- `image_generations` - `/image/generations` endpoint
- `audio_transcriptions` - `/audio/transcriptions` endpoint
- `audio_speech` - `/audio/speech` endpoint
- `moderations` - `/moderations` endpoint
- `batches` - `/batches` endpoint
- `rerank` - `/rerank` endpoint
- `ocr` - `/ocr` endpoint
- `search` - `/search` endpoint
- `vector_stores` - `/vector_stores` endpoint
- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol)
**Checklist:**
- [ ] All new providers from release notes are added to `provider_endpoints_support.json`
- [ ] Endpoint support flags accurately reflect provider capabilities
- [ ] Documentation URL points to correct provider docs page
## Example Command Workflow

37
cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py
View file

@ -361,41 +361,6 @@ async def health():
return {"status": "healthy"}
@app.post(
"/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
response_model=BedrockGuardrailResponse,
)
async def apply_guardrail(
guardrailIdentifier: str,
guardrailVersion: str,
request: BedrockRequest,
token: str = Depends(verify_bearer_token),
) -> BedrockGuardrailResponse:
"""
Apply guardrail to input or output content.
This endpoint mimics the AWS Bedrock ApplyGuardrail API.
Args:
guardrailIdentifier: The guardrail ID
guardrailVersion: The guardrail version
request: The guardrail request containing content to analyze
token: Bearer token (verified by dependency)
Returns:
BedrockGuardrailResponse with analysis results
"""
# Process the request
response, output_texts = process_guardrail_request(request)
# Log the request (optional, for debugging)
print(f"Guardrail applied: {guardrailIdentifier} v{guardrailVersion}")
print(f"Source: {request.source}")
print(f"Action: {response.action}")
return response
"""
LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing.
@ -427,11 +392,13 @@ class LitellmBasicGuardrailRequest(BaseModel):
texts: List[str]
images: Optional[List[str]] = None
tools: Optional[List[dict]] = None
tool_calls: Optional[List[dict]] = None
request_data: Dict[str, Any] = Field(default_factory=dict)
additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
input_type: Literal["request", "response"]
litellm_call_id: Optional[str] = None
litellm_trace_id: Optional[str] = None
structured_messages: Optional[List[Dict[str, Any]]] = None
class LitellmBasicGuardrailResponse(BaseModel):

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.9
version: 0.4.10
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

10
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -6,6 +6,9 @@ metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- if .Values.deploymentLabels }}
{{- toYaml .Values.deploymentLabels | nindent 4 }}
{{- end }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
@ -126,6 +129,12 @@ spec:
- configMapRef:
name: {{ . }}
{{- end }}
{{- if .Values.command }}
command: {{ toYaml .Values.command | nindent 12 }}
{{- end }}
{{- if .Values.args }}
args: {{ toYaml .Values.args | nindent 12 }}
{{- else }}
args:
- --config
- /etc/litellm/config.yaml
@ -133,6 +142,7 @@ spec:
- --num_workers
- {{ .Values.numWorkers | quote }}
{{- end }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.port }}

68
deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml Normal file
View file

@ -0,0 +1,68 @@
suite: test deployment command, args, and deploymentLabels
templates:
- deployment.yaml
- configmap-litellm.yaml
tests:
- it: should override args when custom args specified
template: deployment.yaml
set:
args:
- --custom-arg1
- value1
- --custom-arg2
asserts:
- equal:
path: spec.template.spec.containers[0].args
value:
- --custom-arg1
- value1
- --custom-arg2
- it: should set custom command when specified
template: deployment.yaml
set:
command:
- /bin/sh
- -c
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- /bin/sh
- -c
- it: should set custom command and args together
template: deployment.yaml
set:
command:
- python
- -u
args:
- my_script.py
- --verbose
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- python
- -u
- equal:
path: spec.template.spec.containers[0].args
value:
- my_script.py
- --verbose
- it: should add deploymentLabels to deployment metadata
template: deployment.yaml
set:
deploymentLabels:
environment: production
team: platform
version: v1.2.3
asserts:
- equal:
path: metadata.labels.environment
value: production
- equal:
path: metadata.labels.team
value: platform
- equal:
path: metadata.labels.version
value: v1.2.3

6
deploy/charts/litellm-helm/values.yaml
View file

@ -30,6 +30,7 @@ serviceAccount:
# annotations for litellm deployment
deploymentAnnotations: {}
deploymentLabels: {}
# annotations for litellm pods
podAnnotations: {}
podLabels: {}
@ -253,6 +254,11 @@ envVars: {}
# Additional environment variables to be added to the deployment as a list of k8s env vars
extraEnvVars: {}
# if you want to override the container command, you can do so here
command: {}
# if you want to override the container args, you can do so here
args: {}
# - name: EXTRA_ENV_VAR
# value: EXTRA_ENV_VAR_VALUE
# Pod Disruption Budget

48
docker/Dockerfile.non_root
View file

@ -10,18 +10,20 @@ WORKDIR /app
# Install build dependencies including Node.js for UI build
USER root
RUN apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm \
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
py3-pip \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
nodejs \
npm && break || sleep 5; \
done \
&& pip install --no-cache-dir --upgrade pip build
# Copy project files
@ -37,7 +39,7 @@ RUN npm install -g npm@latest && npm cache clean --force
RUN cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi
RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
@ -50,11 +52,11 @@ RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
RUN cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done
RUN cd /app/ui/litellm-dashboard && rm -rf ./out
@ -72,8 +74,12 @@ WORKDIR /app
# Install runtime dependencies
USER root
RUN apk upgrade --no-cache && \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done \
&& for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done
# Copy only necessary artifacts from builder stage for runtime
COPY . .
@ -91,7 +97,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
# Remove test files and keys from dependencies
RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
find /usr/lib -type d -path "*/tornado/test" -delete
find /usr/lib -type d -path "*/tornado/test" -delete
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh

8
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -1,14 +1,16 @@
FROM cgr.dev/chainguard/python:latest-dev
FROM python:3.13-alpine
USER root
WORKDIR /app
ENV HOME=/home/litellm
ENV PATH="${HOME}/venv/bin:$PATH"
# Install runtime dependencies
# Note: Using Python 3.13 for compatibility with ddtrace and other packages
# rust and cargo are required for building ddtrace from source
# musl-dev and libffi-dev are needed for some Python packages on Alpine
RUN apk update && \
apk add --no-cache gcc python3-dev openssl openssl-dev
apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip

12
docs/my-website/docs/a2a.md
View file

@ -2,7 +2,17 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# /a2a - Agent Gateway (A2A Protocol)
# Agent Gateway (A2A Protocol) - Overview
Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track request/response logs in LiteLLM Logs. Manage which Teams, Keys can access which Agents onboarded.
<Image
img={require('../img/a2a_gateway.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
<br />
<br />
| Feature | Supported |
|---------|-----------|

259
docs/my-website/docs/a2a_agent_permissions.md Normal file
View file

@ -0,0 +1,259 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# Agent Permission Management
Control which A2A agents can be accessed by specific keys or teams in LiteLLM.
## Overview
Agent Permission Management lets you restrict which agents a LiteLLM Virtual Key or Team can access. This is useful for:
- **Multi-tenant environments**: Give different teams access to different agents
- **Security**: Prevent keys from invoking agents they shouldn't have access to
- **Compliance**: Enforce access policies for sensitive agent workflows
When permissions are configured:
- `GET /v1/agents` only returns agents the key/team can access
- `POST /a2a/{agent_id}` (Invoking an agent) returns `403 Forbidden` if access is denied
## Setting Permissions on a Key
This example shows how to create a key with agent permissions and test access.
### 1. Get Your Agent ID
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Agents** in the sidebar
2. Click into the agent you want
3. Copy the **Agent ID**
<Image
img={require('../img/agent_id.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
</TabItem>
<TabItem value="api" label="API">
```bash title="List all agents" showLineNumbers
curl "http://localhost:4000/v1/agents" \
-H "Authorization: Bearer sk-master-key"
```
Response:
```json title="Response" showLineNumbers
{
"agents": [
{"agent_id": "agent-123", "name": "Support Agent"},
{"agent_id": "agent-456", "name": "Sales Agent"}
]
}
```
</TabItem>
</Tabs>
### 2. Create a Key with Agent Permissions
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Keys** → **Create Key**
2. Expand **Agent Settings**
3. Select the agents you want to allow
<Image
img={require('../img/agent_key.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
</TabItem>
<TabItem value="api" label="API">
```bash title="Create key with agent permissions" showLineNumbers
curl -X POST "http://localhost:4000/key/generate" \
-H "Authorization: Bearer sk-master-key" \
-H "Content-Type: application/json" \
-d '{
"object_permission": {
"agents": ["agent-123"]
}
}'
```
</TabItem>
</Tabs>
### 3. Test Access
**Allowed agent (succeeds):**
```bash title="Invoke allowed agent" showLineNumbers
curl -X POST "http://localhost:4000/a2a/agent-123" \
-H "Authorization: Bearer sk-your-new-key" \
-H "Content-Type: application/json" \
-d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
```
**Blocked agent (fails with 403):**
```bash title="Invoke blocked agent" showLineNumbers
curl -X POST "http://localhost:4000/a2a/agent-456" \
-H "Authorization: Bearer sk-your-new-key" \
-H "Content-Type: application/json" \
-d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
```
Response:
```json title="403 Forbidden Response" showLineNumbers
{
"error": {
"message": "Access denied to agent: agent-456",
"code": 403
}
}
```
## Setting Permissions on a Team
Restrict all keys belonging to a team to only access specific agents.
### 1. Create a Team with Agent Permissions
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Teams** → **Create Team**
2. Expand **Agent Settings**
3. Select the agents you want to allow for this team
<Image
img={require('../img/agent_key.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
</TabItem>
<TabItem value="api" label="API">
```bash title="Create team with agent permissions" showLineNumbers
curl -X POST "http://localhost:4000/team/new" \
-H "Authorization: Bearer sk-master-key" \
-H "Content-Type: application/json" \
-d '{
"team_alias": "support-team",
"object_permission": {
"agents": ["agent-123"]
}
}'
```
Response:
```json title="Response" showLineNumbers
{
"team_id": "team-abc-123",
"team_alias": "support-team"
}
```
</TabItem>
</Tabs>
### 2. Create a Key for the Team
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Keys** → **Create Key**
2. Select the **Team** from the dropdown
<Image
img={require('../img/agent_team.png')}
style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
/>
</TabItem>
<TabItem value="api" label="API">
```bash title="Create key for team" showLineNumbers
curl -X POST "http://localhost:4000/key/generate" \
-H "Authorization: Bearer sk-master-key" \
-H "Content-Type: application/json" \
-d '{
"team_id": "team-abc-123"
}'
```
</TabItem>
</Tabs>
### 3. Test Access
The key inherits agent permissions from the team.
**Allowed agent (succeeds):**
```bash title="Invoke allowed agent" showLineNumbers
curl -X POST "http://localhost:4000/a2a/agent-123" \
-H "Authorization: Bearer sk-team-key" \
-H "Content-Type: application/json" \
-d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
```
**Blocked agent (fails with 403):**
```bash title="Invoke blocked agent" showLineNumbers
curl -X POST "http://localhost:4000/a2a/agent-456" \
-H "Authorization: Bearer sk-team-key" \
-H "Content-Type: application/json" \
-d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
```
## How It Works
```mermaid
flowchart TD
A[Request to invoke agent] --> B{LiteLLM Virtual Key has agent restrictions?}
B -->|Yes| C{LiteLLM Team has agent restrictions?}
B -->|No| D{LiteLLM Team has agent restrictions?}
C -->|Yes| E[Use intersection of key + team permissions]
C -->|No| F[Use key permissions only]
D -->|Yes| G[Inherit team permissions]
D -->|No| H[Allow ALL agents]
E --> I{Agent in allowed list?}
F --> I
G --> I
H --> J[Allow request]
I -->|Yes| J
I -->|No| K[Return 403 Forbidden]
```
| Key Permissions | Team Permissions | Result | Notes |
|-----------------|------------------|--------|-------|
| None | None | Key can access **all** agents | Open access by default when no restrictions are set |
| `["agent-1", "agent-2"]` | None | Key can access `agent-1` and `agent-2` | Key uses its own permissions |
| None | `["agent-1", "agent-3"]` | Key can access `agent-1` and `agent-3` | Key inherits team's permissions |
| `["agent-1", "agent-2"]` | `["agent-1", "agent-3"]` | Key can access `agent-1` only | Intersection of both lists (most restrictive wins) |
## Viewing Permissions
<Tabs>
<TabItem value="ui" label="UI">
1. Go to **Keys** or **Teams**
2. Click into the key/team you want to view
3. Agent permissions are displayed in the info view
</TabItem>
<TabItem value="api" label="API">
```bash title="Get key info" showLineNumbers
curl "http://localhost:4000/key/info?key=sk-your-key" \
-H "Authorization: Bearer sk-master-key"
```
</TabItem>
</Tabs>

116
docs/my-website/docs/adding_provider/generic_guardrail_api.md
View file

@ -54,7 +54,7 @@ Implement `POST /beta/litellm_basic_guardrail_api`
{
"texts": ["extracted text from the request"], // array of text strings
"images": ["base64_encoded_image_data"], // optional array of images
"tools": [ // optional array of tools (OpenAI ChatCompletionToolParam format)
"tools": [ // tool calls sent to the LLM (in the OpenAI Chat Completions spec)
{
"type": "function",
"function": {
@ -69,6 +69,20 @@ Implement `POST /beta/litellm_basic_guardrail_api`
}
}
],
"tool_calls": [ // tool calls received from the LLM (in the OpenAI Chat Completions spec)
{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"San Francisco\"}"
}
}
],
"structured_messages": [ // optional, full messages in OpenAI format (for chat endpoints)
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"}
],
"request_data": {
"user_api_key_hash": "hash of the litellm virtual key used",
"user_api_key_alias": "alias of the litellm virtual key used",
@ -137,8 +151,8 @@ The `tools` parameter provides information about available function/tool definit
}
```
**Limitations:**
- **Input only:** Tools are only passed for `input_type="request"` (pre-call guardrails). Output/response guardrails do not currently receive tool information.
**Availability:**
- **Input only:** Tools are only passed for `input_type="request"` (pre-call guardrails). Output/response guardrails do not currently receive tool definitions.
- **Supported endpoints:** The `tools` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`. Other endpoints do not have tool support.
**Use cases:**
@ -147,6 +161,63 @@ The `tools` parameter provides information about available function/tool definit
- Log tool usage for audit purposes
- Block sensitive tools based on user context
### `tool_calls` Parameter
The `tool_calls` parameter contains actual function/tool invocations being made in the request or response.
**Format:** OpenAI `ChatCompletionMessageToolCall` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/object#chat/object-tool_calls))
**Example:**
```json
{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}"
}
}
```
**Key Difference from `tools`:**
- **`tools`** = Tool definitions/schemas (what tools are *available*)
- **`tool_calls`** = Tool invocations/executions (what tools are *being called* with what arguments)
**Availability:**
- **Both input and output:** Tool calls can be present in both `input_type="request"` (assistant messages requesting tool calls) and `input_type="response"` (LLM responses with tool calls).
- **Supported endpoints:** The `tool_calls` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`.
**Use cases:**
- Validate tool call arguments before execution
- Redact sensitive data from tool call arguments (e.g., PII)
- Log tool invocations for audit/debugging
- Block tool calls with dangerous parameters
- Modify tool call arguments (e.g., enforce constraints, sanitize inputs)
- Monitor tool usage patterns across users/teams
### `structured_messages` Parameter
The `structured_messages` parameter provides the full input in OpenAI chat completion spec format, useful for distinguishing between system and user messages.
**Format:** Array of OpenAI chat completion messages (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages))
**Example:**
```json
[
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"}
]
```
**Availability:**
- **Supported endpoints:** `/v1/chat/completions`, `/v1/messages`, `/v1/responses`
- **Input only:** Only passed for `input_type="request"` (pre-call guardrails)
**Use cases:**
- Apply different policies for system vs user messages
- Enforce role-based content restrictions
- Log structured conversation context
## LiteLLM Configuration
Add to `config.yaml`:
@ -210,7 +281,9 @@ app = FastAPI()
class GuardrailRequest(BaseModel):
texts: List[str]
images: Optional[List[str]] = None
tools: Optional[List[Dict[str, Any]]] = None # OpenAI ChatCompletionToolParam format
tools: Optional[List[Dict[str, Any]]] = None # OpenAI ChatCompletionToolParam format (tool definitions)
tool_calls: Optional[List[Dict[str, Any]]] = None # OpenAI ChatCompletionMessageToolCall format (tool invocations)
structured_messages: Optional[List[Dict[str, Any]]] = None # OpenAI messages format (for chat endpoints)
request_data: Dict[str, Any]
input_type: str # "request" or "response"
litellm_call_id: Optional[str] = None
@ -235,18 +308,49 @@ async def apply_guardrail(request: GuardrailRequest):
blocked_reason="Content contains prohibited terms"
)
# Example: Check tools (if present in request)
# Example: Check tool definitions (if present in request)
if request.tools:
for tool in request.tools:
if tool.get("type") == "function":
function_name = tool.get("function", {}).get("name", "")
# Block sensitive tools
# Block sensitive tool definitions
if function_name in ["delete_data", "access_admin_panel"]:
return GuardrailResponse(
action="BLOCKED",
blocked_reason=f"Tool '{function_name}' is not allowed"
)
# Example: Check tool calls (if present in request or response)
if request.tool_calls:
for tool_call in request.tool_calls:
if tool_call.get("type") == "function":
function_name = tool_call.get("function", {}).get("name", "")
arguments_str = tool_call.get("function", {}).get("arguments", "{}")
# Parse arguments and validate
import json
try:
arguments = json.loads(arguments_str)
# Block dangerous arguments
if "file_path" in arguments and ".." in str(arguments["file_path"]):
return GuardrailResponse(
action="BLOCKED",
blocked_reason="Tool call contains path traversal attempt"
)
except json.JSONDecodeError:
pass
# Example: Check structured messages (if present in request)
if request.structured_messages:
for message in request.structured_messages:
if message.get("role") == "system":
# Apply stricter policies to system messages
if "admin" in message.get("content", "").lower():
return GuardrailResponse(
action="BLOCKED",
blocked_reason="System message contains restricted terms"
)
return GuardrailResponse(action="NONE")
```

130
docs/my-website/docs/contributing/adding_openai_compatible_providers.md Normal file
View file

@ -0,0 +1,130 @@
# Adding OpenAI-Compatible Providers
For simple OpenAI-compatible providers (like Hyperbolic, Nscale, etc.), you can add support by editing a single JSON file.
## Quick Start
1. Edit `litellm/llms/openai_like/providers.json`
2. Add your provider configuration
3. Test with: `litellm.completion(model="your_provider/model-name", ...)`
## Basic Configuration
For a fully OpenAI-compatible provider:
```json
{
"your_provider": {
"base_url": "https://api.yourprovider.com/v1",
"api_key_env": "YOUR_PROVIDER_API_KEY"
}
}
```
That's it! The provider is now available.
## Configuration Options
### Required Fields
- `base_url` - API endpoint (e.g., `https://api.provider.com/v1`)
- `api_key_env` - Environment variable name for API key (e.g., `PROVIDER_API_KEY`)
### Optional Fields
- `api_base_env` - Environment variable to override `base_url`
- `base_class` - Use `"openai_gpt"` (default) or `"openai_like"`
- `param_mappings` - Map OpenAI parameter names to provider-specific names
- `constraints` - Parameter value constraints (min/max)
- `special_handling` - Special behaviors like content format conversion
## Examples
### Simple Provider (Fully Compatible)
```json
{
"hyperbolic": {
"base_url": "https://api.hyperbolic.xyz/v1",
"api_key_env": "HYPERBOLIC_API_KEY"
}
}
```
### Provider with Parameter Mapping
```json
{
"publicai": {
"base_url": "https://api.publicai.co/v1",
"api_key_env": "PUBLICAI_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
}
}
```
### Provider with Constraints
```json
{
"custom_provider": {
"base_url": "https://api.custom.com/v1",
"api_key_env": "CUSTOM_API_KEY",
"constraints": {
"temperature_max": 1.0,
"temperature_min": 0.0
}
}
}
```
## Usage
```python
import litellm
import os
# Set your API key
os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here"
# Use the provider
response = litellm.completion(
model="your_provider/model-name",
messages=[{"role": "user", "content": "Hello"}],
)
```
## When to Use Python Instead
Use a Python config class if you need:
- Custom authentication flows (OAuth, JWT, etc.)
- Complex request/response transformations
- Provider-specific streaming logic
- Advanced tool calling modifications
For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
## Testing
Test your provider:
```bash
# Quick test
python -c "
import litellm
import os
os.environ['PROVIDER_API_KEY'] = 'your-key'
response = litellm.completion(
model='provider/model-name',
messages=[{'role': 'user', 'content': 'test'}]
)
print(response.choices[0].message.content)
"
```
## Reference
See existing providers in `litellm/llms/openai_like/providers.json` for examples.

8
docs/my-website/docs/projects/GraphRAG.md Normal file
View file

@ -0,0 +1,8 @@
# Microsoft GraphRAG
GraphRAG is a data pipeline and transformation suite that extracts meaningful, structured data from unstructured text using the power of LLMs. It uses a graph-based approach to RAG (Retrieval-Augmented Generation) that leverages knowledge graphs to improve reasoning over private datasets.
- [Github](https://github.com/microsoft/graphrag)
- [Docs](https://microsoft.github.io/graphrag/)
- [Paper](https://arxiv.org/pdf/2404.16130)

6
docs/my-website/docs/provider_registration/index.md
View file

@ -2,6 +2,12 @@
title: "Integrate as a Model Provider"
---
## Quick Start for OpenAI-Compatible Providers
If your API is OpenAI-compatible, you can add support by editing a single JSON file. See [Adding OpenAI-Compatible Providers](/docs/contributing/adding_openai_compatible_providers) for the simple approach.
---
This guide focuses on how to setup the classes and configuration necessary to act as a chat provider.
Please see this guide first and look at the existing code in the codebase to understand how to act as a different provider, e.g. handling embeddings or image-generation.

291
docs/my-website/docs/providers/amazon_nova.md Normal file
View file

@ -0,0 +1,291 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Amazon Nova
| Property | Details |
|-------|-------|
| Description | Amazon Nova is a family of foundation models built by Amazon that deliver frontier intelligence and industry-leading price performance. |
| Provider Route on LiteLLM | `amazon_nova/` |
| Provider Doc | [Amazon Nova ↗](https://docs.aws.amazon.com/nova/latest/userguide/what-is-nova.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `v1/responses` |
| Other Supported Endpoints | `v1/messages`, `/generateContent` |
## Authentication
Amazon Nova uses API key authentication. You can obtain your API key from the [Amazon Nova developer console ↗](https://nova.amazon.com/dev/documentation).
```bash
export AMAZON_NOVA_API_KEY="your-api-key"
```
## Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
# Set your API key
os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key"
response = completion(
model="amazon_nova/nova-micro-v1",
messages=[
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello, how are you?"}
]
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
### 1. Setup config.yaml
```yaml
model_list:
- model_name: amazon-nova-micro
litellm_params:
model: amazon_nova/nova-micro-v1
api_key: os.environ/AMAZON_NOVA_API_KEY
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Test it
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "amazon-nova-micro",
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
]
}'
```
</TabItem>
</Tabs>
## Supported Models
| Model Name | Usage | Context Window |
|------------|-------|----------------|
| Nova Micro | `completion(model="amazon_nova/nova-micro-v1", messages=messages)` | 128K tokens |
| Nova Lite | `completion(model="amazon_nova/nova-lite-v1", messages=messages)` | 300K tokens |
| Nova Pro | `completion(model="amazon_nova/nova-pro-v1", messages=messages)` | 300K tokens |
| Nova Premier | `completion(model="amazon_nova/nova-premier-v1", messages=messages)` | 1M tokens |
## Usage - Streaming
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key"
response = completion(
model="amazon_nova/nova-micro-v1",
messages=[
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Tell me about machine learning"}
],
stream=True
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "amazon-nova-micro",
"messages": [
{
"role": "user",
"content": "Tell me about machine learning"
}
],
"stream": true
}'
```
</TabItem>
</Tabs>
## Usage - Function Calling / Tool Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key"
tools = [
{
"type": "function",
"function": {
"name": "getCurrentWeather",
"description": "Get the current weather in a given city",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City and country e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}
]
response = completion(
model="amazon_nova/nova-micro-v1",
messages=[
{"role": "user", "content": "What's the weather like in San Francisco?"}
],
tools=tools
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "amazon-nova-micro",
"messages": [
{
"role": "user",
"content": "What'\''s the weather like in San Francisco?"
}
],
"tools": [
{
"type": "function",
"function": {
"name": "getCurrentWeather",
"description": "Get the current weather in a given city",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "City and country e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}
]
}'
```
</TabItem>
</Tabs>
## Set temperature, top_p, etc.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AMAZON_NOVA_API_KEY"] = "your-api-key"
response = completion(
model="amazon_nova/nova-pro-v1",
messages=[
{"role": "user", "content": "Write a creative story"}
],
temperature=0.8,
max_tokens=500,
top_p=0.9
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on yaml**
```yaml
model_list:
- model_name: amazon-nova-pro
litellm_params:
model: amazon_nova/nova-pro-v1
temperature: 0.8
max_tokens: 500
top_p: 0.9
```
**Set on request**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "amazon-nova-pro",
"messages": [
{
"role": "user",
"content": "Write a creative story"
}
],
"temperature": 0.8,
"max_tokens": 500,
"top_p": 0.9
}'
```
</TabItem>
</Tabs>
## Model Comparison
| Model | Best For | Speed | Cost | Context |
|-------|----------|-------|------|---------|
| **Nova Micro** | Simple tasks, high throughput | Fastest | Lowest | 128K |
| **Nova Lite** | Balanced performance | Fast | Low | 300K |
| **Nova Pro** | Complex reasoning | Medium | Medium | 300K |
| **Nova Premier** | Most advanced tasks | Slower | Higher | 1M |
## Error Handling
Common error codes and their meanings:
- `401 Unauthorized`: Invalid API key
- `429 Too Many Requests`: Rate limit exceeded
- `400 Bad Request`: Invalid request format
- `500 Internal Server Error`: Service temporarily unavailable

31
docs/my-website/docs/providers/gemini.md
View file

@ -2006,3 +2006,34 @@ curl -L -X POST 'http://localhost:4000/v1/chat/completions' \
</TabItem>
</Tabs>
### Image Generation Pricing
Gemini image generation models (like `gemini-3-pro-image-preview`) return `image_tokens` in the response usage. These tokens are priced differently from text tokens:
| Token Type | Price per 1M tokens | Price per token |
|------------|---------------------|-----------------|
| Text output | $12 | $0.000012 |
| Image output | $120 | $0.00012 |
The number of image tokens depends on the output resolution:
| Resolution | Tokens per image | Cost per image |
|------------|------------------|----------------|
| 1K-2K (1024x1024 to 2048x2048) | 1,120 | $0.134 |
| 4K (4096x4096) | 2,000 | $0.24 |
LiteLLM automatically calculates costs using `output_cost_per_image_token` from the model pricing configuration.
**Example response usage:**
```json
{
"completion_tokens_details": {
"reasoning_tokens": 225,
"text_tokens": 0,
"image_tokens": 1120
}
}
```
For more details, see [Google's Gemini pricing documentation](https://ai.google.dev/gemini-api/docs/pricing).

9
docs/my-website/docs/providers/openai.md
View file

@ -191,6 +191,7 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1" # OPTIONAL
| gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |
| gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` |
| gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` |
| gpt-5.1-codex-max | `response = completion(model="gpt-5.1-codex-max", messages=messages)` |
| gpt-4.1 | `response = completion(model="gpt-4.1", messages=messages)` |
| gpt-4.1-mini | `response = completion(model="gpt-4.1-mini", messages=messages)` |
| gpt-4.1-nano | `response = completion(model="gpt-4.1-nano", messages=messages)` |
@ -427,7 +428,7 @@ Expected Response:
### Advanced: Using `reasoning_effort` with `summary` field
By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`) and only sets the effort level without including a reasoning summary.
By default, `reasoning_effort` accepts a string value (`"none"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"`—`"xhigh"` is only supported on `gpt-5.1-codex-max`) and only sets the effort level without including a reasoning summary.
To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI.
@ -494,10 +495,12 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex-mini` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex-max` | `adaptive` | `low`, `medium`, `high`, `xhigh` (no `minimal`) |
| `gpt-5-pro` | `high` | `high` only |
**Note:**
- GPT-5.1 introduced a new `reasoning_effort="none"` setting for faster, lower-latency responses. This replaces the `"minimal"` setting from GPT-5.
- `gpt-5.1-codex-max` is the only model that supports `reasoning_effort="xhigh"`. All other models will reject this value.
- `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error.
- When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column.
@ -509,7 +512,7 @@ The `verbosity` parameter controls the length and detail of responses from GPT-5
**Supported models:** `gpt-5`, `gpt-5.1`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-pro`
**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`) do **not** support the `verbosity` parameter.
**Note:** GPT-5-Codex models (`gpt-5-codex`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`, `gpt-5.1-codex-max`) do **not** support the `verbosity` parameter.
**Use cases:**
- **`"low"`**: Best for concise answers or simple code generation (e.g., SQL queries)
@ -988,4 +991,4 @@ response = completion(
LiteLLM supports OpenAI's video generation models including Sora.
For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md)
For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md)

4
docs/my-website/docs/providers/openai_compatible.md
View file

@ -11,7 +11,7 @@ Selecting `openai` as the provider routes your request to an OpenAI-compatible e
This library **requires** an API key for all requests, either through the `api_key` parameter
or the `OPENAI_API_KEY` environment variable.
If you don’t want to provide a fake API key in each request, consider using a provider that directly matches your
If you don't want to provide a fake API key in each request, consider using a provider that directly matches your
OpenAI-compatible endpoint, such as [`hosted_vllm`](/docs/providers/vllm) or [`llamafile`](/docs/providers/llamafile).
:::
@ -150,4 +150,4 @@ model_list:
api_base: http://my-custom-base
api_key: ""
supports_system_message: False # 👈 KEY CHANGE
```
```

50
docs/my-website/docs/providers/vertex.md
View file

@ -1604,6 +1604,56 @@ litellm.vertex_location = "us-central1 # Your Location
| gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
| gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
## Private Service Connect (PSC) Endpoints
LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments.
### Usage
```python
from litellm import completion
# Use PSC endpoint with custom api_base
response = completion(
model="vertex_ai/1234567890", # Numeric endpoint ID
messages=[{"role": "user", "content": "Hello!"}],
api_base="http://10.96.32.8", # Your PSC endpoint
vertex_project="my-project-id",
vertex_location="us-central1",
use_psc_endpoint_format=True
)
```
**Key Features:**
- Supports both numeric endpoint IDs and custom model names
- Works with both completion and embedding endpoints
- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}`
- Compatible with streaming requests
### Configuration
Add PSC endpoints to your `config.yaml`:
```yaml
model_list:
- model_name: psc-gemini
litellm_params:
model: vertex_ai/1234567890 # Numeric endpoint ID
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
use_psc_endpoint_format: True
- model_name: psc-embedding
litellm_params:
model: vertex_ai/text-embedding-004
api_base: "http://10.96.32.8" # Your PSC endpoint
vertex_project: "my-project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json"
use_psc_endpoint_format: True
```
## Fine-tuned Models
You can call fine-tuned Vertex AI Gemini models through LiteLLM

587
docs/my-website/docs/providers/vertex_embedding.md Normal file
View file

@ -0,0 +1,587 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vertex AI Embedding
## Usage - Embedding
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
from litellm import embedding
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = embedding(
model="vertex_ai/textembedding-gecko",
input=["good morning from litellm"],
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: snowflake-arctic-embed-m-long-1731622468876
litellm_params:
model: vertex_ai/<your-model-id>
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK, Langchain Python SDK
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="snowflake-arctic-embed-m-long-1731622468876",
input = ["good morning from litellm", "this is another item"],
)
print(response)
```
</TabItem>
</Tabs>
#### Supported Embedding Models
All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` |
| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` |
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` |
### Supported OpenAI (Unified) Params
| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) |
|-------|-------------|--------------------|
| `input` | **string or List[string]** | `instances` |
| `dimensions` | **int** | `output_dimensionality` |
| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` |
#### Usage with OpenAI (Unified) Params
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
input_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"input_type": "RETRIEVAL_QUERY",
}
)
print(response)
```
</TabItem>
</Tabs>
### Supported Vertex Specific Params
| param | type |
|-------|-------------|
| `auto_truncate` | **bool** |
| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** |
| `title` | **str** |
#### Usage with Vertex Specific Params (Use `task_type` and `title`)
You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this:
[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
title = "test",
dimensions=1,
auto_truncate=True,
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="text-embedding-004",
input = ["good morning from litellm", "gm"],
dimensions=1,
extra_body = {
"task_type": "RETRIEVAL_QUERY",
"auto_truncate": True,
"title": "test",
}
)
print(response)
```
</TabItem>
</Tabs>
## **BGE Embeddings**
Use BGE (Baidu General Embedding) models deployed on Vertex AI.
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Using BGE on Vertex AI"
import litellm
response = litellm.embedding(
model="vertex_ai/bge/<your-endpoint-id>",
input=["Hello", "World"],
vertex_project="your-project-id",
vertex_location="your-location"
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY">
1. Add model to config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: bge-embedding
litellm_params:
model: vertex_ai/bge/<your-endpoint-id>
vertex_project: "your-project-id"
vertex_location: "us-central1"
vertex_credentials: your-credentials.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```bash
$ litellm --config /path/to/config.yaml
```
3. Make Request using OpenAI Python SDK
```python showLineNumbers title="Making requests to BGE"
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
response = client.embeddings.create(
model="bge-embedding",
input=["good morning from litellm", "this is another item"]
)
print(response)
```
Using a Private Service Connect (PSC) endpoint
```yaml showLineNumbers title="config.yaml (PSC)"
model_list:
- model_name: bge-small-en-v1.5
litellm_params:
model: vertex_ai/bge/1234567890
api_base: http://10.96.32.8 # Your PSC IP
vertex_project: my-project-id #optional
vertex_location: us-central1 #optional
```
</TabItem>
</Tabs>
## **Multi-Modal Embeddings**
Known Limitations:
- Only supports 1 image / video / image per request
- Only supports GCS or base64 encoded images / videos
### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
Using GCS Images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image
)
```
Using base 64 encoded images
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
<Tabs>
<TabItem value="OpenAI SDK" label="OpenAI SDK">
Requests with GCS Image / Video URI
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
)
print(response)
```
Requests with base64 encoded images
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = "data:image/jpeg;base64,...",
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain">
Requests with GCS Image / Video URI
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
print(query_result)
```
Requests with base64 encoded images
```python
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"data:image/jpeg;base64,..."
)
print(query_result)
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="proxy-vtx" label="LiteLLM PROXY (Vertex SDK)">
1. Add model to config.yaml
```yaml
default_vertex_config:
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK
```python
import vertexai
from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video
from vertexai.vision_models import VideoSegmentConfig
from google.auth.credentials import Credentials
LITELLM_PROXY_API_KEY = "sk-1234"
LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai"
import datetime
class CredentialsWrapper(Credentials):
def __init__(self, token=None):
super().__init__()
self.token = token
self.expiry = None # or set to a future date if needed
def refresh(self, request):
pass
def apply(self, headers, token=None):
headers['Authorization'] = f'Bearer {self.token}'
@property
def expired(self):
return False # Always consider the token as non-expired
@property
def valid(self):
return True # Always consider the credentials as valid
credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY)
vertexai.init(
project="adroit-crow-413218",
location="us-central1",
api_endpoint=LITELLM_PROXY_BASE,
credentials = credentials,
api_transport="rest",
)
model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding")
image = Image.load_from_file(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
embeddings = model.get_embeddings(
image=image,
contextual_text="Colosseum",
dimension=1408,
)
print(f"Image Embedding: {embeddings.image_embedding}")
print(f"Text Embedding: {embeddings.text_embedding}")
```
</TabItem>
</Tabs>
### Text + Image + Video Embeddings
<Tabs>
<TabItem value="sdk" label="SDK">
Text + Image
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image
)
```
Text + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
Image + Video
```python
response = await litellm.aembedding(
model="vertex_ai/multimodalembedding@001",
input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
1. Add model to config.yaml
```yaml
model_list:
- model_name: multimodalembedding@001
litellm_params:
model: vertex_ai/multimodalembedding@001
vertex_project: "adroit-crow-413218"
vertex_location: "us-central1"
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
litellm_settings:
drop_params: True
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request use OpenAI Python SDK, Langchain Python SDK
Text + Image
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"],
)
print(response)
```
Text + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
Image + Video
```python
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="multimodalembedding@001",
input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"],
)
print(response)
```
</TabItem>
</Tabs>

17
docs/my-website/docs/proxy/config_settings.md
View file

@ -29,7 +29,8 @@ litellm_settings:
request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
force_ipv4: boolean # If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API
set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION
# Debugging - see debugging docs for more options
# Use `--debug` or `--detailed_debug` CLI flags, or set LITELLM_LOG env var to "INFO", "DEBUG", or "ERROR"
json_logs: boolean # if true, logs will be in json format
# Fallbacks, reliability
@ -171,7 +172,7 @@ router_settings:
| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
| mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) |
| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION |
| set_verbose | boolean | [DEPRECATED - see debugging docs](./debugging) Use `--debug` or `--detailed_debug` CLI flags, or set `LITELLM_LOG` env var to "INFO", "DEBUG", or "ERROR" instead. |
| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) |
| default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) |
| request_timeout | integer | The timeout for requests in seconds. If not set, the default value is `6000 seconds`. [For reference OpenAI Python SDK defaults to `600 seconds`.](https://github.com/openai/openai-python/blob/main/src/openai/_constants.py) |
@ -333,7 +334,7 @@ router_settings:
| caching_groups | Optional[List[tuple]] | List of model groups for caching across model groups. Defaults to None. - e.g. caching_groups=[("openai-gpt-3.5-turbo", "azure-gpt-3.5-turbo")]|
| alerting_config | AlertingConfig | [SDK-only arg] Slack alerting configuration. Defaults to None. [Further Docs](../routing.md#alerting-) |
| assistants_config | AssistantsConfig | Set on proxy via `assistant_settings`. [Further docs](../assistants.md) |
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging.md) If true, sets the logging level to verbose. |
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
| retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
| provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
@ -359,6 +360,7 @@ router_settings:
| AISPEND_ACCOUNT_ID | Account ID for AI Spend
| AISPEND_API_KEY | API Key for AI Spend
| AIOHTTP_CONNECTOR_LIMIT | Connection limit for aiohttp connector. When set to 0, no limit is applied. **Default is 0**
| AIOHTTP_CONNECTOR_LIMIT_PER_HOST | Connection limit per host for aiohttp connector. When set to 0, no limit is applied. **Default is 0**
| AIOHTTP_KEEPALIVE_TIMEOUT | Keep-alive timeout for aiohttp connections in seconds. **Default is 120**
| AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
| AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300**
@ -378,6 +380,7 @@ router_settings:
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
@ -440,6 +443,7 @@ router_settings:
| CYBERARK_CLIENT_CERT | Path to client certificate for CyberArk authentication
| CYBERARK_CLIENT_KEY | Path to client key for CyberArk authentication
| CYBERARK_USERNAME | Username for CyberArk authentication
| CYBERARK_SSL_VERIFY | Flag to enable or disable SSL certificate verification for CyberArk. Default is True
| CONFIDENT_API_KEY | API key for DeepEval integration
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
@ -654,6 +658,8 @@ router_settings:
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@ -798,7 +804,7 @@ router_settings:
| SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_USER_ID | Flag to send user API key user ID to Zscaler AI Guard. Default is False
| SET_VERBOSE | Flag to enable verbose logging
| SET_VERBOSE | [DEPRECATED] Use `LITELLM_LOG` instead with values "INFO", "DEBUG", or "ERROR". See [debugging docs](./debugging)
| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000
| SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly)
| SLACK_WEBHOOK_URL | Webhook URL for Slack integration
@ -840,6 +846,9 @@ router_settings:
| UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
| WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration
| WANDB_HOST | Host URL for Weights & Biases (W&B) service
| WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration
| WEBHOOK_URL | URL for receiving webhooks from external services
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000

108
docs/my-website/docs/proxy/cursor.md Normal file
View file

@ -0,0 +1,108 @@
---
id: cursor
title: /cursor/chat/completions - Cursor Endpoint
description: Accept Responses API input from Cursor and return OpenAI Chat Completions output
---
LiteLLM provides a Cursor-specific endpoint to make Cursor IDE work seamlessly with the LiteLLM Proxy when using BYOK + custom `base_url`.
- Accepts Requests in OpenAI Responses API input format (Cursor sends this)
- Returns Responses in OpenAI Chat Completions format (Cursor expects this)
- Supports streaming and non‑streaming
## Endpoint
- Path: `/cursor/chat/completions`
- Auth: Standard LiteLLM Proxy auth (`Authorization: Bearer <key>`)
- Behavior: Internally routes to LiteLLM `/responses` flow and transforms output to Chat Completions
## Why this exists
When setting up Cursor with BYOK against a custom `base_url`, Cursor sends requests to the Chat Completions endpoint but in the OpenAI Responses API input shape. Without translation, Cursor won’t display streamed output. This endpoint bridges the formats:
- Input: Responses API (`input`, tool calls, etc.)
- Output: Chat Completions (`choices`, `delta`, `finish_reason`, etc.)
## Usage
### Non-streaming
```bash
curl -X POST https://litellm-internal/cursor/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"input": [{"role": "user", "content": "Hello"}]
}'
```
Example response (shape):
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1733333333,
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hello! How can I help you?"
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 8,
"total_tokens": 18
}
}
```
### Streaming
```bash
curl -N -X POST https://litellm-internal/cursor/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-4o",
"input": [{"role": "user", "content": "Hello"}],
"stream": true
}'
```
- Server-Sent Events (SSE)
- Emits `chat.completion.chunk` deltas (`choices[].delta`) and ends with `data: [DONE]`
## Configuration
### Base URL Setup
**Important**: When configuring Cursor IDE to use this endpoint, you must include `/cursor` in the base URL.
Cursor automatically appends `/chat/completions` to the base URL you provide. To ensure requests go to `/cursor/chat/completions`, configure your base URL in Cursor as:
```
Base URL: https://litellm-internal/cursor
```
This way, when Cursor appends `/chat/completions`, the full path becomes `/cursor/chat/completions`, which is the correct endpoint.
**Example**: If your LiteLLM Proxy is running at `https://litellm-internal`, set the base URL in Cursor to `https://litellm-internal/cursor` (not just `https://litellm-internal`).
### General Setup
No special configuration is required beyond your normal LiteLLM Proxy setup. Ensure that:
- Your `config.yaml` includes the models you want to call via this endpoint
- Your Cursor project uses your LiteLLM Proxy `base_url` (with `/cursor` included) and a valid API key
## Notes
- This endpoint is intended specifically for Cursor’s request/response expectations. Other clients should continue to use `/v1/chat/completions` or `/v1/responses` as appropriate.

110
docs/my-website/docs/proxy/customer_usage.md Normal file
View file

@ -0,0 +1,110 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Customer Usage
Track and visualize end-user spend directly in the dashboard. Monitor customer-level usage analytics, spend logs, and activity metrics to understand how your customers are using your LLM services.
This feature is **available in v1.80.8-stable and above**.
## Overview
Customer Usage enables you to track spend and usage for individual customers (end users) by passing an ID in your API requests. This allows you to:
- Track spend per customer automatically
- View customer-level usage analytics in the Admin UI
- Filter spend logs and activity metrics by customer ID
- Set budgets and rate limits per customer
- Monitor customer usage patterns and trends
<Image img={require('../../img/customer_usage.png')} />
## How to Track Spend
Track customer spend by including a `user` field in your API requests. The customer ID will be automatically tracked and associated with all spend from that request.
### Example using cURL
Make a `/chat/completions` call with the `user` field containing your customer ID:
```bash showLineNumbers title="Track spend with customer ID"
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data '{
"model": "gpt-3.5-turbo",
"user": "customer-123", # 👈 CUSTOMER ID
"messages": [
{
"role": "user",
"content": "What is the capital of France?"
}
]
}'
```
The customer ID (`customer-123`) will be automatically upserted into the database with the new spend. If the customer ID already exists, spend will be incremented.
### Example using OpenWebUI
See the [Open WebUI tutorial](../tutorials/openweb_ui.md) for detailed instructions on connecting Open WebUI to LiteLLM and tracking customer usage.
## How to View Spend
### View Spend in Admin UI
Navigate to the Customer Usage tab in the Admin UI to view customer-level spend analytics:
#### 1. Access Customer Usage
Go to the Usage page in the Admin UI (`PROXY_BASE_URL/ui/?login=success&page=new_usage`) and click on the **Customer Usage** tab.
<Image img={require('../../img/customer_usage_ui_navigation.png')} />
#### 2. View Customer Analytics
The Customer Usage dashboard provides:
- **Total spend per customer**: View aggregated spend across all customers
- **Daily spend trends**: See how customer spend changes over time
- **Model usage breakdown**: Understand which models each customer uses
- **Activity metrics**: Track requests, tokens, and success rates per customer
<Image img={require('../../img/customer_usage_analytics.png')} />
#### 3. Filter by Customer
Use the customer filter dropdown to view spend for specific customers:
- Select one or more customer IDs from the dropdown
- View filtered analytics, spend logs, and activity metrics
- Compare spend across different customers
<Image img={require('../../img/customer_usage_filter.png')} />
## Use Cases
### Customer Billing
Track spend per customer to accurately bill your end users:
- Monitor individual customer usage
- Generate invoices based on actual spend
- Set spending limits per customer
### Usage Analytics
Understand how different customers use your service:
- Identify high-value customers
- Analyze usage patterns
- Optimize resource allocation
---
## Related Features
- [Customers / End-User Budgets](./customers.md) - Set budgets and rate limits for customers
- [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics
- [Billing](./billing.md) - Bill customers based on their usage

4
docs/my-website/docs/proxy/deploy.md
View file

@ -26,8 +26,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
docker compose up
```
@ -1072,4 +1070,4 @@ A: We explored MySQL but that was hard to maintain and led to bugs for customers
**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?**
A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)
A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)

2
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -52,8 +52,6 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
source .env
# Start
docker compose up
```

39
docs/my-website/docs/proxy/dynamic_rate_limit.md
View file

@ -175,7 +175,37 @@ general_settings:
litellm --config /path/to/config.yaml
```
#### 2. Create Keys with Priority Levels
### Set priority on either a team or a key
Priority can be set at either the **team level** or **key level**. Team-level priority takes precedence over key-level priority.
**Option A: Set Priority on Team (Recommended)**
All keys within a team will inherit the team's priority. This is useful when you want all keys for a specific environment or project to have the same priority.
```bash
curl -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_alias": "production-team",
"metadata": {"priority": "prod"}
}'
```
Create a key for this team:
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "team-id-from-previous-response"
}'
```
**Option B: Set Priority on Individual Keys**
Set priority directly on the key. This is useful when you need fine-grained control per key.
**Production Key:**
```bash
@ -205,7 +235,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
-d '{}'
```
**Expected Response for both:**
**Expected Response:**
```json
{
"key": "sk-...",
@ -214,6 +244,11 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
}
```
**Priority Resolution Order:**
1. If key belongs to a team with `metadata.priority` set → use team priority
2. Else if key has `metadata.priority` set → use key priority
3. Else → use `default_priority` from config
#### 3. Test Priority Allocation
**Test Production Key (should get 9 RPM):**

79
docs/my-website/docs/response_api.md
View file

@ -81,6 +81,85 @@ for event in stream:
f.write(image_bytes)
```
#### Image Generation (Non-streaming)
Image generation is supported for models that generate images. Generated images are returned in the `output` array with `type: "image_generation_call"`.
**Gemini (Google AI Studio):**
```python showLineNumbers title="Gemini Image Generation"
import litellm
import base64
# Gemini image generation models don't require tools parameter
response = litellm.responses(
model="gemini/gemini-2.5-flash-image",
input="Generate a cute cat playing with yarn"
)
# Access generated images from output
for item in response.output:
if item.type == "image_generation_call":
# item.result contains pure base64 (no data: prefix)
image_bytes = base64.b64decode(item.result)
# Save the image
with open(f"generated_{item.id}.png", "wb") as f:
f.write(image_bytes)
print(f"Image saved: generated_{response.output[0].id}.png")
```
**OpenAI:**
```python showLineNumbers title="OpenAI Image Generation"
import litellm
import base64
# OpenAI models require tools parameter for image generation
response = litellm.responses(
model="openai/gpt-4o",
input="Generate a futuristic city at sunset",
tools=[{"type": "image_generation"}]
)
# Access generated images from output
for item in response.output:
if item.type == "image_generation_call":
image_bytes = base64.b64decode(item.result)
with open(f"generated_{item.id}.png", "wb") as f:
f.write(image_bytes)
```
**Response Format:**
When image generation is successful, the response contains:
```json
{
"id": "resp_abc123",
"status": "completed",
"output": [
{
"type": "image_generation_call",
"id": "resp_abc123_img_0",
"status": "completed",
"result": "iVBORw0KGgo..." // Pure base64 string (no data: prefix)
}
]
}
```
**Supported Models:**
| Provider | Models | Requires `tools` Parameter |
|----------|--------|---------------------------|
| Google AI Studio | `gemini/gemini-2.5-flash-image` | ❌ No |
| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` | ❌ No |
| OpenAI | `gpt-4o`, `gpt-4o-mini`, `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `o3` | ✅ Yes |
| AWS Bedrock | Stability AI, Amazon Nova Canvas models | Model-specific |
| Fal AI | Various image generation models | Check model docs |
**Note:** The `result` field contains pure base64-encoded image data without the `data:image/png;base64,` prefix. You must decode it with `base64.b64decode()` before saving.
#### GET a Response
```python showLineNumbers title="Get Response by ID"
import litellm

226
docs/my-website/docs/tutorials/cursor_integration.md Normal file
View file

@ -0,0 +1,226 @@
---
sidebar_label: "Cursor IDE"
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Cursor IDE Integration with LiteLLM
This tutorial shows you how to integrate Cursor IDE with LiteLLM Proxy, allowing you to use any LiteLLM-supported model through Cursor's interface with BYOK (Bring Your Own Key) and custom base URL.
## Benefits of using Cursor with LiteLLM
When you use Cursor IDE with LiteLLM you get the following benefits:
**Developer Benefits:**
- Universal Model Access: Use any LiteLLM supported model (Anthropic, OpenAI, Vertex AI, Bedrock, etc.) through the Cursor IDE interface.
- Higher Rate Limits & Reliability: Load balance across multiple models and providers to avoid hitting individual provider limits, with fallbacks to ensure you get responses even if one provider fails.
- Streaming Support: Full streaming support with proper response transformation for Cursor's expected format.
**Proxy Admin Benefits:**
- Centralized Management: Control access to all models through a single LiteLLM proxy instance without giving your developers API Keys to each provider.
- Budget Controls: Set spending limits and track costs across all Cursor usage.
- Request Logging: Track all requests made through Cursor for debugging and monitoring.
## Prerequisites
Before you begin, ensure you have:
- Cursor IDE installed
- A running LiteLLM Proxy instance with **HTTPS enabled** (HTTP is not supported)
- A valid LiteLLM Proxy API key
- An HTTPS domain for your LiteLLM Proxy (required by Cursor)
## Quick Start Guide
### Step 1: Install LiteLLM
Install LiteLLM with proxy support:
```bash
pip install litellm[proxy]
```
### Step 2: Configure LiteLLM Proxy
Create a `config.yaml` file with your model configurations:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
general_settings:
master_key: sk-1234567890 # Change this to a secure key
```
### Step 3: Start LiteLLM Proxy
Start the proxy server with HTTPS enabled:
```bash
litellm --config config.yaml --port 4000
```
:::warning HTTPS Required
**Important**: Cursor IDE requires HTTPS connections. HTTP (`http://`) will not work. You must:
- Deploy your LiteLLM Proxy with HTTPS enabled
- Use a valid SSL certificate
- Access the proxy via an HTTPS domain (e.g., `https://your-proxy-domain.com`)
For local development, you'll need to set up HTTPS (e.g., using a reverse proxy like nginx with SSL, or deploying to a cloud service with HTTPS).
:::
### Step 4: Configure Cursor IDE
Configure Cursor IDE to use your LiteLLM proxy with the `/cursor/chat/completions` endpoint:
1. Open Cursor IDE
2. Go to **Settings** → **Features** → **AI**
3. Enable **"Use Custom API"** or **"Bring Your Own Key"**
4. Set the following:
- **Base URL**: `https://your-proxy-domain.com/cursor` (⚠️ **Important**: Must use HTTPS and include `/cursor`)
- **API Key**: Your LiteLLM Proxy API key (e.g., `sk-1234567890`)
:::warning HTTPS Required
Cursor IDE **requires HTTPS** connections. HTTP (`http://`) will not work. You must:
- Use an HTTPS URL for your base URL (e.g., `https://your-proxy-domain.com/cursor`)
- Ensure your LiteLLM Proxy is accessible via HTTPS
- Have a valid SSL certificate configured
:::
**Example Configuration:**
```
Base URL: https://your-proxy-domain.com/cursor
API Key: sk-1234567890
```
Replace `your-proxy-domain.com` with your actual HTTPS domain where LiteLLM Proxy is running.
:::info Why `/cursor` in the base URL?
Cursor automatically appends `/chat/completions` to the base URL you provide. By setting the base URL to `https://your-proxy-domain.com/cursor`, Cursor will send requests to `/cursor/chat/completions`, which is the special endpoint that handles Cursor's Responses API input format and transforms it to Chat Completions output format.
If you set the base URL to just `https://your-proxy-domain.com`, Cursor would send requests to `/chat/completions`, which won't work correctly with Cursor's request format.
:::
### Step 5: Test the Integration
1. Restart Cursor IDE to apply the settings
2. Open a code file and try using Cursor's AI features (completions, chat, etc.)
3. Your requests will now be routed through LiteLLM Proxy
You can verify it's working by:
- Checking the LiteLLM Proxy logs for incoming requests
- Using Cursor's chat feature and seeing responses stream correctly
- Checking your LiteLLM dashboard for request logs and cost tracking
## How It Works
The `/cursor/chat/completions` endpoint is specifically designed to handle Cursor's unique request format:
1. **Input**: Cursor sends requests in OpenAI Responses API format (with `input` field)
2. **Processing**: LiteLLM processes the request through its internal `/responses` flow
3. **Output**: The response is transformed to OpenAI Chat Completions format (with `choices` field) that Cursor expects
This transformation happens automatically for both streaming and non-streaming responses.
## Advanced Configuration
### Using Different Models
You can configure Cursor to use different models by updating your `config.yaml`:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4o
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-3-5-sonnet
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gemini-pro
litellm_params:
model: gemini/gemini-1.5-pro
api_key: os.environ/GEMINI_API_KEY
```
Then in Cursor, you can specify which model to use in your requests.
### Rate Limiting and Budgets
Set up rate limits and budgets in your `config.yaml`:
```yaml showLineNumbers title="config.yaml"
general_settings:
master_key: sk-1234567890
litellm_settings:
# Set max budget per user
max_budget: 100.0
# Set rate limits
rate_limit: 100 # requests per minute
```
### Request Logging
All requests from Cursor will be logged by LiteLLM Proxy. You can:
- View logs in the LiteLLM Admin UI
- Export logs to your preferred logging service
- Track costs per user/team
## Troubleshooting
### Cursor shows no output
- **Check base URL**: Ensure it uses HTTPS and includes `/cursor` (e.g., `https://your-proxy-domain.com/cursor`, not `http://` or without `/cursor`)
- **Verify HTTPS**: Cursor requires HTTPS - HTTP connections will not work
- **Check API key**: Verify your LiteLLM Proxy API key is correct
- **Check proxy logs**: Look for errors in the LiteLLM Proxy logs
### Requests failing
- **Verify HTTPS is enabled**: Cursor requires HTTPS connections. Ensure your LiteLLM Proxy is accessible via HTTPS with a valid SSL certificate
- **Verify proxy is running**: Check that LiteLLM Proxy is accessible at your HTTPS base URL
- **Check SSL certificate**: Ensure your SSL certificate is valid and not expired
- **Check model configuration**: Ensure the model you're trying to use is configured in `config.yaml`
- **Check API keys**: Verify provider API keys are set correctly in environment variables
### HTTP not working
If you're trying to use HTTP (`http://`) and it's not working:
- **This is expected**: Cursor IDE requires HTTPS connections
- **Solution**: Deploy your LiteLLM Proxy with HTTPS enabled (use a reverse proxy like nginx, or deploy to a cloud service that provides HTTPS)
### Streaming not working
The `/cursor/chat/completions` endpoint automatically handles streaming. If streaming isn't working:
- Check that your model supports streaming
- Verify the proxy logs for any transformation errors
- Ensure Cursor IDE is up to date
## Related Documentation
- [Cursor Endpoint Documentation](/docs/proxy/cursor) - Detailed endpoint documentation
- [LiteLLM Proxy Setup](/docs/proxy/quick_start) - General proxy setup guide
- [Model Configuration](/docs/proxy/configs) - How to configure models

BIN
docs/my-website/img/a2a_gateway.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1 MiB

BIN
docs/my-website/img/agent_id.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 230 KiB

BIN
docs/my-website/img/agent_key.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 176 KiB

BIN
docs/my-website/img/agent_team.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 352 KiB

BIN
docs/my-website/img/customer_usage.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 468 KiB

BIN
docs/my-website/img/customer_usage_analytics.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 252 KiB

BIN
docs/my-website/img/customer_usage_filter.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 265 KiB

BIN
docs/my-website/img/customer_usage_ui_navigation.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 390 KiB

6
docs/my-website/package-lock.json generated
View file

@ -14619,9 +14619,9 @@
}
},
"node_modules/mdast-util-to-hast": {
"version": "13.2.0",
"resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz",
"integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==",
"version": "13.2.1",
"resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz",
"integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",

5
docs/my-website/package.json
View file

@ -61,6 +61,7 @@
"mermaid": ">=11.10.0",
"gray-matter": "4.0.3",
"glob": ">=11.1.0",
"node-forge": ">=1.3.2"
"node-forge": ">=1.3.2",
"mdast-util-to-hast": ">=13.2.1"
}
}
}

607
docs/my-website/release_notes/v1.80.8-stable/index.md Normal file
View file

@ -0,0 +1,607 @@
---
title: "[Preview] v1.80.8.rc.1 - Introducing A2A Agent Gateway"
slug: "v1-80-8"
date: 2025-12-06T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.80.8.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.80.8
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Agent Gateway (A2A)** - [Invoke agents through the AI Gateway with request/response logging and access controls](../../docs/a2a)
- **Guardrails API v2** - [Generic Guardrail API with streaming support, structured messages, and tool call checks](../../docs/adding_provider/generic_guardrail_api)
- **Customer (End User) Usage UI** - [Track and visualize end-user spend directly in the dashboard](../../docs/proxy/customer_usage)
- **vLLM Batch + Files API** - [Support for batch and files API with vLLM deployments](../../docs/batches)
- **Dynamic Rate Limiting on Teams** - [Enable dynamic rate limits and priority reservation on team-level](../../docs/proxy/team_budgets)
- **Google Cloud Chirp3 HD** - [New text-to-speech provider with Chirp3 HD voices](../../docs/text_to_speech)
---
### Agent Gateway (A2A)
<Image
img={require('../../img/a2a_gateway.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
This release introduces **A2A Agent Gateway** for LiteLLM, allowing you to invoke and manage A2A agents with the same controls you have for LLM APIs.
As a **LiteLLM Gateway Admin**, you can now do the following:
- **Request/Response Logging** - Every agent invocation is logged to the Logs page with full request and response tracking.
- **Access Control** - Control which Team/Key can access which agents.
As a developer, you can continue using the A2A SDK, all you need to do is point you `A2AClient` to the LiteLLM proxy URL and your API key.
**Works with the A2A SDK:**
```python
from a2a.client import A2AClient
client = A2AClient(
base_url="http://localhost:4000", # Your LiteLLM proxy
api_key="sk-1234" # LiteLLM API key
)
response = client.send_message(
agent_id="my-agent",
message="What's the status of my order?"
)
```
Get started with Agent Gateway here: [Agent Gateway Documentation](../../docs/a2a)
---
### Customer (End User) Usage UI
<Image
img={require('../../img/customer_usage.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
Users can now filter usage statistics by customers, providing the same granular filtering capabilities available for teams and organizations.
**Details:**
- Filter usage analytics, spend logs, and activity metrics by customer ID
- View customer-level breakdowns alongside existing team and user-level filters
- Consistent filtering experience across all usage and analytics views
---
## New Providers and Endpoints
### New Providers (5 new providers)
| Provider | Supported LiteLLM Endpoints | Description |
| -------- | ------------------- | ----------- |
| **[Z.AI (Zhipu AI)](../../docs/providers/zai)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages` | Built-in support for Zhipu AI GLM models |
| **[RAGFlow](../../docs/providers/ragflow)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages`, `/v1/vector_stores` | RAG-based chat completions with vector store support |
| **[PublicAI](../../docs/providers/publicai)** | `/v1/chat/completions`, `/v1/responses`, `/v1/messages` | OpenAI-compatible provider via JSON config |
| **[Google Cloud Chirp3 HD](../../docs/text_to_speech)** | `/v1/audio/speech`, `/v1/audio/speech/stream` | Text-to-speech with Google Cloud Chirp3 HD voices |
### New LLM API Endpoints (2 new endpoints)
| Endpoint | Method | Description | Documentation |
| -------- | ------ | ----------- | ------------- |
| `/v1/agents/invoke` | POST | Invoke A2A agents through the AI Gateway | [Agent Gateway](../../docs/a2a) |
| `/cursor/chat/completions` | POST | Cursor BYOK endpoint - accepts Responses API input, returns Chat Completions output | [Cursor Integration](../../docs/tutorials/cursor_integration) |
---
## New Models / Updated Models
#### New Model Support (33 new models)
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| OpenAI | `gpt-5.1-codex-max` | 400K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API |
| Azure | `azure/gpt-5.1-codex-max` | 400K | $1.25 | $10.00 | Reasoning, vision, PDF input, responses API |
| Anthropic | `claude-opus-4-5` | 200K | $5.00 | $25.00 | Computer use, reasoning, vision |
| Bedrock | `global.anthropic.claude-opus-4-5-20251101-v1:0` | 200K | $5.00 | $25.00 | Computer use, reasoning, vision |
| Bedrock | `amazon.nova-2-lite-v1:0` | 1M | $0.30 | $2.50 | Reasoning, vision, video, PDF input |
| Bedrock | `amazon.titan-image-generator-v2:0` | - | - | $0.008/image | Image generation |
| Fireworks | `fireworks_ai/deepseek-v3p2` | 164K | $1.20 | $1.20 | Function calling, response schema |
| Fireworks | `fireworks_ai/kimi-k2-instruct-0905` | 262K | $0.60 | $2.50 | Function calling, response schema |
| DeepSeek | `deepseek/deepseek-v3.2` | 164K | $0.28 | $0.40 | Reasoning, function calling |
| Mistral | `mistral/mistral-large-3` | 256K | $0.50 | $1.50 | Function calling, vision |
| Azure AI | `azure_ai/mistral-large-3` | 256K | $0.50 | $1.50 | Function calling, vision |
| Moonshot | `moonshot/kimi-k2-0905-preview` | 262K | $0.60 | $2.50 | Function calling, web search |
| Moonshot | `moonshot/kimi-k2-turbo-preview` | 262K | $1.15 | $8.00 | Function calling, web search |
| Moonshot | `moonshot/kimi-k2-thinking-turbo` | 262K | $1.15 | $8.00 | Function calling, web search |
| OpenRouter | `openrouter/deepseek/deepseek-v3.2` | 164K | $0.28 | $0.40 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-haiku-4-5` | 200K | $1.00 | $5.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-opus-4` | 200K | $15.00 | $75.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-opus-4-1` | 200K | $15.00 | $75.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-opus-4-5` | 200K | $5.00 | $25.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-sonnet-4` | 200K | $3.00 | $15.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-claude-sonnet-4-1` | 200K | $3.00 | $15.00 | Reasoning, function calling |
| Databricks | `databricks/databricks-gemini-2-5-flash` | 1M | $0.30 | $2.50 | Function calling |
| Databricks | `databricks/databricks-gemini-2-5-pro` | 1M | $1.25 | $10.00 | Function calling |
| Databricks | `databricks/databricks-gpt-5` | 400K | $1.25 | $10.00 | Function calling |
| Databricks | `databricks/databricks-gpt-5-1` | 400K | $1.25 | $10.00 | Function calling |
| Databricks | `databricks/databricks-gpt-5-mini` | 400K | $0.25 | $2.00 | Function calling |
| Databricks | `databricks/databricks-gpt-5-nano` | 400K | $0.05 | $0.40 | Function calling |
| Vertex AI | `vertex_ai/chirp` | - | $30.00/1M chars | - | Text-to-speech (Chirp3 HD) |
| Z.AI | `zai/glm-4.6` | 200K | $0.60 | $2.20 | Function calling |
| Z.AI | `zai/glm-4.5` | 128K | $0.60 | $2.20 | Function calling |
| Z.AI | `zai/glm-4.5v` | 128K | $0.60 | $1.80 | Function calling, vision |
| Z.AI | `zai/glm-4.5-flash` | 128K | Free | Free | Function calling |
| Vertex AI | `vertex_ai/bge-large-en-v1.5` | - | - | - | BGE Embeddings |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Add `gpt-5.1-codex-max` model pricing and configuration - [PR #17541](https://github.com/BerriAI/litellm/pull/17541)
- Add xhigh reasoning effort for gpt-5.1-codex-max - [PR #17585](https://github.com/BerriAI/litellm/pull/17585)
- Add clear error message for empty LLM endpoint responses - [PR #17445](https://github.com/BerriAI/litellm/pull/17445)
- **[Azure OpenAI](../../docs/providers/azure/azure)**
- Allow reasoning_effort='none' for Azure gpt-5.1 models - [PR #17311](https://github.com/BerriAI/litellm/pull/17311)
- **[Anthropic](../../docs/providers/anthropic)**
- Add `claude-opus-4-5` alias to pricing data - [PR #17313](https://github.com/BerriAI/litellm/pull/17313)
- Parse `<budget:thinking>` blocks for opus 4.5 - [PR #17534](https://github.com/BerriAI/litellm/pull/17534)
- Update new Anthropic features as reviewed - [PR #17142](https://github.com/BerriAI/litellm/pull/17142)
- Skip empty text blocks in Anthropic system messages - [PR #17442](https://github.com/BerriAI/litellm/pull/17442)
- **[Bedrock](../../docs/providers/bedrock)**
- Add Nova embedding support - [PR #17253](https://github.com/BerriAI/litellm/pull/17253)
- Add support for Bedrock Qwen 2 imported model - [PR #17461](https://github.com/BerriAI/litellm/pull/17461)
- Bedrock OpenAI model support - [PR #17368](https://github.com/BerriAI/litellm/pull/17368)
- Add support for file content download for Bedrock batches - [PR #17470](https://github.com/BerriAI/litellm/pull/17470)
- Make streaming chunk size configurable in Bedrock API - [PR #17357](https://github.com/BerriAI/litellm/pull/17357)
- Add experimental latest-user filtering for Bedrock - [PR #17282](https://github.com/BerriAI/litellm/pull/17282)
- Handle Cohere v4 embed response dictionary format - [PR #17220](https://github.com/BerriAI/litellm/pull/17220)
- Remove not compatible beta header from Bedrock - [PR #17301](https://github.com/BerriAI/litellm/pull/17301)
- Add model price and details for Global Opus 4.5 Bedrock endpoint - [PR #17380](https://github.com/BerriAI/litellm/pull/17380)
- **[Gemini (Google AI Studio + Vertex AI)](../../docs/providers/gemini)**
- Add better handling in image generation for Gemini models - [PR #17292](https://github.com/BerriAI/litellm/pull/17292)
- Fix reasoning_content showing duplicate content in streaming responses - [PR #17266](https://github.com/BerriAI/litellm/pull/17266)
- Handle partial JSON chunks after first valid chunk - [PR #17496](https://github.com/BerriAI/litellm/pull/17496)
- Fix Gemini 3 last chunk thinking block - [PR #17403](https://github.com/BerriAI/litellm/pull/17403)
- Fix Gemini image_tokens treated as text tokens in cost calculation - [PR #17554](https://github.com/BerriAI/litellm/pull/17554)
- Make sure that media resolution is only for Gemini 3 model - [PR #17137](https://github.com/BerriAI/litellm/pull/17137)
- **[Vertex AI](../../docs/providers/vertex)**
- Add Google Cloud Chirp3 HD support on /speech - [PR #17391](https://github.com/BerriAI/litellm/pull/17391)
- Add BGE Embeddings support - [PR #17362](https://github.com/BerriAI/litellm/pull/17362)
- Handle global location for Vertex AI image generation endpoint - [PR #17255](https://github.com/BerriAI/litellm/pull/17255)
- Add Google Private API Endpoint to Vertex AI fields - [PR #17382](https://github.com/BerriAI/litellm/pull/17382)
- **[Z.AI (Zhipu AI)](../../docs/providers/zai)**
- Add Z.AI as built-in provider - [PR #17307](https://github.com/BerriAI/litellm/pull/17307)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Add Embedding API support - [PR #17278](https://github.com/BerriAI/litellm/pull/17278)
- Preserve encrypted_content in reasoning items for multi-turn conversations - [PR #17130](https://github.com/BerriAI/litellm/pull/17130)
- **[Databricks](../../docs/providers/databricks)**
- Update Databricks model pricing and add new models - [PR #17277](https://github.com/BerriAI/litellm/pull/17277)
- **[OVHcloud](../../docs/providers/ovhcloud)**
- Add support of audio transcription for OVHcloud - [PR #17305](https://github.com/BerriAI/litellm/pull/17305)
- **[Mistral](../../docs/providers/mistral)**
- Add Mistral Large 3 model support - [PR #17547](https://github.com/BerriAI/litellm/pull/17547)
- **[Moonshot](../../docs/providers/moonshot)**
- Fix missing Moonshot turbo models and fix incorrect pricing - [PR #17432](https://github.com/BerriAI/litellm/pull/17432)
- **[Together AI](../../docs/providers/togetherai)**
- Add context window exception mapping for Together AI - [PR #17284](https://github.com/BerriAI/litellm/pull/17284)
- **[WatsonX](../../docs/providers/watsonx/index)**
- Allow passing zen_api_key dynamically - [PR #16655](https://github.com/BerriAI/litellm/pull/16655)
- Fix Watsonx Audio Transcription API - [PR #17326](https://github.com/BerriAI/litellm/pull/17326)
- Fix audio transcriptions, don't force content type in request headers - [PR #17546](https://github.com/BerriAI/litellm/pull/17546)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Add new model `fireworks_ai/kimi-k2-instruct-0905` - [PR #17328](https://github.com/BerriAI/litellm/pull/17328)
- Add `fireworks/deepseek-v3p2` - [PR #17395](https://github.com/BerriAI/litellm/pull/17395)
- **[DeepSeek](../../docs/providers/deepseek)**
- Support Deepseek 3.2 with Reasoning - [PR #17384](https://github.com/BerriAI/litellm/pull/17384)
- **[Nova Lite 2](../../docs/providers/bedrock)**
- Add Nova Lite 2 reasoning support with reasoningConfig - [PR #17371](https://github.com/BerriAI/litellm/pull/17371)
- **[Ollama](../../docs/providers/ollama)**
- Fix auth not working with ollama.com - [PR #17191](https://github.com/BerriAI/litellm/pull/17191)
- **[Groq](../../docs/providers/groq)**
- Fix supports_response_schema before using json_tool_call workaround - [PR #17438](https://github.com/BerriAI/litellm/pull/17438)
- **[vLLM](../../docs/providers/vllm)**
- Fix empty response + vLLM streaming - [PR #17516](https://github.com/BerriAI/litellm/pull/17516)
- **[Azure AI](../../docs/providers/azure_ai)**
- Migrate Anthropic provider to Azure AI - [PR #17202](https://github.com/BerriAI/litellm/pull/17202)
- Fix GA path for Azure OpenAI realtime models - [PR #17260](https://github.com/BerriAI/litellm/pull/17260)
- **[Bedrock TwelveLabs](../../docs/providers/bedrock#twelvelabs-pegasus---video-understanding)**
- Add support for TwelveLabs Pegasus video understanding - [PR #17193](https://github.com/BerriAI/litellm/pull/17193)
### Bug Fixes
- **[Bedrock](../../docs/providers/bedrock)**
- Fix extra_headers in messages API bedrock invoke - [PR #17271](https://github.com/BerriAI/litellm/pull/17271)
- Fix Bedrock models in model map - [PR #17419](https://github.com/BerriAI/litellm/pull/17419)
- Make Bedrock converse messages respect modify_params as expected - [PR #17427](https://github.com/BerriAI/litellm/pull/17427)
- Fix Anthropic beta headers for Bedrock imported Qwen models - [PR #17467](https://github.com/BerriAI/litellm/pull/17467)
- Preserve usage from JSON response for OpenAI provider in Bedrock - [PR #17589](https://github.com/BerriAI/litellm/pull/17589)
- **[SambaNova](../../docs/providers/sambanova)**
- Fix acompletion throws error with SambaNova models - [PR #17217](https://github.com/BerriAI/litellm/pull/17217)
- **General**
- Fix AttributeError when metadata is null in request body - [PR #17306](https://github.com/BerriAI/litellm/pull/17306)
- Fix 500 error for malformed request - [PR #17291](https://github.com/BerriAI/litellm/pull/17291)
- Respect custom LLM provider in header - [PR #17290](https://github.com/BerriAI/litellm/pull/17290)
- Replace deprecated .dict() with .model_dump() in streaming_handler - [PR #17359](https://github.com/BerriAI/litellm/pull/17359)
---
## LLM API Endpoints
#### Features
- **[Responses API](../../docs/response_api)**
- Add cost tracking for responses API - [PR #17258](https://github.com/BerriAI/litellm/pull/17258)
- Map output_tokens_details of responses API to completion_tokens_details - [PR #17458](https://github.com/BerriAI/litellm/pull/17458)
- Add image generation support for Responses API - [PR #16586](https://github.com/BerriAI/litellm/pull/16586)
- **[Batch API](../../docs/batches)**
- Add vLLM batch+files API support - [PR #15823](https://github.com/BerriAI/litellm/pull/15823)
- Fix optional parameter default value - [PR #17434](https://github.com/BerriAI/litellm/pull/17434)
- Add status parameter as optional for FileObject - [PR #17431](https://github.com/BerriAI/litellm/pull/17431)
- **[Video Generation API](../../docs/videos)**
- Add passthrough cost tracking for Veo - [PR #17296](https://github.com/BerriAI/litellm/pull/17296)
- **[OCR API](../../docs/ocr)**
- Add missing OCR and aOCR to CallTypes enum - [PR #17435](https://github.com/BerriAI/litellm/pull/17435)
- **General**
- Support routing to only websearch supported deployments - [PR #17500](https://github.com/BerriAI/litellm/pull/17500)
#### Bugs
- **General**
- Fix streaming error validation - [PR #17242](https://github.com/BerriAI/litellm/pull/17242)
- Add length validation for empty tool_calls in delta - [PR #17523](https://github.com/BerriAI/litellm/pull/17523)
---
## Management Endpoints / UI
#### Features
- **New Login Page**
- New Login Page UI - [PR #17443](https://github.com/BerriAI/litellm/pull/17443)
- Refactor /login route - [PR #17379](https://github.com/BerriAI/litellm/pull/17379)
- Add auto_redirect_to_sso to UI Config - [PR #17399](https://github.com/BerriAI/litellm/pull/17399)
- Add Auto Redirect to SSO to New Login Page - [PR #17451](https://github.com/BerriAI/litellm/pull/17451)
- **Customer (End User) Usage**
- Customer (end user) Usage feature - [PR #17498](https://github.com/BerriAI/litellm/pull/17498)
- Customer Usage UI - [PR #17506](https://github.com/BerriAI/litellm/pull/17506)
- Add Info Banner for Customer Usage - [PR #17598](https://github.com/BerriAI/litellm/pull/17598)
- **Virtual Keys**
- Standardize API Key vs Virtual Key in UI - [PR #17325](https://github.com/BerriAI/litellm/pull/17325)
- Add User Alias Column to Internal User Table - [PR #17321](https://github.com/BerriAI/litellm/pull/17321)
- Delete Credential Enhancements - [PR #17317](https://github.com/BerriAI/litellm/pull/17317)
- **Models + Endpoints**
- Show all credential values on Edit Credential Modal - [PR #17397](https://github.com/BerriAI/litellm/pull/17397)
- Change Edit Team Models Shown to Match Create Team - [PR #17394](https://github.com/BerriAI/litellm/pull/17394)
- Support Images in Compare UI - [PR #17562](https://github.com/BerriAI/litellm/pull/17562)
- **Callbacks**
- Show all callbacks on UI - [PR #16335](https://github.com/BerriAI/litellm/pull/16335)
- Credentials to use React Query - [PR #17465](https://github.com/BerriAI/litellm/pull/17465)
- **Management Routes**
- Allow admin viewer to access global tag usage - [PR #17501](https://github.com/BerriAI/litellm/pull/17501)
- Allow wildcard routes for nonproxy admin (SCIM) - [PR #17178](https://github.com/BerriAI/litellm/pull/17178)
- Return 404 when a user is not found on /user/info - [PR #16850](https://github.com/BerriAI/litellm/pull/16850)
- **OCI Configuration**
- Enable Oracle Cloud Infrastructure configuration via UI - [PR #17159](https://github.com/BerriAI/litellm/pull/17159)
#### Bugs
- **UI Fixes**
- Fix Request and Response Panel JSONViewer - [PR #17233](https://github.com/BerriAI/litellm/pull/17233)
- Adding Button Loading States to Edit Settings - [PR #17236](https://github.com/BerriAI/litellm/pull/17236)
- Fix Various Text, button state, and test changes - [PR #17237](https://github.com/BerriAI/litellm/pull/17237)
- Fix Fallbacks Immediately Deleting before API resolves - [PR #17238](https://github.com/BerriAI/litellm/pull/17238)
- Remove Feature Flags - [PR #17240](https://github.com/BerriAI/litellm/pull/17240)
- Fix metadata tags and model name display in UI for Azure passthrough - [PR #17258](https://github.com/BerriAI/litellm/pull/17258)
- Change labeling around Vertex Fields - [PR #17383](https://github.com/BerriAI/litellm/pull/17383)
- Remove second scrollbar when sidebar is expanded + tooltip z index - [PR #17436](https://github.com/BerriAI/litellm/pull/17436)
- Fix Select in Edit Membership Modal - [PR #17524](https://github.com/BerriAI/litellm/pull/17524)
- Change useAuthorized Hook to redirect to new Login Page - [PR #17553](https://github.com/BerriAI/litellm/pull/17553)
- **SSO**
- Fix the generic SSO provider - [PR #17227](https://github.com/BerriAI/litellm/pull/17227)
- Clear SSO integration for all users - [PR #17287](https://github.com/BerriAI/litellm/pull/17287)
- Fix SSO users not added to Entra synced team - [PR #17331](https://github.com/BerriAI/litellm/pull/17331)
- **Auth / JWT**
- JWT Auth - Allow using regular OIDC flow with user info endpoints - [PR #17324](https://github.com/BerriAI/litellm/pull/17324)
- Fix litellm user auth not passing issue - [PR #17342](https://github.com/BerriAI/litellm/pull/17342)
- Add other routes in JWT auth - [PR #17345](https://github.com/BerriAI/litellm/pull/17345)
- Fix new org team validate against org - [PR #17333](https://github.com/BerriAI/litellm/pull/17333)
- Fix litellm_enterprise ensure imported routes exist - [PR #17337](https://github.com/BerriAI/litellm/pull/17337)
- Use organization.members instead of deprecated organization field - [PR #17557](https://github.com/BerriAI/litellm/pull/17557)
- **Organizations/Teams**
- Fix organization max budget not enforced - [PR #17334](https://github.com/BerriAI/litellm/pull/17334)
- Fix budget update to allow null max_budget - [PR #17545](https://github.com/BerriAI/litellm/pull/17545)
---
## AI Integrations (2 new integrations)
### Logging (1 new integration)
#### New Integration
- **[Weave](../../docs/proxy/logging)**
- Basic Weave OTEL integration - [PR #17439](https://github.com/BerriAI/litellm/pull/17439)
#### Improvements & Fixes
- **[DataDog](../../docs/proxy/logging#datadog)**
- Fix Datadog callback regression when ddtrace is installed - [PR #17393](https://github.com/BerriAI/litellm/pull/17393)
- **[Arize Phoenix](../../docs/observability/arize_integration)**
- Fix clean arize-phoenix traces - [PR #16611](https://github.com/BerriAI/litellm/pull/16611)
- **[MLflow](../../docs/proxy/logging#mlflow)**
- Fix MLflow streaming spans for Anthropic passthrough - [PR #17288](https://github.com/BerriAI/litellm/pull/17288)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Fix Langfuse logger test mock setup - [PR #17591](https://github.com/BerriAI/litellm/pull/17591)
- **General**
- Improve PII anonymization handling in logging callbacks - [PR #17207](https://github.com/BerriAI/litellm/pull/17207)
### Guardrails (1 new integration)
#### New Integration
- **[Generic Guardrail API](../../docs/adding_provider/generic_guardrail_api)**
- Generic Guardrail API - allows guardrail providers to add INSTANT support for LiteLLM w/out PR to repo - [PR #17175](https://github.com/BerriAI/litellm/pull/17175)
- Guardrails API V2 - user api key metadata, session id, specify input type (request/response), image support - [PR #17338](https://github.com/BerriAI/litellm/pull/17338)
- Guardrails API - add streaming support - [PR #17400](https://github.com/BerriAI/litellm/pull/17400)
- Guardrails API - support tool call checks on OpenAI `/chat/completions`, OpenAI `/responses`, Anthropic `/v1/messages` - [PR #17459](https://github.com/BerriAI/litellm/pull/17459)
- Guardrails API - new `structured_messages` param - [PR #17518](https://github.com/BerriAI/litellm/pull/17518)
- Correctly map a v1/messages call to the anthropic unified guardrail - [PR #17424](https://github.com/BerriAI/litellm/pull/17424)
- Support during_call event type for unified guardrails - [PR #17514](https://github.com/BerriAI/litellm/pull/17514)
#### Improvements & Fixes
- **[Noma Guardrail](../../docs/proxy/guardrails/noma_security)**
- Refactor Noma guardrail to use shared Responses transformation and include system instructions - [PR #17315](https://github.com/BerriAI/litellm/pull/17315)
- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)**
- Handle empty content and error dict responses in guardrails - [PR #17489](https://github.com/BerriAI/litellm/pull/17489)
- Fix Presidio guardrail test TypeError and license base64 decoding error - [PR #17538](https://github.com/BerriAI/litellm/pull/17538)
- **[Tool Permissions](../../docs/proxy/guardrails/tool_permission)**
- Add regex-based tool_name/tool_type matching for tool-permission - [PR #17164](https://github.com/BerriAI/litellm/pull/17164)
- Add images for tool permission guardrail documentation - [PR #17322](https://github.com/BerriAI/litellm/pull/17322)
- **[AIM Guardrails](../../docs/proxy/guardrails/aim_security)**
- Fix AIM guardrail tests - [PR #17499](https://github.com/BerriAI/litellm/pull/17499)
- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)**
- Fix Bedrock Guardrail indent and import - [PR #17378](https://github.com/BerriAI/litellm/pull/17378)
- **General Guardrails**
- Mask all matching keywords in content filter - [PR #17521](https://github.com/BerriAI/litellm/pull/17521)
- Ensure guardrail metadata is preserved in request_data - [PR #17593](https://github.com/BerriAI/litellm/pull/17593)
- Fix apply_guardrail method and improve test isolation - [PR #17555](https://github.com/BerriAI/litellm/pull/17555)
### Secret Managers
- **[CyberArk](../../docs/secret_managers/cyberark)**
- Allow setting SSL verify to false - [PR #17433](https://github.com/BerriAI/litellm/pull/17433)
- **General**
- Make email and secret manager operations independent in key management hooks - [PR #17551](https://github.com/BerriAI/litellm/pull/17551)
---
## Spend Tracking, Budgets and Rate Limiting
- **Rate Limiting**
- Parallel Request Limiter with /messages - [PR #17426](https://github.com/BerriAI/litellm/pull/17426)
- Allow using dynamic rate limit/priority reservation on teams - [PR #17061](https://github.com/BerriAI/litellm/pull/17061)
- Dynamic Rate Limiter - Fix token count increases/decreases by 1 instead of actual count + Redis TTL - [PR #17558](https://github.com/BerriAI/litellm/pull/17558)
- **Spend Logs**
- Deprecate `spend/logs` & add `spend/logs/v2` - [PR #17167](https://github.com/BerriAI/litellm/pull/17167)
- Optimize SpendLogs queries to use timestamp filtering for index usage - [PR #17504](https://github.com/BerriAI/litellm/pull/17504)
- **Enforce User Param**
- Enforce support of enforce_user_param to OpenAI post endpoints - [PR #17407](https://github.com/BerriAI/litellm/pull/17407)
---
## MCP Gateway
- **MCP Configuration**
- Remove URL format validation for MCP server endpoints - [PR #17270](https://github.com/BerriAI/litellm/pull/17270)
- Add stack trace to MCP error message - [PR #17269](https://github.com/BerriAI/litellm/pull/17269)
- **MCP Tool Results**
- Preserve tool metadata in CallToolResult - [PR #17561](https://github.com/BerriAI/litellm/pull/17561)
---
## Agent Gateway (A2A)
- **Agent Invocation**
- Allow invoking agents through AI Gateway - [PR #17440](https://github.com/BerriAI/litellm/pull/17440)
- Allow tracking request/response in "Logs" Page - [PR #17449](https://github.com/BerriAI/litellm/pull/17449)
- **Agent Access Control**
- Enforce Allowed agents by key, team + add agent access groups on backend - [PR #17502](https://github.com/BerriAI/litellm/pull/17502)
- **Agent Gateway UI**
- Allow testing agents on UI - [PR #17455](https://github.com/BerriAI/litellm/pull/17455)
- Set allowed agents by key, team - [PR #17511](https://github.com/BerriAI/litellm/pull/17511)
---
## Performance / Loadbalancing / Reliability improvements
- **Audio/Speech Performance**
- Fix `/audio/speech` performance by using `shared_sessions` - [PR #16739](https://github.com/BerriAI/litellm/pull/16739)
- **Memory Optimization**
- Prevent memory leak in aiohttp connection pooling - [PR #17388](https://github.com/BerriAI/litellm/pull/17388)
- Lazy-load utils to reduce memory + import time - [PR #17171](https://github.com/BerriAI/litellm/pull/17171)
- **Database**
- Update default database connection number - [PR #17353](https://github.com/BerriAI/litellm/pull/17353)
- Update default proxy_batch_write_at number - [PR #17355](https://github.com/BerriAI/litellm/pull/17355)
- Add background health checks to db - [PR #17528](https://github.com/BerriAI/litellm/pull/17528)
- **Proxy Caching**
- Fix proxy caching between requests in aiohttp transport - [PR #17122](https://github.com/BerriAI/litellm/pull/17122)
- **Session Management**
- Fix session consistency, move Lasso API version away from source code - [PR #17316](https://github.com/BerriAI/litellm/pull/17316)
- Conditionally pass enable_cleanup_closed to aiohttp TCPConnector - [PR #17367](https://github.com/BerriAI/litellm/pull/17367)
- **Vector Store**
- Fix vector store configuration synchronization failure - [PR #17525](https://github.com/BerriAI/litellm/pull/17525)
---
## Documentation Updates
- **Provider Documentation**
- Add Azure AI Foundry documentation for Claude models - [PR #17104](https://github.com/BerriAI/litellm/pull/17104)
- Document responses and embedding API for GitHub Copilot - [PR #17456](https://github.com/BerriAI/litellm/pull/17456)
- Add gpt-5.1-codex-max to OpenAI provider documentation - [PR #17602](https://github.com/BerriAI/litellm/pull/17602)
- Update Instructions For Phoenix Integration - [PR #17373](https://github.com/BerriAI/litellm/pull/17373)
- **Guides**
- Add guide on how to debug gateway error vs provider error - [PR #17387](https://github.com/BerriAI/litellm/pull/17387)
- Agent Gateway documentation - [PR #17454](https://github.com/BerriAI/litellm/pull/17454)
- A2A Permission management documentation - [PR #17515](https://github.com/BerriAI/litellm/pull/17515)
- Update docs to link agent hub - [PR #17462](https://github.com/BerriAI/litellm/pull/17462)
- **Projects**
- Add Google ADK and Harbor to projects - [PR #17352](https://github.com/BerriAI/litellm/pull/17352)
- Add Microsoft Agent Lightning to projects - [PR #17422](https://github.com/BerriAI/litellm/pull/17422)
- **Cleanup**
- Cleanup: Remove orphan docs pages and Docusaurus template files - [PR #17356](https://github.com/BerriAI/litellm/pull/17356)
- Remove `source .env` from docs - [PR #17466](https://github.com/BerriAI/litellm/pull/17466)
---
## Infrastructure / CI/CD
- **Helm Chart**
- Add ingress-only labels - [PR #17348](https://github.com/BerriAI/litellm/pull/17348)
- **Docker**
- Add retry logic to apk package installation in Dockerfile.non_root - [PR #17596](https://github.com/BerriAI/litellm/pull/17596)
- Chainguard fixes - [PR #17406](https://github.com/BerriAI/litellm/pull/17406)
- **OpenAPI Schema**
- Refactor add_schema_to_components to move definitions to components/schemas - [PR #17389](https://github.com/BerriAI/litellm/pull/17389)
- **Security**
- Fix security vulnerability: update mdast-util-to-hast to 13.2.1 - [PR #17601](https://github.com/BerriAI/litellm/pull/17601)
- Bump jws from 3.2.2 to 3.2.3 - [PR #17494](https://github.com/BerriAI/litellm/pull/17494)
---
## New Contributors
* @weichiet made their first contribution in [PR #17242](https://github.com/BerriAI/litellm/pull/17242)
* @AndyForest made their first contribution in [PR #17220](https://github.com/BerriAI/litellm/pull/17220)
* @omkar806 made their first contribution in [PR #17217](https://github.com/BerriAI/litellm/pull/17217)
* @v0rtex20k made their first contribution in [PR #17178](https://github.com/BerriAI/litellm/pull/17178)
* @hxomer made their first contribution in [PR #17207](https://github.com/BerriAI/litellm/pull/17207)
* @orgersh92 made their first contribution in [PR #17316](https://github.com/BerriAI/litellm/pull/17316)
* @dannykopping made their first contribution in [PR #17313](https://github.com/BerriAI/litellm/pull/17313)
* @rioiart made their first contribution in [PR #17333](https://github.com/BerriAI/litellm/pull/17333)
* @codgician made their first contribution in [PR #17278](https://github.com/BerriAI/litellm/pull/17278)
* @epistoteles made their first contribution in [PR #17277](https://github.com/BerriAI/litellm/pull/17277)
* @kothamah made their first contribution in [PR #17368](https://github.com/BerriAI/litellm/pull/17368)
* @flozonn made their first contribution in [PR #17371](https://github.com/BerriAI/litellm/pull/17371)
* @richardmcsong made their first contribution in [PR #17389](https://github.com/BerriAI/litellm/pull/17389)
* @matt-greathouse made their first contribution in [PR #17384](https://github.com/BerriAI/litellm/pull/17384)
* @mossbanay made their first contribution in [PR #17380](https://github.com/BerriAI/litellm/pull/17380)
* @mhielpos-asapp made their first contribution in [PR #17376](https://github.com/BerriAI/litellm/pull/17376)
* @Joilence made their first contribution in [PR #17367](https://github.com/BerriAI/litellm/pull/17367)
* @deepaktammali made their first contribution in [PR #17357](https://github.com/BerriAI/litellm/pull/17357)
* @axiomofjoy made their first contribution in [PR #16611](https://github.com/BerriAI/litellm/pull/16611)
* @DevajMody made their first contribution in [PR #17445](https://github.com/BerriAI/litellm/pull/17445)
* @andrewtruong made their first contribution in [PR #17439](https://github.com/BerriAI/litellm/pull/17439)
* @AnasAbdelR made their first contribution in [PR #17490](https://github.com/BerriAI/litellm/pull/17490)
* @dominicfeliton made their first contribution in [PR #17516](https://github.com/BerriAI/litellm/pull/17516)
* @kristianmitk made their first contribution in [PR #17504](https://github.com/BerriAI/litellm/pull/17504)
* @rgshr made their first contribution in [PR #17130](https://github.com/BerriAI/litellm/pull/17130)
* @dominicfallows made their first contribution in [PR #17489](https://github.com/BerriAI/litellm/pull/17489)
* @irfansofyana made their first contribution in [PR #17467](https://github.com/BerriAI/litellm/pull/17467)
* @GusBricker made their first contribution in [PR #17191](https://github.com/BerriAI/litellm/pull/17191)
* @OlivverX made their first contribution in [PR #17255](https://github.com/BerriAI/litellm/pull/17255)
* @withsmilo made their first contribution in [PR #17585](https://github.com/BerriAI/litellm/pull/17585)
---
## Full Changelog
**[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.7-nightly...v1.80.8)**

31
docs/my-website/sidebars.js
View file

@ -105,6 +105,7 @@ const sidebars = {
items: [
"tutorials/claude_responses_api",
"tutorials/cost_tracking_coding",
"tutorials/cursor_integration",
"tutorials/github_copilot_integration",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
@ -129,6 +130,16 @@ const sidebars = {
},
items: [
"proxy/docker_quick_start",
{
type: "link",
label: "A2A Agent Gateway",
href: "https://docs.litellm.ai/docs/a2a",
},
{
type: "link",
label: "MCP Gateway",
href: "https://docs.litellm.ai/docs/mcp",
},
{
"type": "category",
"label": "Config.yaml",
@ -224,6 +235,7 @@ const sidebars = {
"proxy/team_budgets",
"proxy/tag_budgets",
"proxy/customers",
"proxy/customer_usage",
"proxy/dynamic_rate_limit",
"proxy/rate_limit_tiers",
"proxy/temporary_budget_increase",
@ -317,7 +329,14 @@ const sidebars = {
slug: "/supported_endpoints",
},
items: [
"a2a",
{
type: "category",
label: "/a2a - A2A Agent Gateway",
items: [
"a2a",
"a2a_agent_permissions",
],
},
"assistants",
{
type: "category",
@ -473,6 +492,11 @@ const sidebars = {
id: "provider_registration/index",
label: "Integrate as a Model Provider",
},
{
type: "doc",
id: "contributing/adding_openai_compatible_providers",
label: "Add OpenAI-Compatible Provider (JSON)",
},
{
type: "doc",
id: "provider_registration/add_model_pricing",
@ -522,6 +546,7 @@ const sidebars = {
"providers/vertex_ai/videos",
"providers/vertex_partner",
"providers/vertex_self_deployed",
"providers/vertex_embedding",
"providers/vertex_image",
"providers/vertex_speech",
"providers/vertex_batch",
@ -797,6 +822,7 @@ const sidebars = {
type: "category",
label: "Adding Providers",
items: [
"contributing/adding_openai_compatible_providers",
"adding_provider/directory_structure",
"adding_provider/new_rerank_provider",
]
@ -823,13 +849,14 @@ const sidebars = {
"Learn how to deploy + call models from different providers on LiteLLM",
slug: "/project",
},
items: [
items: [
"projects/smolagents",
"projects/mini-swe-agent",
"projects/openai-agents",
"projects/Google ADK",
"projects/Agent Lightning",
"projects/Harbor",
"projects/GraphRAG",
"projects/Docq.AI",
"projects/PDL",
"projects/OpenInterpreter",

BIN
enterprise/dist/litellm_enterprise-0.1.23-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
enterprise/dist/litellm_enterprise-0.1.23.tar.gz vendored Normal file
View file

Binary file not shown.

42
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -141,28 +141,36 @@ async def list_vector_stores(
"""
from litellm.proxy.proxy_server import prisma_client
seen_vector_store_ids = set()
try:
# Get in-memory vector stores
in_memory_vector_stores: List[LiteLLM_ManagedVectorStore] = []
if litellm.vector_store_registry is not None:
in_memory_vector_stores = copy.deepcopy(
litellm.vector_store_registry.vector_stores
)
# Get vector stores from database
# Get vector stores from database (source of truth)
# Only return what's in the database to ensure consistency across instances
vector_stores_from_db = await VectorStoreRegistry._get_vector_stores_from_db(
prisma_client=prisma_client
)
# Also clean up in-memory registry to remove any deleted vector stores
if litellm.vector_store_registry is not None:
db_vector_store_ids = {
vs.get("vector_store_id")
for vs in vector_stores_from_db
if vs.get("vector_store_id")
}
# Remove any in-memory vector stores that no longer exist in database
vector_stores_to_remove = []
for vs in litellm.vector_store_registry.vector_stores:
vs_id = vs.get("vector_store_id")
if vs_id and vs_id not in db_vector_store_ids:
vector_stores_to_remove.append(vs_id)
for vs_id in vector_stores_to_remove:
litellm.vector_store_registry.delete_vector_store_from_registry(
vector_store_id=vs_id
)
verbose_proxy_logger.debug(
f"Removed deleted vector store {vs_id} from in-memory registry"
)
# Combine in-memory and database vector stores
combined_vector_stores: List[LiteLLM_ManagedVectorStore] = []
for vector_store in in_memory_vector_stores + vector_stores_from_db:
vector_store_id = vector_store.get("vector_store_id", None)
if vector_store_id not in seen_vector_store_ids:
combined_vector_stores.append(vector_store)
seen_vector_store_ids.add(vector_store_id)
# Use database as single source of truth for listing
combined_vector_stores: List[LiteLLM_ManagedVectorStore] = vector_stores_from_db
total_count = len(combined_vector_stores)
total_pages = (total_count + page_size - 1) // page_size

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-enterprise"
version = "0.1.22"
version = "0.1.23"
description = "Package for LiteLLM Enterprise features"
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.1.22"
version = "0.1.23"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-enterprise==",

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.10.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.11.tar.gz vendored Normal file
View file

Binary file not shown.

42
litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql Normal file
View file

@ -0,0 +1,42 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyEndUserSpend" (
"id" TEXT NOT NULL,
"end_user_id" TEXT,
"date" TEXT NOT NULL,
"api_key" TEXT NOT NULL,
"model" TEXT,
"model_group" TEXT,
"custom_llm_provider" TEXT,
"mcp_namespaced_tool_name" TEXT,
"prompt_tokens" BIGINT NOT NULL DEFAULT 0,
"completion_tokens" BIGINT NOT NULL DEFAULT 0,
"cache_read_input_tokens" BIGINT NOT NULL DEFAULT 0,
"cache_creation_input_tokens" BIGINT NOT NULL DEFAULT 0,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
"api_requests" BIGINT NOT NULL DEFAULT 0,
"successful_requests" BIGINT NOT NULL DEFAULT 0,
"failed_requests" BIGINT NOT NULL DEFAULT 0,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "LiteLLM_DailyEndUserSpend_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_date_idx" ON "LiteLLM_DailyEndUserSpend"("date");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_idx" ON "LiteLLM_DailyEndUserSpend"("end_user_id");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_api_key_idx" ON "LiteLLM_DailyEndUserSpend"("api_key");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_model_idx" ON "LiteLLM_DailyEndUserSpend"("model");
-- CreateIndex
CREATE INDEX "LiteLLM_DailyEndUserSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyEndUserSpend"("mcp_namespaced_tool_name");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_DailyEndUserSpend_end_user_id_date_api_key_model_cu_key" ON "LiteLLM_DailyEndUserSpend"("end_user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name");

28
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -465,6 +465,34 @@ model LiteLLM_DailyOrganizationSpend {
@@index([mcp_namespaced_tool_name])
}
// Track daily end user (customer) spend metrics per model and key
model LiteLLM_DailyEndUserSpend {
id String @id @default(uuid())
end_user_id String?
date String
api_key String
model String?
model_group String?
custom_llm_provider String?
mcp_namespaced_tool_name String?
prompt_tokens BigInt @default(0)
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
failed_requests BigInt @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name])
@@index([date])
@@index([end_user_id])
@@index([api_key])
@@index([model])
@@index([mcp_namespaced_tool_name])
}
// Track daily team spend metrics per model and key
model LiteLLM_DailyTeamSpend {
id String @id @default(uuid())

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.9"
version = "0.4.11"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.9"
version = "0.4.11"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

50
litellm/__init__.py
View file

@ -265,6 +265,7 @@ heroku_key: Optional[str] = None
cometapi_key: Optional[str] = None
ovhcloud_key: Optional[str] = None
lemonade_key: Optional[str] = None
amazon_nova_api_key: Optional[str] = None
common_cloud_provider_auth_params: dict = {
"params": ["project", "region_name", "token"],
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
@ -520,6 +521,7 @@ perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
zai_models: Set = set()
deepseek_models: Set = set()
runwayml_models: Set = set()
azure_ai_models: Set = set()
@ -571,6 +573,7 @@ ovhcloud_models: Set = set()
ovhcloud_embedding_models: Set = set()
lemonade_models: Set = set()
docker_model_runner_models: Set = set()
amazon_nova_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -711,6 +714,8 @@ def add_known_models():
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.add(key)
elif value.get("litellm_provider") == "zai":
zai_models.add(key)
elif value.get("litellm_provider") == "fal_ai":
fal_ai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
@ -811,6 +816,8 @@ def add_known_models():
lemonade_models.add(key)
elif value.get("litellm_provider") == "docker_model_runner":
docker_model_runner_models.add(key)
elif value.get("litellm_provider") == "amazon_nova":
amazon_nova_models.add(key)
add_known_models()
@ -872,6 +879,7 @@ model_list = list(
| gemini_models
| text_completion_codestral_models
| xai_models
| zai_models
| fal_ai_models
| deepseek_models
| azure_ai_models
@ -960,6 +968,7 @@ models_by_provider: dict = {
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"xai": xai_models,
"zai": zai_models,
"fal_ai": fal_ai_models,
"deepseek": deepseek_models,
"runwayml": runwayml_models,
@ -1010,6 +1019,7 @@ models_by_provider: dict = {
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
"lemonade": lemonade_models,
"clarifai": clarifai_models,
"amazon_nova": amazon_nova_models,
}
# mapping for those models which have larger equivalents
@ -1300,6 +1310,7 @@ from .llms.friendliai.chat.transformation import FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig
from .llms.xai.common_utils import XAIModelInfo
from .llms.zai.chat.transformation import ZAIChatConfig
from .llms.aiml.chat.transformation import AIMLChatConfig
from .llms.volcengine.chat.transformation import (
VolcEngineChatConfig as VolcEngineConfig,
@ -1339,7 +1350,7 @@ from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
from .llms.publicai.chat.transformation import PublicAIChatConfig
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig
@ -1353,6 +1364,7 @@ from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig
from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig
from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig
from .main import * # type: ignore
# Skills API
@ -1497,10 +1509,46 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
# Lazy loading system for heavy modules to reduce initial import time and memory usage
if TYPE_CHECKING:
from litellm.types.utils import ModelInfo as _ModelInfoType
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
completion_cost: Callable[..., float]
response_cost_calculator: Any
modify_integration: Any
# Utils functions - type stubs for truly lazy loaded functions only
# (functions NOT imported via "from .main import *")
get_response_string: Callable[..., str]
supports_function_calling: Callable[..., bool]
supports_web_search: Callable[..., bool]
supports_url_context: Callable[..., bool]
supports_response_schema: Callable[..., bool]
supports_parallel_function_calling: Callable[..., bool]
supports_vision: Callable[..., bool]
supports_audio_input: Callable[..., bool]
supports_audio_output: Callable[..., bool]
supports_system_messages: Callable[..., bool]
supports_reasoning: Callable[..., bool]
acreate: Callable[..., Any]
get_max_tokens: Callable[..., int]
get_model_info: Callable[..., _ModelInfoType]
register_prompt_template: Callable[..., None]
validate_environment: Callable[..., dict]
check_valid_key: Callable[..., bool]
register_model: Callable[..., None]
encode: Callable[..., list]
decode: Callable[..., str]
_calculate_retry_after: Callable[..., float]
_should_retry: Callable[..., bool]
get_supported_openai_params: Callable[..., Optional[list]]
get_api_base: Callable[..., Optional[str]]
get_first_chars_messages: Callable[..., str]
get_provider_fields: Callable[..., List]
get_valid_models: Callable[..., list]
# Response types - truly lazy loaded only (not in main.py or elsewhere)
ModelResponseListIterator: Type[Any]
def __getattr__(name: str) -> Any:

61
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -367,49 +367,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
reasoning_content = None # flush reasoning content
index += 1
elif isinstance(item, ResponseFunctionToolCall):
provider_specific_fields = getattr(
item, "provider_specific_fields", None
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
if provider_specific_fields and not isinstance(
provider_specific_fields, dict
):
provider_specific_fields = (
dict(provider_specific_fields)
if hasattr(provider_specific_fields, "__dict__")
else {}
)
elif hasattr(item, "get") and callable(item.get): # type: ignore
provider_fields = item.get("provider_specific_fields") # type: ignore
if provider_fields:
provider_specific_fields = (
provider_fields
if isinstance(provider_fields, dict)
else (
dict(provider_fields) # type: ignore
if hasattr(provider_fields, "__dict__")
else {}
)
)
function_dict: Dict[str, Any] = {
"name": item.name,
"arguments": item.arguments,
}
if provider_specific_fields:
function_dict["provider_specific_fields"] = provider_specific_fields
tool_call_dict: Dict[str, Any] = {
"id": item.call_id,
"function": function_dict,
"type": "function",
}
if provider_specific_fields:
tool_call_dict["provider_specific_fields"] = (
provider_specific_fields
)
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=index,
)
msg = Message(
content=None,
@ -667,6 +632,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return Reasoning(effort="none") # type: ignore
elif reasoning_effort == "high":
return Reasoning(effort="high")
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh") # type: ignore[typeddict-item]
elif reasoning_effort == "medium":
return Reasoning(effort="medium")
elif reasoning_effort == "low":
@ -718,17 +685,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
}
}
elif format_type == "json_object":
return {
"format": {
"type": "json_object"
}
}
return {"format": {"type": "json_object"}}
elif format_type == "text":
return {
"format": {
"type": "text"
}
}
return {"format": {"type": "text"}}
return None

6
litellm/constants.py
View file

@ -149,6 +149,7 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer"
REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer"
REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer"
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer"
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000))
@ -413,6 +414,7 @@ LITELLM_CHAT_PROVIDERS = [
"ovhcloud",
"lemonade",
"docker_model_runner",
"amazon_nova",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@ -538,6 +540,7 @@ openai_compatible_endpoints: List = [
"https://api.friendli.ai/serverless/v1",
"api.sambanova.ai/v1",
"api.x.ai/v1",
"ollama.com",
"api.galadriel.ai/v1",
"api.llama.com/compat/v1/",
"api.featherless.ai/v1",
@ -545,7 +548,7 @@ openai_compatible_endpoints: List = [
"api.studio.nebius.ai/v1",
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"https://api.moonshot.ai/v1",
"https://platform.publicai.co/v1",
"https://api.publicai.co/v1",
"https://api.v0.dev/v1",
"https://api.morphllm.com/v1",
"https://api.lambda.ai/v1",
@ -587,6 +590,7 @@ openai_compatible_providers: List = [
"github_copilot", # GitHub Copilot Chat API
"novita",
"meta_llama",
"publicai", # PublicAI - JSON-configured provider
"featherless_ai",
"nscale",
"nebius",

36
litellm/cost_calculator.py
View file

@ -860,9 +860,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
usage_obj: Optional[Union[dict, Usage]] = (
completion_response.get("usage", {})
)
usage_obj: Optional[
Union[dict, Usage]
] = completion_response.get("usage", {})
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@ -1066,13 +1066,14 @@ def completion_cost( # noqa: PLR0915
# If model is like "tavily-search", construct "tavily/search" for cost lookup
search_model = f"{custom_llm_provider}/search"
prompt_cost, completion_cost_result = (
search_provider_cost_per_query(
model=search_model,
custom_llm_provider=custom_llm_provider,
number_of_queries=number_of_queries,
optional_params=optional_params,
)
(
prompt_cost,
completion_cost_result,
) = search_provider_cost_per_query(
model=search_model,
custom_llm_provider=custom_llm_provider,
number_of_queries=number_of_queries,
optional_params=optional_params,
)
# Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost)
@ -1080,11 +1081,13 @@ def completion_cost( # noqa: PLR0915
# Apply discount
original_cost = _final_cost
_final_cost, discount_percent, discount_amount = (
_apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
(
_final_cost,
discount_percent,
discount_amount,
) = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
# Store cost breakdown in logging object if available
@ -1329,9 +1332,8 @@ def response_cost_calculator(
response_cost = 0.0
else:
if isinstance(response_object, BaseModel):
response_object._hidden_params["optional_params"] = optional_params
if hasattr(response_object, "_hidden_params"):
response_object._hidden_params["optional_params"] = optional_params
provider_response_cost = get_response_cost_from_hidden_params(
response_object._hidden_params
)

38
litellm/integrations/custom_logger.py
View file

@ -80,6 +80,44 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
self.turn_off_message_logging = turn_off_message_logging
pass
@staticmethod
def get_callback_env_vars(callback_name: Optional[str] = None) -> List[str]:
"""
Return the environment variables associated with a given callback
name as defined in the proxy callback registry.
Args:
callback_name: The name of the callback to look up.
Returns:
List[str]: A list of required environment variable names.
"""
if callback_name is None:
return []
normalized_name = callback_name.lower()
alias_map = {
"langfuse_otel": "langfuse",
}
lookup_name = alias_map.get(normalized_name, normalized_name)
try:
from litellm.proxy._types import AllCallbacks
except Exception:
return []
callbacks = AllCallbacks()
callback_info = getattr(callbacks, lookup_name, None)
if callback_info is None:
return []
params = getattr(callback_info, "litellm_callback_params", None)
if not params:
return []
return list(params)
def log_pre_api_call(self, model, messages, kwargs):
pass

11
litellm/integrations/mlflow.py
View file

@ -129,8 +129,11 @@ class MlflowLogger(CustomLogger):
self._add_chunk_events(span, response_obj)
# If this is the final chunk, end the span. The final chunk
# has complete_streaming_response that gathers the full response.
if final_response := kwargs.get("complete_streaming_response"):
# has the assembled streaming response (key differs between sync/async paths).
final_response = kwargs.get("complete_streaming_response") or kwargs.get(
"async_complete_streaming_response"
)
if final_response:
end_time_ns = int(end_time.timestamp() * 1e9)
self._extract_and_set_chat_attributes(span, kwargs, final_response)
@ -153,7 +156,9 @@ class MlflowLogger(CustomLogger):
span.add_event(
SpanEvent(
name="streaming_chunk",
attributes={"delta": json.dumps(choice.delta.model_dump())},
attributes={
"delta": json.dumps(choice.delta.model_dump, default=str)
},
)
)
except Exception:

15
litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py
View file

@ -74,9 +74,20 @@ class VectorStorePreCallHook(CustomLogger):
if litellm.vector_store_registry is None:
return model, messages, non_default_params
# Get prisma_client for database fallback
prisma_client = None
try:
from litellm.proxy.proxy_server import prisma_client as _prisma_client
prisma_client = _prisma_client
except ImportError:
pass
# Use database fallback to ensure synchronization across instances
vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = (
litellm.vector_store_registry.pop_vector_stores_to_run(
non_default_params=non_default_params, tools=tools
await litellm.vector_store_registry.pop_vector_stores_to_run_with_db_fallback(
non_default_params=non_default_params,
tools=tools,
prisma_client=prisma_client
)
)

44
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -229,6 +229,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "api.deepseek.com/v1":
custom_llm_provider = "deepseek"
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
elif endpoint == "ollama.com":
custom_llm_provider = "ollama"
dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
elif endpoint == "https://api.friendli.ai/serverless/v1":
custom_llm_provider = "friendliai"
dynamic_api_key = get_secret_str(
@ -401,6 +404,8 @@ def get_llm_provider( # noqa: PLR0915
custom_llm_provider = "lemonade"
elif model.startswith("clarifai/"):
custom_llm_provider = "clarifai"
elif model.startswith("amazon_nova"):
custom_llm_provider = "amazon_nova"
if not custom_llm_provider:
if litellm.suppress_debug_info is False:
print() # noqa
@ -468,6 +473,20 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
custom_llm_provider = model.split("/", 1)[0]
model = model.split("/", 1)[1]
# Check JSON providers FIRST (before hardcoded ones)
from litellm.llms.openai_like.dynamic_config import create_config_class
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
if JSONProviderRegistry.exists(custom_llm_provider):
provider_config = JSONProviderRegistry.get(custom_llm_provider)
if provider_config is None:
raise ValueError(f"Provider {custom_llm_provider} not found")
config_class = create_config_class(provider_config)
api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info(
api_base, api_key
)
return model, custom_llm_provider, dynamic_api_key, api_base
if custom_llm_provider == "perplexity":
# perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai
(
@ -544,6 +563,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "https://api.studio.nebius.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("NEBIUS_API_KEY")
elif custom_llm_provider == "ollama":
api_base = (
api_base
or get_secret("OLLAMA_API_BASE")
or "http://localhost:11434"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("OLLAMA_API_KEY")
elif (custom_llm_provider == "ai21_chat") or (
custom_llm_provider == "ai21" and model in litellm.ai21_chat_models
):
@ -663,12 +689,12 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
api_base, api_key
)
elif custom_llm_provider == "zai":
api_base = (
api_base
or get_secret_str("ZAI_API_BASE")
or "https://api.z.ai/api/paas/v4"
(
api_base,
dynamic_api_key,
) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
dynamic_api_key = api_key or get_secret_str("ZAI_API_KEY")
elif custom_llm_provider == "together_ai":
api_base = (
api_base
@ -763,13 +789,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "publicai":
(
api_base,
dynamic_api_key,
) = litellm.PublicAIChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
# publicai is now handled by JSON config (see litellm/llms/openai_like/providers.json)
elif custom_llm_provider == "docker_model_runner":
(
api_base,

6
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -583,9 +583,11 @@ def generic_cost_per_token(
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
image_tokens = completion_tokens_details["image_tokens"]
if text_tokens == 0:
# Only assume all tokens are text if there's NO breakdown at all
# If image_tokens, audio_tokens, or reasoning_tokens exist, respect text_tokens=0
has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
if text_tokens == 0 and not has_token_breakdown:
text_tokens = usage.completion_tokens
if text_tokens == usage.completion_tokens:
is_text_tokens_total = True
## TEXT COST
completion_cost = float(text_tokens) * completion_base_cost

2
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1071,7 +1071,7 @@ def _parse_content_for_reasoning(
return None, message_text
reasoning_match = re.match(
r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)", message_text, re.DOTALL
)
if reasoning_match:

1
litellm/litellm_core_utils/streaming_handler.py
View file

@ -737,6 +737,7 @@ class CustomStreamWrapper:
or (
"tool_calls" in model_response.choices[0].delta
and model_response.choices[0].delta["tool_calls"] is not None
and len(model_response.choices[0].delta["tool_calls"]) > 0
)
or (
"function_call" in model_response.choices[0].delta

115
litellm/llms/amazon_nova/chat/transformation.py Normal file
View file

@ -0,0 +1,115 @@
"""
Translate from OpenAI's `/v1/chat/completions` to Amazon Nova's `/v1/chat/completions`
"""
from typing import Any, List, Optional, Tuple
import httpx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
)
from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
class AmazonNovaChatConfig(OpenAILikeChatConfig):
max_completion_tokens: Optional[int] = None
max_tokens: Optional[int] = None
metadata: Optional[int] = None
temperature: Optional[int] = None
top_p: Optional[int] = None
tools: Optional[list] = None
reasoning_effort: Optional[list] = None
def __init__(
self,
max_completion_tokens: Optional[int] = None,
max_tokens: Optional[int] = None,
temperature: Optional[int] = None,
top_p: Optional[int] = None,
tools: Optional[list] = None,
reasoning_effort: Optional[list] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@property
def custom_llm_provider(self) -> Optional[str]:
return "amazon_nova"
@classmethod
def get_config(cls):
return super().get_config()
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
# Amazon Nova is openai compatible, we just need to set this to custom_openai and have the api_base be Nova's endpoint
api_base = (
api_base
or get_secret_str("AMAZON_NOVA_API_BASE")
or "https://api.nova.amazon.com/v1"
) # type: ignore
# Get API key from multiple sources
key = (
api_key
or litellm.amazon_nova_api_key
or get_secret_str("AMAZON_NOVA_API_KEY")
or litellm.api_key
)
return api_base, key
def get_supported_openai_params(self, model: str) -> List:
return [
"top_p",
"temperature",
"max_tokens",
"max_completion_tokens",
"metadata",
"stop",
"stream",
"stream_options",
"tools",
"tool_choice",
"reasoning_effort"
]
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
model_response = super().transform_response(
model=model,
model_response=model_response,
raw_response=raw_response,
messages=messages,
logging_obj=logging_obj,
request_data=request_data,
encoding=encoding,
optional_params=optional_params,
json_mode=json_mode,
litellm_params=litellm_params,
api_key=api_key,
)
# Storing amazon_nova in the model response for easier cost calculation later
setattr(model_response, "model", "amazon-nova/" + model)
return model_response

21
litellm/llms/amazon_nova/cost_calculation.py Normal file
View file

@ -0,0 +1,21 @@
"""
Helper util for handling amazon nova cost calculation
- e.g.: prompt caching
"""
from typing import TYPE_CHECKING, Tuple
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
if TYPE_CHECKING:
from litellm.types.utils import Usage
def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
Follows the same logic as Anthropic's cost per token calculation.
"""
return generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="amazon_nova"
)

92
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -16,13 +16,20 @@ import json
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
from litellm._logging import verbose_proxy_logger
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.guardrails import GenericGuardrailAPIInputs
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.types.llms.anthropic import (
AllAnthropicToolsValues,
AnthropicMessagesRequest,
)
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
)
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -57,13 +64,22 @@ class AnthropicMessagesHandler(BaseTranslation):
Process input messages by applying guardrails to text content.
"""
messages = data.get("messages")
tools = data.get("tools", None)
if messages is None:
return data
chat_completion_compatible_request = (
LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, data)
)
)
structured_messages = chat_completion_compatible_request.get("messages", [])
texts_to_check: List[str] = []
images_to_check: List[str] = []
tools_to_check: List[ChatCompletionToolParam] = []
tools_to_check: List[ChatCompletionToolParam] = (
chat_completion_compatible_request.get("tools", [])
)
task_mappings: List[Tuple[int, Optional[int]]] = []
# Track (message_index, content_index) for each text
# content_index is None for string content, int for list content
@ -78,12 +94,6 @@ class AnthropicMessagesHandler(BaseTranslation):
task_mappings=task_mappings,
)
if tools is not None:
self._extract_input_tools(
tools=tools,
tools_to_check=tools_to_check,
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
@ -91,6 +101,8 @@ class AnthropicMessagesHandler(BaseTranslation):
inputs["images"] = images_to_check
if tools_to_check:
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -209,7 +221,7 @@ class AnthropicMessagesHandler(BaseTranslation):
user_api_key_dict: Optional[Any] = None,
) -> Any:
"""
Process output response by applying guardrails to text content.
Process output response by applying guardrails to text content and tool calls.
Args:
response: Anthropic MessagesResponse object
@ -221,17 +233,15 @@ class AnthropicMessagesHandler(BaseTranslation):
Modified response with guardrail applied to content
Response Format Support:
- List content: response.content = [{"type": "text", "text": "text here"}, ...]
- List content: response.content = [
{"type": "text", "text": "text here"},
{"type": "tool_use", "id": "...", "name": "...", "input": {...}},
...
]
"""
# Step 0: Check if response has any text content to process
if not self._has_text_content(response):
verbose_proxy_logger.warning(
"Anthropic Messages: No text content in response, skipping guardrail"
)
return response
texts_to_check: List[str] = []
images_to_check: List[str] = []
tool_calls_to_check: List[ChatCompletionToolCallChunk] = []
task_mappings: List[Tuple[int, Optional[int]]] = []
# Track (content_index, None) for each text
@ -239,10 +249,13 @@ class AnthropicMessagesHandler(BaseTranslation):
if not response_content:
return response
# Step 1: Extract all text content from response
# Step 1: Extract all text content and tool calls from response
for content_idx, content_block in enumerate(response_content):
# Check if this is a text block by checking the 'type' field
if isinstance(content_block, dict) and content_block.get("type") == "text":
# Check if this is a text or tool_use block by checking the 'type' field
if isinstance(content_block, dict) and content_block.get("type") in [
"text",
"tool_use",
]:
# Cast to dict to handle the union type properly
self._extract_output_text_and_images(
content_block=cast(Dict[str, Any], content_block),
@ -250,10 +263,11 @@ class AnthropicMessagesHandler(BaseTranslation):
texts_to_check=texts_to_check,
images_to_check=images_to_check,
task_mappings=task_mappings,
tool_calls_to_check=tool_calls_to_check,
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
if texts_to_check or tool_calls_to_check:
# Create a request_data dict with response info and user API key metadata
request_data: dict = {"response": response}
@ -267,6 +281,9 @@ class AnthropicMessagesHandler(BaseTranslation):
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
@ -419,17 +436,32 @@ class AnthropicMessagesHandler(BaseTranslation):
texts_to_check: List[str],
images_to_check: List[str],
task_mappings: List[Tuple[int, Optional[int]]],
tool_calls_to_check: Optional[List[ChatCompletionToolCallChunk]] = None,
) -> None:
"""
Extract text content and images from a response content block.
Extract text content, images, and tool calls from a response content block.
Override this method to customize text/image extraction logic.
Override this method to customize text/image/tool extraction logic.
"""
content_text = content_block.get("text")
if content_text and isinstance(content_text, str):
# Simple string content
texts_to_check.append(content_text)
task_mappings.append((content_idx, None))
content_type = content_block.get("type")
# Extract text content
if content_type == "text":
content_text = content_block.get("text")
if content_text and isinstance(content_text, str):
# Simple string content
texts_to_check.append(content_text)
task_mappings.append((content_idx, None))
# Extract tool calls
elif content_type == "tool_use":
tool_call = AnthropicConfig.convert_tool_use_to_openai_format(
anthropic_tool_content=content_block,
index=content_idx,
)
if tool_calls_to_check is None:
tool_calls_to_check = []
tool_calls_to_check.append(tool_call)
async def _apply_guardrail_responses_to_output(
self,

132
litellm/llms/anthropic/chat/transformation.py
View file

@ -54,10 +54,7 @@ from litellm.types.utils import (
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import (
PromptTokensDetailsWrapper,
ServerToolUse,
)
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.utils import (
ModelResponse,
Usage,
@ -119,6 +116,36 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def get_config(cls):
return super().get_config()
@staticmethod
def convert_tool_use_to_openai_format(
anthropic_tool_content: Dict[str, Any],
index: int,
) -> ChatCompletionToolCallChunk:
"""
Convert Anthropic tool_use format to OpenAI ChatCompletionToolCallChunk format.
Args:
anthropic_tool_content: Anthropic tool_use content block with format:
{"type": "tool_use", "id": "...", "name": "...", "input": {...}}
index: The index of this tool call
Returns:
ChatCompletionToolCallChunk in OpenAI format
"""
tool_call = ChatCompletionToolCallChunk(
id=anthropic_tool_content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=anthropic_tool_content["name"],
arguments=json.dumps(anthropic_tool_content["input"]),
),
index=index,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in anthropic_tool_content:
tool_call["caller"] = cast(Dict[str, Any], anthropic_tool_content["caller"]) # type: ignore[typeddict-item]
return tool_call
def _is_claude_opus_4_5(self, model: str) -> bool:
"""Check if the model is Claude Opus 4.5."""
return "opus-4-5" in model.lower() or "opus_4_5" in model.lower()
@ -279,7 +306,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif tool["type"] == "tool_search_tool_regex_20251119":
# Tool search tool using regex
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
tool_name_obj = tool.get("name", "tool_search_tool_regex")
if not isinstance(tool_name_obj, str):
raise ValueError("Tool search tool must have a valid name")
@ -291,7 +318,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif tool["type"] == "tool_search_tool_bm25_20251119":
# Tool search tool using BM25
from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25
tool_name_obj = tool.get("name", "tool_search_tool_bm25")
if not isinstance(tool_name_obj, str):
raise ValueError("Tool search tool must have a valid name")
@ -309,7 +336,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if returned_tool is not None:
# Only set cache_control on tools that support it (not tool search tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
if tool_type not in (
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
):
if _cache_control is not None:
returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item]
elif _cache_control_function is not None and isinstance(
@ -318,14 +348,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item]
**_cache_control_function # type: ignore
)
## check if defer_loading is set in the tool
_defer_loading = tool.get("defer_loading", None)
_defer_loading_function = tool.get("function", {}).get("defer_loading", None)
if returned_tool is not None:
# Only set defer_loading on tools that support it (not tool search tools or computer tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
if tool_type not in (
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
"computer_20241022",
"computer_20250124",
):
if _defer_loading is not None:
if not isinstance(_defer_loading, bool):
raise ValueError("defer_loading must be a boolean")
@ -334,14 +369,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if not isinstance(_defer_loading_function, bool):
raise ValueError("defer_loading must be a boolean")
returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item]
## check if allowed_callers is set in the tool
_allowed_callers = tool.get("allowed_callers", None)
_allowed_callers_function = tool.get("function", {}).get("allowed_callers", None)
_allowed_callers_function = tool.get("function", {}).get(
"allowed_callers", None
)
if returned_tool is not None:
# Only set allowed_callers on tools that support it (not tool search tools or computer tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
if tool_type not in (
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
"computer_20241022",
"computer_20250124",
):
if _allowed_callers is not None:
if not isinstance(_allowed_callers, list) or not all(
isinstance(item, str) for item in _allowed_callers
@ -354,7 +396,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
):
raise ValueError("allowed_callers must be a list of strings")
returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item]
## check if input_examples is set in the tool
_input_examples = tool.get("input_examples", None)
_input_examples_function = tool.get("function", {}).get("input_examples", None)
@ -423,31 +465,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""Check if tool search tools are present in the tools list."""
if not tools:
return False
for tool in tools:
tool_type = tool.get("type", "")
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
if tool_type in [
"tool_search_tool_regex_20251119",
"tool_search_tool_bm25_20251119",
]:
return True
return False
def _separate_deferred_tools(
self, tools: List
) -> Tuple[List, List]:
def _separate_deferred_tools(self, tools: List) -> Tuple[List, List]:
"""
Separate tools into deferred and non-deferred lists.
Returns:
Tuple of (non_deferred_tools, deferred_tools)
"""
non_deferred = []
deferred = []
for tool in tools:
if tool.get("defer_loading", False):
deferred.append(tool)
else:
non_deferred.append(tool)
return non_deferred, deferred
def _expand_tool_references(
@ -457,28 +500,28 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
) -> List:
"""
Expand tool_reference blocks to full tool definitions.
When Anthropic's tool search returns results, it includes tool_reference blocks
that reference tools by name. This method expands those references to full
tool definitions from the deferred_tools catalog.
Args:
content: Response content that may contain tool_reference blocks
deferred_tools: List of deferred tools that can be referenced
Returns:
Content with tool_reference blocks expanded to full tool definitions
"""
if not deferred_tools:
return content
# Create a mapping of tool names to tool definitions
tool_map = {}
for tool in deferred_tools:
tool_name = tool.get("name") or tool.get("function", {}).get("name")
if tool_name:
tool_map[tool_name] = tool
# Expand tool references in content
expanded_content = []
for item in content:
@ -492,7 +535,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
expanded_content.append(item)
else:
expanded_content.append(item)
return expanded_content
def _map_stop_sequences(
@ -995,7 +1038,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"messages": anthropic_messages,
**optional_params,
}
## Handle output_config (Anthropic-specific parameter)
if "output_config" in optional_params:
output_config = optional_params.get("output_config")
@ -1054,34 +1097,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_content += content["text"]
## TOOL CALLING
elif content["type"] == "tool_use":
tool_call = ChatCompletionToolCallChunk(
id=content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content["name"],
arguments=json.dumps(content["input"]),
),
tool_call = AnthropicConfig.convert_tool_use_to_openai_format(
anthropic_tool_content=content,
index=idx,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in content:
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
tool_calls.append(tool_call)
## SERVER TOOL USE (for tool search)
elif content["type"] == "server_tool_use":
# Server tool use blocks are for tool search - treat as tool calls
tool_call = ChatCompletionToolCallChunk(
id=content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content["name"],
arguments=json.dumps(content.get("input", {})),
),
# Note: using .get("input", {}) for server_tool_use as input may not be present
content_with_input = {**content, "input": content.get("input", {})}
tool_call = AnthropicConfig.convert_tool_use_to_openai_format(
anthropic_tool_content=content_with_input,
index=idx,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in content:
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
tool_calls.append(tool_call)
## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery)
elif content["type"] == "tool_search_tool_result":
@ -1122,7 +1151,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return text_content, citations, thinking_blocks, reasoning_content, tool_calls
def calculate_usage(
self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None
self,
usage_object: dict,
reasoning_content: Optional[str],
completion_response: Optional[dict] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
@ -1160,7 +1192,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_search_requests = cast(
int, _usage["server_tool_use"]["tool_search_requests"]
)
# Count tool_search_requests from content blocks if not in usage
# Anthropic doesn't always include tool_search_requests in the usage object
if tool_search_requests is None and completion_response is not None:

23
litellm/llms/anthropic/common_utils.py
View file

@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool
from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool, ANTHROPIC_HOSTED_TOOLS
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import TokenCountResponse
@ -72,6 +72,17 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return tool["type"]
return None
def is_web_search_tool_used(
self, tools: Optional[List[AllAnthropicToolsValues]]
) -> bool:
"""Returns True if web_search tool is used"""
if tools is None:
return False
for tool in tools:
if "type" in tool and tool["type"].startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
return True
return False
def is_pdf_used(self, messages: List[AllMessageValues]) -> bool:
"""
Set to true if media passed into messages.
@ -252,6 +263,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
pdf_used: bool = False,
file_id_used: bool = False,
mcp_server_used: bool = False,
web_search_tool_used: bool = False,
tool_search_used: bool = False,
programmatic_tool_calling_used: bool = False,
input_examples_used: bool = False,
@ -292,9 +304,12 @@ class AnthropicModelInfo(BaseLLMModelInfo):
if user_anthropic_beta_headers is not None:
betas.update(user_anthropic_beta_headers)
# Don't send any beta headers to Vertex, Vertex has failed requests when they are sent
# Don't send any beta headers to Vertex, except web search which is required
if is_vertex_request is True:
pass
# Vertex AI requires web search beta header for web search to work
if web_search_tool_used:
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
elif len(betas) > 0:
headers["anthropic-beta"] = ",".join(betas)
@ -325,6 +340,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
)
pdf_used = self.is_pdf_used(messages=messages)
file_id_used = self.is_file_id_used(messages=messages)
web_search_tool_used = self.is_web_search_tool_used(tools=tools)
tool_search_used = self.is_tool_search_used(tools=tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
input_examples_used = self.is_input_examples_used(tools=tools)
@ -338,6 +354,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
pdf_used=pdf_used,
api_key=api_key,
file_id_used=file_id_used,
web_search_tool_used=web_search_tool_used,
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,
mcp_server_used=mcp_server_used,

2
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -103,7 +103,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
status_code=400,
)
####### get required params for all anthropic messages requests ######
verbose_logger.debug(f"🔍 TRANSFORMATION DEBUG - Messages: {messages}")
verbose_logger.debug(f"TRANSFORMATION DEBUG - Messages: {messages}")
anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest(
messages=messages,
max_tokens=max_tokens,

5
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -960,7 +960,10 @@ class AmazonConverseConfig(BaseConfig):
bedrock_tools = _bedrock_tools_pt(filtered_tools)
# Set anthropic_beta in additional_request_params if we have any beta features
if anthropic_beta_list:
# ONLY apply to Anthropic/Claude models - other models (e.g., Qwen, Llama) don't support this field
# and will error with "unknown variant anthropic_beta" if included
base_model = BedrockModelInfo.get_base_model(model)
if anthropic_beta_list and base_model.startswith("anthropic"):
# Remove duplicates while preserving order
unique_betas = []
seen = set()

56
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -674,33 +674,39 @@ class BedrockLLM(BaseAWSLLM):
)
## CALCULATING USAGE - bedrock returns usage in the headers
bedrock_input_tokens = response.headers.get(
"x-amzn-bedrock-input-token-count", None
)
bedrock_output_tokens = response.headers.get(
"x-amzn-bedrock-output-token-count", None
)
prompt_tokens = int(
bedrock_input_tokens or litellm.token_counter(messages=messages)
)
completion_tokens = int(
bedrock_output_tokens
or litellm.token_counter(
text=model_response.choices[0].message.content, # type: ignore
count_response_tokens=True,
# Skip if usage was already set (e.g., from JSON response for OpenAI provider)
if not hasattr(model_response, "usage") or getattr(model_response, "usage", None) is None:
bedrock_input_tokens = response.headers.get(
"x-amzn-bedrock-input-token-count", None
)
bedrock_output_tokens = response.headers.get(
"x-amzn-bedrock-output-token-count", None
)
)
model_response.created = int(time.time())
model_response.model = model
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
setattr(model_response, "usage", usage)
prompt_tokens = int(
bedrock_input_tokens or litellm.token_counter(messages=messages)
)
completion_tokens = int(
bedrock_output_tokens
or litellm.token_counter(
text=model_response.choices[0].message.content, # type: ignore
count_response_tokens=True,
)
)
model_response.created = int(time.time())
model_response.model = model
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
setattr(model_response, "usage", usage)
else:
# Ensure created and model are set even if usage was already set
model_response.created = int(time.time())
model_response.model = model
return model_response

15
litellm/llms/gemini/chat/transformation.py
View file

@ -114,20 +114,27 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
img_element = element
_image_url: Optional[str] = None
format: Optional[str] = None
detail: Optional[str] = None
if isinstance(img_element.get("image_url"), dict):
_image_url = img_element["image_url"].get("url") # type: ignore
format = img_element["image_url"].get("format") # type: ignore
detail = img_element["image_url"].get("detail") # type: ignore
else:
_image_url = img_element.get("image_url") # type: ignore
if _image_url and "https://" in _image_url:
image_obj = convert_to_anthropic_image_obj(
_image_url, format=format
)
img_element["image_url"] = ( # type: ignore
convert_generic_image_chunk_to_openai_image_obj(
image_obj
)
converted_image_url = convert_generic_image_chunk_to_openai_image_obj(
image_obj
)
if detail is not None:
img_element["image_url"] = { # type: ignore
"url": converted_image_url,
"detail": detail
}
else:
img_element["image_url"] = converted_image_url # type: ignore
elif element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_id = file_element["file"].get("file_id")

38
litellm/llms/github_copilot/responses/transformation.py
View file

@ -177,6 +177,44 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Return the responses endpoint
return f"{api_base}/responses"
def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
"""
Handle reasoning items for GitHub Copilot, preserving encrypted_content.
GitHub Copilot uses encrypted_content in reasoning items to maintain
conversation state across turns. The parent class strips this field
when converting to OpenAI's ResponseReasoningItem model, which causes
"encrypted content could not be verified" errors on multi-turn requests.
This override preserves encrypted_content while still filtering out
status=None which OpenAI's API rejects.
"""
if item.get("type") == "reasoning":
# Preserve encrypted_content before parent processing
encrypted_content = item.get("encrypted_content")
# Filter out None values for known problematic fields,
# but preserve encrypted_content even if it exists
filtered_item: Dict[str, Any] = {}
for k, v in item.items():
# Always include encrypted_content if present (even if None)
if k == "encrypted_content":
if encrypted_content is not None:
filtered_item[k] = v
continue
# Filter out status=None which OpenAI API rejects
if k == "status" and v is None:
continue
# Include all other non-None values
if v is not None:
filtered_item[k] = v
verbose_logger.debug(
f"GitHub Copilot reasoning item processed, encrypted_content preserved: {encrypted_content is not None}"
)
return filtered_item
return item
# ==================== Helper Methods ====================
def _get_input_from_params(

27
litellm/llms/oci/chat/transformation.py
View file

@ -416,15 +416,34 @@ class OCIChatConfig(BaseConfig):
"Please install it with: pip install cryptography"
) from e
# Handle oci_key - it should be a string (PEM content)
oci_key_content = None
if oci_key:
if isinstance(oci_key, str):
oci_key_content = oci_key
# Fix common issues with PEM content
# Replace escaped newlines with actual newlines
oci_key_content = oci_key_content.replace("\\n", "\n")
# Ensure proper line endings
if "\r\n" in oci_key_content:
oci_key_content = oci_key_content.replace("\r\n", "\n")
else:
raise OCIError(
status_code=400,
message=f"oci_key must be a string containing the PEM private key content. "
f"Got type: {type(oci_key).__name__}",
)
private_key = (
load_private_key_from_str(oci_key)
if oci_key
load_private_key_from_str(oci_key_content)
if oci_key_content
else load_private_key_from_file(oci_key_file) if oci_key_file else None
)
if private_key is None:
raise Exception(
"Private key is required for OCI authentication. Please provide either oci_key or oci_key_file."
raise OCIError(
status_code=400,
message="Private key is required for OCI authentication. Please provide either oci_key or oci_key_file.",
)
signature = private_key.sign(

26
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -25,6 +25,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
def is_model_gpt_5_codex_model(cls, model: str) -> bool:
"""Check if the model is specifically a GPT-5 Codex variant."""
return "gpt-5-codex" in model
@classmethod
def is_model_gpt_5_1_codex_max_model(cls, model: str) -> bool:
"""Check if the model is the gpt-5.1-codex-max variant."""
model_name = model.split("/")[-1] # handle provider prefixes
return model_name == "gpt-5.1-codex-max"
@classmethod
def is_model_gpt_5_1_model(cls, model: str) -> bool:
@ -66,6 +72,22 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
model: str,
drop_params: bool,
) -> dict:
reasoning_effort = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
if reasoning_effort is not None and reasoning_effort == "xhigh":
if not self.is_model_gpt_5_1_codex_max_model(model):
if litellm.drop_params or drop_params:
non_default_params.pop("reasoning_effort", None)
else:
raise litellm.utils.UnsupportedParamsError(
message=(
"reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max."
),
status_code=400,
)
################################################################
# max_tokens is not supported for gpt-5 models on OpenAI API
# Relevant issue: https://github.com/BerriAI/litellm/issues/13381
@ -79,10 +101,6 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
temperature_value: Optional[float] = non_default_params.pop("temperature")
if temperature_value is not None:
is_gpt_5_1 = self.is_model_gpt_5_1_model(model)
reasoning_effort = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
# gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none")
if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None):

58
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -80,6 +80,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check # type: ignore
if messages:
inputs["structured_messages"] = (
messages # pass the openai /chat/completions messages to the guardrail, as-is
)
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
@ -89,7 +93,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
)
guardrailed_texts = guardrailed_inputs.get("texts", [])
guardrailed_tool_calls = guardrailed_inputs.get("tools", [])
guardrailed_tool_calls = guardrailed_inputs.get("tool_calls", [])
# Step 3: Map guardrail responses back to original message structure
if guardrailed_texts and texts_to_check:
@ -155,12 +159,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
images_to_check.append(url)
# Extract tool calls (typically in assistant messages)
tool_calls = message.get("tools", None)
tool_calls = message.get("tool_calls", None)
if tool_calls is not None and isinstance(tool_calls, list):
for tool_call_idx, tool_call in enumerate(tool_calls):
if isinstance(tool_call, dict):
# Add the full tool call object to the list
tool_calls_to_check.append(ChatCompletionToolParam(**tool_call))
tool_calls_to_check.append(cast(ChatCompletionToolParam, tool_call))
tool_call_task_mappings.append((msg_idx, int(tool_call_idx)))
async def _apply_guardrail_responses_to_input_texts(
@ -261,7 +265,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# Step 1: Extract all text content, images, and tool calls from response choices
for choice_idx, choice in enumerate(response.choices):
self._extract_output_text_and_images(
self._extract_output_text_images_and_tool_calls(
choice=choice,
choice_idx=choice_idx,
texts_to_check=texts_to_check,
@ -376,20 +380,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if isinstance(content, str):
# String content - accumulate for this choice
key = (choice_idx, None)
if key not in combined_texts:
combined_texts[key] = ""
combined_texts[key] += content
str_key: Tuple[int, Optional[int]] = (choice_idx, None)
if str_key not in combined_texts:
combined_texts[str_key] = ""
combined_texts[str_key] += content
elif isinstance(content, list):
# List content - accumulate for each content item
for content_idx, content_item in enumerate(content):
text_str = content_item.get("text")
if text_str:
key = (choice_idx, content_idx)
if key not in combined_texts:
combined_texts[key] = ""
combined_texts[key] += text_str
list_key: Tuple[int, Optional[int]] = (choice_idx, content_idx)
if list_key not in combined_texts:
combined_texts[list_key] = ""
combined_texts[list_key] += text_str
# Step 2: Create lists for guardrail processing
texts_to_check: List[str] = []
@ -397,9 +401,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
task_mappings: List[Tuple[int, Optional[int]]] = []
# Track (choice_index, content_index) for each combined text
for (choice_idx, content_idx), combined_text in combined_texts.items():
for (map_choice_idx, map_content_idx), combined_text in combined_texts.items():
texts_to_check.append(combined_text)
task_mappings.append((choice_idx, content_idx))
task_mappings.append((map_choice_idx, map_content_idx))
# Step 3: Apply guardrail to all combined texts in batch
if texts_to_check:
@ -478,7 +482,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return True
return False
def _extract_output_text_and_images(
def _extract_output_text_images_and_tool_calls(
self,
choice: Union[Choices, StreamingChoices],
choice_idx: int,
@ -499,7 +503,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# Determine content source and tool calls based on choice type
content = None
tool_calls = None
tool_calls: Optional[List[Any]] = None
if isinstance(choice, litellm.Choices):
content = choice.message.content
tool_calls = choice.message.tool_calls
@ -682,15 +686,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if isinstance(content, str):
# String content
key = (choice_idx_in_response, None)
if key in guardrail_map:
if key not in already_set:
str_key: Tuple[int, Optional[int]] = (choice_idx_in_response, None)
if str_key in guardrail_map:
if str_key not in already_set:
# First chunk - set the complete guardrailed text
if isinstance(choice, litellm.StreamingChoices):
choice.delta.content = guardrail_map[key]
choice.delta.content = guardrail_map[str_key]
elif isinstance(choice, litellm.Choices):
choice.message.content = guardrail_map[key]
already_set[key] = True
choice.message.content = guardrail_map[str_key]
already_set[str_key] = True
else:
# Subsequent chunks - clear the content
if isinstance(choice, litellm.StreamingChoices):
@ -702,12 +706,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# List content - handle each content item
for content_idx, content_item in enumerate(content):
if "text" in content_item:
key = (choice_idx_in_response, content_idx)
if key in guardrail_map:
if key not in already_set:
list_key: Tuple[int, Optional[int]] = (choice_idx_in_response, content_idx)
if list_key in guardrail_map:
if list_key not in already_set:
# First chunk - set the complete guardrailed text
content_item["text"] = guardrail_map[key]
already_set[key] = True
content_item["text"] = guardrail_map[list_key]
already_set[list_key] = True
else:
# Subsequent chunks - clear the text
content_item["text"] = ""

4
litellm/llms/openai/openai.py
View file

@ -444,7 +444,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
else:
headers = {}
response = raw_response.parse()
if not hasattr(response, "model_dump"):
if not data.get("stream") and not hasattr(response, "model_dump"):
raise OpenAIError(
status_code=500,
message=f"Empty or invalid response from LLM endpoint. Received: {response!r}. Check the reverse proxy or model server configuration.",
@ -482,7 +482,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
else:
headers = {}
response = raw_response.parse()
if not hasattr(response, "model_dump"):
if not data.get("stream") and not hasattr(response, "model_dump"):
raise OpenAIError(
status_code=500,
message=f"Empty or invalid response from LLM endpoint. Received: {response!r}. Check the reverse proxy or model server configuration.",

92
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -38,8 +38,15 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
from litellm.types.guardrails import GenericGuardrailAPIInputs
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolParam,
)
from litellm.types.responses.main import (
GenericResponseOutputItem,
OutputFunctionToolCall,
OutputText,
)
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -74,6 +81,13 @@ class OpenAIResponsesHandler(BaseTranslation):
if input_data is None:
return data
structured_messages = (
LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=input_data,
responses_api_request=data,
)
)
# Handle simple string input
if isinstance(input_data, str):
inputs = GenericGuardrailAPIInputs(texts=[input_data])
@ -84,6 +98,8 @@ class OpenAIResponsesHandler(BaseTranslation):
self._extract_and_transform_tools(data["tools"], tools_to_check)
if tools_to_check:
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages # type: ignore
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
@ -127,6 +143,8 @@ class OpenAIResponsesHandler(BaseTranslation):
inputs["images"] = images_to_check
if tools_to_check:
inputs["tools"] = tools_to_check
if structured_messages:
inputs["structured_messages"] = structured_messages # type: ignore
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=data,
@ -251,7 +269,7 @@ class OpenAIResponsesHandler(BaseTranslation):
user_api_key_dict: Optional[Any] = None,
) -> Any:
"""
Process output response by applying guardrails to text content.
Process output response by applying guardrails to text content and tool calls.
Args:
response: LiteLLM ResponsesAPIResponse object
@ -264,22 +282,19 @@ class OpenAIResponsesHandler(BaseTranslation):
Response Format Support:
- response.output is a list of output items
- Each output item has a content list with OutputText objects
- Each output item can be:
* GenericResponseOutputItem with a content list of OutputText objects
* OutputFunctionToolCall with tool call data
- Each OutputText object has a text field
"""
# Step 0: Check if response has any text content to process
if not self._has_text_content(response):
verbose_proxy_logger.warning(
"OpenAI Responses API: No text content in response, skipping guardrail"
)
return response
texts_to_check: List[str] = []
images_to_check: List[str] = []
tool_calls_to_check: List[ChatCompletionToolCallChunk] = []
task_mappings: List[Tuple[int, int]] = []
# Track (output_item_index, content_index) for each text
# Step 1: Extract all text content from response output
# Step 1: Extract all text content and tool calls from response output
for output_idx, output_item in enumerate(response.output):
self._extract_output_text_and_images(
output_item=output_item,
@ -287,10 +302,11 @@ class OpenAIResponsesHandler(BaseTranslation):
texts_to_check=texts_to_check,
images_to_check=images_to_check,
task_mappings=task_mappings,
tool_calls_to_check=tool_calls_to_check,
)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
if texts_to_check or tool_calls_to_check:
# Create a request_data dict with response info and user API key metadata
request_data: dict = {"response": response}
@ -304,6 +320,9 @@ class OpenAIResponsesHandler(BaseTranslation):
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
@ -398,12 +417,57 @@ class OpenAIResponsesHandler(BaseTranslation):
texts_to_check: List[str],
images_to_check: List[str],
task_mappings: List[Tuple[int, int]],
tool_calls_to_check: Optional[List[ChatCompletionToolCallChunk]] = None,
) -> None:
"""
Extract text content and images from a response output item.
Extract text content, images, and tool calls from a response output item.
Override this method to customize text/image extraction logic.
Override this method to customize text/image/tool extraction logic.
"""
# Check if this is a tool call (OutputFunctionToolCall)
if isinstance(output_item, OutputFunctionToolCall):
if tool_calls_to_check is not None:
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=output_item,
index=output_idx,
)
tool_calls_to_check.append(
cast(ChatCompletionToolCallChunk, tool_call_dict)
)
return
elif (
isinstance(output_item, BaseModel)
and hasattr(output_item, "type")
and getattr(output_item, "type") == "function_call"
):
if tool_calls_to_check is not None:
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=output_item,
index=output_idx,
)
tool_calls_to_check.append(
cast(ChatCompletionToolCallChunk, tool_call_dict)
)
return
elif (
isinstance(output_item, dict) and output_item.get("type") == "function_call"
):
# Handle dict representation of tool call
if tool_calls_to_check is not None:
# Convert dict to OutputFunctionToolCall for processing
try:
tool_call_obj = OutputFunctionToolCall(**output_item)
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=tool_call_obj,
index=output_idx,
)
tool_calls_to_check.append(
cast(ChatCompletionToolCallChunk, tool_call_dict)
)
except Exception:
pass
return
# Handle both GenericResponseOutputItem and dict
content: Optional[Union[List[OutputText], List[dict]]] = None
if isinstance(output_item, BaseModel):

129
litellm/llms/openai_like/README.md Normal file
View file

@ -0,0 +1,129 @@
# JSON-Based OpenAI-Compatible Provider Configuration
This directory contains the new JSON-based configuration system for OpenAI-compatible providers.
## Overview
Instead of creating a full Python module for simple OpenAI-compatible providers, you can now define them in a single JSON file.
## Files
- `providers.json` - Configuration file for all JSON-based providers
- `json_loader.py` - Loads and parses the JSON configuration
- `dynamic_config.py` - Generates Python config classes from JSON
- `chat/` - Existing OpenAI-like chat completion handlers
## Adding a New Provider
### For Simple OpenAI-Compatible Providers
Edit `providers.json` and add your provider:
```json
{
"your_provider": {
"base_url": "https://api.yourprovider.com/v1",
"api_key_env": "YOUR_PROVIDER_API_KEY"
}
}
```
That's it! The provider will be automatically loaded and available.
### Optional Configuration Fields
```json
{
"your_provider": {
"base_url": "https://api.yourprovider.com/v1",
"api_key_env": "YOUR_PROVIDER_API_KEY",
// Optional: Override base_url via environment variable
"api_base_env": "YOUR_PROVIDER_API_BASE",
// Optional: Which base class to use (default: "openai_gpt")
"base_class": "openai_gpt", // or "openai_like"
// Optional: Parameter name mappings
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
// Optional: Parameter constraints
"constraints": {
"temperature_max": 1.0,
"temperature_min": 0.0,
"temperature_min_with_n_gt_1": 0.3
},
// Optional: Special handling flags
"special_handling": {
"convert_content_list_to_string": true
}
}
}
```
## Example: PublicAI
The first JSON-configured provider:
```json
{
"publicai": {
"base_url": "https://api.publicai.co/v1",
"api_key_env": "PUBLICAI_API_KEY",
"api_base_env": "PUBLICAI_API_BASE",
"base_class": "openai_gpt",
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"special_handling": {
"convert_content_list_to_string": true
}
}
}
```
## Usage
```python
import litellm
response = litellm.completion(
model="publicai/swiss-ai/apertus-8b-instruct",
messages=[{"role": "user", "content": "Hello"}],
)
```
## Benefits
- **Simple**: 2-5 lines of JSON vs 100+ lines of Python
- **Fast**: Add a provider in 5 minutes
- **Safe**: No Python code to mess up
- **Consistent**: All providers follow the same pattern
- **Maintainable**: Centralized configuration
## When to Use Python Instead
Use a Python config class if you need:
- Custom authentication (OAuth, rotating tokens, etc.)
- Complex request/response transformations
- Provider-specific streaming logic
- Advanced tool calling transformations
## Implementation Details
### How It Works
1. `json_loader.py` loads `providers.json` on import
2. `dynamic_config.py` generates config classes on-demand
3. Provider resolution checks JSON registry first
4. ProviderConfigManager returns JSON-based configs
### Integration Points
The JSON system is integrated at:
- `litellm/litellm_core_utils/get_llm_provider_logic.py` - Provider resolution
- `litellm/utils.py` - ProviderConfigManager
- `litellm/constants.py` - openai_compatible_providers list

148
litellm/llms/openai_like/dynamic_config.py Normal file
View file

@ -0,0 +1,148 @@
"""
Dynamic configuration class generator for JSON-based providers.
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from .json_loader import SimpleProviderConfig
def create_config_class(provider: SimpleProviderConfig):
"""Generate config class dynamically from JSON configuration"""
# Choose base class
base_class: type = (
OpenAIGPTConfig if provider.base_class == "openai_gpt" else OpenAILikeChatConfig
)
class JSONProviderConfig(base_class): # type: ignore[valid-type,misc]
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
def _transform_messages(
self,
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
"""Transform messages based on special_handling config"""
# Handle content list to string conversion if configured
if provider.special_handling.get("convert_content_list_to_string"):
messages = handle_messages_with_content_list_to_str_conversion(messages)
if is_async:
return super()._transform_messages(
messages=messages, model=model, is_async=True
)
else:
return super()._transform_messages(
messages=messages, model=model, is_async=False
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
"""Get API base and key from JSON config"""
# Resolve base URL
resolved_base = api_base
if not resolved_base and provider.api_base_env:
resolved_base = get_secret_str(provider.api_base_env)
if not resolved_base:
resolved_base = provider.base_url
# Resolve API key
resolved_key = api_key or get_secret_str(provider.api_key_env)
return resolved_base, resolved_key
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""Build complete URL for the API endpoint"""
if not api_base:
api_base = provider.base_url
if api_base is None:
raise ValueError(f"api_base is required for provider {provider.slug}")
if not api_base.endswith("/chat/completions"):
api_base = f"{api_base}/chat/completions"
return api_base
def get_supported_openai_params(self, model: str) -> list:
"""Get supported OpenAI params from base class"""
return super().get_supported_openai_params(model=model)
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""Apply parameter mappings and constraints"""
supported_params = self.get_supported_openai_params(model)
# Apply supported params
for param, value in non_default_params.items():
# Check parameter mappings first
if param in provider.param_mappings:
optional_params[provider.param_mappings[param]] = value
elif param in supported_params:
optional_params[param] = value
# Apply temperature constraints if present
if "temperature" in optional_params:
temp = optional_params["temperature"]
constraints = provider.constraints
# Clamp to max
if "temperature_max" in constraints:
temp = min(temp, constraints["temperature_max"])
# Clamp to min
if "temperature_min" in constraints:
temp = max(temp, constraints["temperature_min"])
# Special case: temperature_min_with_n_gt_1
if "temperature_min_with_n_gt_1" in constraints:
n = optional_params.get("n", 1)
if n > 1 and temp < constraints["temperature_min_with_n_gt_1"]:
temp = constraints["temperature_min_with_n_gt_1"]
optional_params["temperature"] = temp
return optional_params
@property
def custom_llm_provider(self) -> Optional[str]:
return provider.slug
return JSONProviderConfig

74
litellm/llms/openai_like/json_loader.py Normal file
View file

@ -0,0 +1,74 @@
"""
JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from pathlib import Path
from typing import Dict, Optional
from litellm._logging import verbose_logger
class SimpleProviderConfig:
"""Simple data class for JSON provider config"""
def __init__(self, slug: str, data: dict):
self.slug = slug
self.base_url = data["base_url"]
self.api_key_env = data["api_key_env"]
self.api_base_env = data.get("api_base_env")
self.base_class = data.get("base_class", "openai_gpt")
self.param_mappings = data.get("param_mappings", {})
self.constraints = data.get("constraints", {})
self.special_handling = data.get("special_handling", {})
class JSONProviderRegistry:
"""Load providers from JSON once on import"""
_providers: Dict[str, SimpleProviderConfig] = {}
_loaded = False
@classmethod
def load(cls):
"""Load providers from JSON configuration file"""
if cls._loaded:
return
json_path = Path(__file__).parent / "providers.json"
if not json_path.exists():
# No JSON file yet, that's okay
cls._loaded = True
return
try:
with open(json_path) as f:
data = json.load(f)
for slug, config in data.items():
cls._providers[slug] = SimpleProviderConfig(slug, config)
cls._loaded = True
except Exception as e:
verbose_logger.warning(f"Warning: Failed to load JSON provider configs: {e}")
cls._loaded = True
@classmethod
def get(cls, slug: str) -> Optional[SimpleProviderConfig]:
"""Get a provider configuration by slug"""
return cls._providers.get(slug)
@classmethod
def exists(cls, slug: str) -> bool:
"""Check if a provider is defined via JSON"""
return slug in cls._providers
@classmethod
def list_providers(cls) -> list:
"""List all registered provider slugs"""
return list(cls._providers.keys())
# Load on import
JSONProviderRegistry.load()

14
litellm/llms/openai_like/providers.json Normal file
View file

@ -0,0 +1,14 @@
{
"publicai": {
"base_url": "https://api.publicai.co/v1",
"api_key_env": "PUBLICAI_API_KEY",
"api_base_env": "PUBLICAI_API_BASE",
"base_class": "openai_gpt",
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"special_handling": {
"convert_content_list_to_string": true
}
}
}

114
litellm/llms/publicai/chat/transformation.py
View file

@ -1,114 +0,0 @@
"""
Translates from OpenAI's `/v1/chat/completions` to PublicAI's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class PublicAIChatConfig(OpenAIGPTConfig):
@overload
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
) -> Coroutine[Any, Any, List[AllMessageValues]]:
...
@overload
def _transform_messages(
self,
messages: List[AllMessageValues],
model: str,
is_async: Literal[False] = False,
) -> List[AllMessageValues]:
...
def _transform_messages(
self, messages: List[AllMessageValues], model: str, is_async: bool = False
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
"""
PublicAI does not support content in list format.
"""
messages = handle_messages_with_content_list_to_str_conversion(messages)
if is_async:
return super()._transform_messages(
messages=messages, model=model, is_async=True
)
else:
return super()._transform_messages(
messages=messages, model=model, is_async=False
)
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = (
api_base
or get_secret_str("PUBLICAI_API_BASE")
or "https://platform.publicai.co/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("PUBLICAI_API_KEY")
return api_base, dynamic_api_key
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
If api_base is not provided, use the default PublicAI /chat/completions endpoint.
"""
if not api_base:
api_base = "https://platform.publicai.co/v1"
if not api_base.endswith("/chat/completions"):
api_base = f"{api_base}/chat/completions"
return api_base
def get_supported_openai_params(self, model: str) -> list:
"""
Get the supported OpenAI params for PublicAI models
PublicAI limitations:
- functions parameter is not supported (use tools instead)
"""
excluded_params: List[str] = ["functions"]
base_openai_params = super().get_supported_openai_params(model=model)
final_params: List[str] = []
for param in base_openai_params:
if param not in excluded_params:
final_params.append(param)
return final_params
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
Map OpenAI parameters to PublicAI parameters
"""
supported_openai_params = self.get_supported_openai_params(model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params

8
litellm/llms/vertex_ai/batches/handler.py
View file

@ -61,6 +61,10 @@ class VertexAIBatchPrediction(VertexLLM):
stream=None,
auth_header=None,
url=default_api_base,
model=None,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1",
)
headers = {
@ -166,6 +170,10 @@ class VertexAIBatchPrediction(VertexLLM):
stream=None,
auth_header=None,
url=default_api_base,
model=None,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1",
)
headers = {

110
litellm/llms/vertex_ai/common_utils.py
View file

@ -32,9 +32,12 @@ class VertexAIModelRoute(str, Enum):
PARTNER_MODELS = "partner_models"
GEMINI = "gemini"
GEMMA = "gemma"
BGE = "bge"
MODEL_GARDEN = "model_garden"
NON_GEMINI = "non_gemini"
OPENAI_COMPATIBLE = "openai"
VERTEX_AI_MODEL_ROUTES = [f"{route.value}/" for route in VertexAIModelRoute]
def get_vertex_ai_model_route(
model: str, litellm_params: Optional[dict] = None
@ -61,6 +64,9 @@ def get_vertex_ai_model_route(
>>> get_vertex_ai_model_route("openai/gpt-oss-120b")
VertexAIModelRoute.MODEL_GARDEN
>>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"})
VertexAIModelRoute.GEMINI # Numeric endpoints with api_base use HTTP path
"""
from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
VertexAIPartnerModels,
@ -70,11 +76,20 @@ def get_vertex_ai_model_route(
if litellm_params and litellm_params.get("base_model") is not None:
if "gemini" in litellm_params["base_model"]:
return VertexAIModelRoute.GEMINI
# Check if numeric endpoint ID with custom api_base (PSC endpoint)
# Route to GEMINI (HTTP path) to support PSC endpoints properly
if model.isdigit() and litellm_params and litellm_params.get("api_base"):
return VertexAIModelRoute.GEMINI
# Check for partner models (llama, mistral, claude, etc.)
if VertexAIPartnerModels.is_vertex_partner_model(model=model):
return VertexAIModelRoute.PARTNER_MODELS
# Check for BGE models
if "bge/" in model or "bge" in model.lower():
return VertexAIModelRoute.BGE
# Check for gemma models
if "gemma/" in model:
return VertexAIModelRoute.GEMMA
@ -137,6 +152,69 @@ all_gemini_url_modes = Literal[
]
def get_vertex_base_model_name(model: str) -> str:
"""
Strip routing prefixes from model name for PSC/endpoint URL construction.
Patterns like "bge/", "gemma/", "openai/" are used for internal routing but
should not appear in the actual endpoint URL. Routing prefixes are derived
from VertexAIModelRoute enum values.
Args:
model: The model name with potential prefix (e.g., "bge/123456", "gemma/gemma-3-12b-it")
Returns:
str: The model name without routing prefix (e.g., "123456", "gemma-3-12b-it")
Examples:
>>> get_vertex_base_model_name("bge/378943383978115072")
"378943383978115072"
>>> get_vertex_base_model_name("gemma/gemma-3-12b-it")
"gemma-3-12b-it"
>>> get_vertex_base_model_name("openai/gpt-oss-120b")
"gpt-oss-120b"
>>> get_vertex_base_model_name("1234567890")
"1234567890"
"""
# Derive routing prefixes from VertexAIModelRoute enum
# Map specific routes to their prefixes (some routes like PARTNER_MODELS, GEMINI don't have prefixes)
for route in VERTEX_AI_MODEL_ROUTES:
if model.startswith(route):
return model.replace(route, "", 1)
return model
def _get_embedding_url(
model: str,
vertex_project: Optional[str],
vertex_location: Optional[str],
vertex_api_version: Literal["v1", "v1beta1"],
) -> Tuple[str, str]:
"""
Get URL for embedding models.
Handles special patterns:
- bge/endpoint_id -> strips to endpoint_id for endpoints/ routing
- numeric model -> routes to endpoints/
- regular model -> routes to publishers/google/models/
"""
endpoint = "predict"
# Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction
model = get_vertex_base_model_name(model=model)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if model.isdigit():
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
return url, endpoint
def _get_vertex_url(
mode: all_gemini_url_modes,
model: str,
@ -149,6 +227,7 @@ def _get_vertex_url(
endpoint: Optional[str] = None
model = litellm.VertexGeminiConfig.get_model_for_vertex_ai_url(model=model)
if mode == "chat":
### SET RUNTIME ENDPOINT ###
endpoint = "generateContent"
@ -173,11 +252,12 @@ def _get_vertex_url(
if stream is True:
url += "?alt=sse"
elif mode == "embedding":
endpoint = "predict"
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if model.isdigit():
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
return _get_embedding_url(
model=model,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version=vertex_api_version,
)
elif mode == "image_generation":
endpoint = "predict"
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
@ -200,18 +280,24 @@ def _get_gemini_url(
stream: Optional[bool],
gemini_api_key: Optional[str],
) -> Tuple[str, str]:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
_gemini_model_name = "models/{}".format(model)
api_version = "v1alpha" if VertexGeminiConfig._is_gemini_3_or_newer(model) else "v1beta"
if mode == "chat":
endpoint = "generateContent"
if stream is True:
endpoint = "streamGenerateContent"
url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}&alt=sse".format(
_gemini_model_name, endpoint, gemini_api_key
url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}&alt=sse".format(
api_version, _gemini_model_name, endpoint, gemini_api_key
)
else:
url = (
"https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format(
_gemini_model_name, endpoint, gemini_api_key
"https://generativelanguage.googleapis.com/{}/{}:{}?key={}".format(
api_version, _gemini_model_name, endpoint, gemini_api_key
)
)
elif mode == "embedding":
@ -863,4 +949,4 @@ class VertexAITokenCounter(BaseTokenCounter):
original_response=result,
)
return None
return None

4
litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py
View file

@ -85,6 +85,10 @@ class ContextCachingEndpoints(VertexBase):
stream=None,
auth_header=auth_header,
url=url,
model=None,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version="v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1",
)
def check_cache(

69
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -5,7 +5,7 @@ Why separate file? Make it easy to see how transformation works
"""
import os
from typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union, cast
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union, cast
import httpx
from pydantic import BaseModel
@ -28,7 +28,6 @@ from litellm.types.files import (
get_file_type_from_extension,
is_gemini_1_5_accepted_file_type,
)
from litellm.types.utils import LlmProviders
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -48,7 +47,7 @@ from litellm.types.llms.vertex_ai import (
ToolConfig,
Tools,
)
from litellm.types.utils import GenericImageParsingChunk
from litellm.types.utils import GenericImageParsingChunk, LlmProviders
from ..common_utils import (
_check_text_in_content,
@ -64,24 +63,21 @@ else:
LiteLLMLoggingObj = Any
def _map_openai_detail_to_media_resolution(
def _convert_detail_to_media_resolution_enum(
detail: Optional[str],
) -> Optional[Literal["low", "medium", "high"]]:
"""
Map OpenAI's "detail" parameter to Gemini's "media_resolution" parameter.
"""
) -> Optional[Dict[str, str]]:
if detail == "low":
return "low"
return {"level": "MEDIA_RESOLUTION_LOW"}
elif detail == "high":
return "high"
# "auto" or None means let the model decide, so we don't set media_resolution
return {"level": "MEDIA_RESOLUTION_HIGH"}
return None
def _process_gemini_image(
image_url: str,
format: Optional[str] = None,
media_resolution: Optional[Literal["low", "medium", "high"]] = None,
media_resolution_enum: Optional[Dict[str, str]] = None,
model: Optional[str] = None,
) -> PartType:
"""
Given an image URL, return the appropriate PartType for Gemini
@ -105,31 +101,43 @@ def _process_gemini_image(
else:
mime_type = format
file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
return PartType(file_data=file_data)
part: PartType = {"file_data": file_data}
if media_resolution_enum is not None and model is not None:
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
if VertexGeminiConfig._is_gemini_3_or_newer(model):
part_dict = dict(part)
part_dict["media_resolution"] = media_resolution_enum
return cast(PartType, part_dict)
return part
elif (
"https://" in image_url
and (image_type := format or _get_image_mime_type_from_url(image_url))
is not None
):
file_data = FileDataType(file_uri=image_url, mime_type=image_type)
return PartType(file_data=file_data)
part = {"file_data": file_data}
if media_resolution_enum is not None and model is not None:
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
if VertexGeminiConfig._is_gemini_3_or_newer(model):
part_dict = dict(part)
part_dict["media_resolution"] = media_resolution_enum
return cast(PartType, part_dict)
return part
elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
# https links for unsupported mime types and base64 images
image = convert_to_anthropic_image_obj(image_url, format=format)
_blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]}
if media_resolution is not None:
_blob["media_resolution"] = media_resolution
# Convert snake_case keys to camelCase for JSON serialization
# The TypedDict uses snake_case, but the API expects camelCase
_blob_dict = dict(_blob)
if "media_resolution" in _blob_dict:
_blob_dict["mediaResolution"] = _blob_dict.pop("media_resolution")
if "mime_type" in _blob_dict:
_blob_dict["mimeType"] = _blob_dict.pop("mime_type")
part = {"inline_data": cast(BlobType, _blob)}
return PartType(inline_data=cast(BlobType, _blob_dict))
if media_resolution_enum is not None and model is not None:
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
if VertexGeminiConfig._is_gemini_3_or_newer(model):
part_dict = dict(part)
part_dict["media_resolution"] = media_resolution_enum
return cast(PartType, part_dict)
return part
raise Exception("Invalid image received - {}".format(image_url))
except Exception as e:
raise e
@ -235,18 +243,19 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
element = cast(ChatCompletionImageObject, element)
img_element = element
format: Optional[str] = None
media_resolution: Optional[Literal["low", "medium", "high"]] = None
media_resolution_enum: Optional[Dict[str, str]] = None
if isinstance(img_element["image_url"], dict):
image_url = img_element["image_url"]["url"]
format = img_element["image_url"].get("format")
detail = img_element["image_url"].get("detail")
media_resolution = _map_openai_detail_to_media_resolution(detail)
media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
else:
image_url = img_element["image_url"]
_part = _process_gemini_image(
image_url=image_url,
format=format,
media_resolution=media_resolution,
media_resolution_enum=media_resolution_enum,
model=model,
)
_parts.append(_part)
elif element["type"] == "input_audio":
@ -271,6 +280,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
_part = _process_gemini_image(
image_url=openai_image_str,
format=audio_format_modified,
model=model,
)
_parts.append(_part)
elif element["type"] == "file":
@ -287,6 +297,7 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
_part = _process_gemini_image(
image_url=passed_file,
format=format,
model=model,
)
_parts.append(_part)
except Exception:

59
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1085,24 +1085,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
def _extract_thinking_blocks_from_parts(
self, parts: List[HttpxPartType]
) -> List[ChatCompletionThinkingBlock]:
"""Extract thinking blocks from parts if present"""
"""Extract thinking blocks from parts if present.
Per Google's docs (https://ai.google.dev/gemini-api/docs/thinking):
- Parts with `thought: true` contain thinking/reasoning content
- `thoughtSignature` is a separate token for multi-turn context preservation,
it does NOT indicate that the content is thinking (a part can have
thoughtSignature without thought: true, e.g., function calls)
"""
thinking_blocks: List[ChatCompletionThinkingBlock] = []
for part in parts:
if "thoughtSignature" in part:
part_copy = part.copy()
part_copy.pop("thoughtSignature")
text_content = part_copy.get("text")
if isinstance(text_content, str) and text_content.strip() == "":
continue
thinking_blocks.append(
ChatCompletionThinkingBlock(
type="thinking",
thinking=json.dumps(part_copy),
signature=part["thoughtSignature"],
)
)
if part.get("thought") is True:
thinking_text = part.get("text", "")
block: ChatCompletionThinkingBlock = {
"type": "thinking",
"thinking": thinking_text,
}
signature = part.get("thoughtSignature")
if signature is not None:
block["signature"] = signature
thinking_blocks.append(block)
return thinking_blocks
def _extract_image_response_from_parts(
@ -2123,6 +2125,9 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
)
# Extract use_psc_endpoint_format from optional_params
use_psc_endpoint_format = optional_params.get("use_psc_endpoint_format", False)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -2134,6 +2139,7 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
api_base=api_base,
should_use_v1beta1_features=should_use_v1beta1_features,
use_psc_endpoint_format=use_psc_endpoint_format,
)
headers = VertexGeminiConfig().validate_environment(
@ -2217,6 +2223,9 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
)
# Extract use_psc_endpoint_format from optional_params
use_psc_endpoint_format = optional_params.get("use_psc_endpoint_format", False)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -2228,6 +2237,7 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
api_base=api_base,
should_use_v1beta1_features=should_use_v1beta1_features,
use_psc_endpoint_format=use_psc_endpoint_format,
)
headers = VertexGeminiConfig().validate_environment(
@ -2401,6 +2411,9 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
)
# Extract use_psc_endpoint_format from optional_params
use_psc_endpoint_format = optional_params.get("use_psc_endpoint_format", False)
auth_header, url = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -2412,6 +2425,7 @@ class VertexLLM(VertexBase):
custom_llm_provider=custom_llm_provider,
api_base=api_base,
should_use_v1beta1_features=should_use_v1beta1_features,
use_psc_endpoint_format=use_psc_endpoint_format,
)
headers = VertexGeminiConfig().validate_environment(
api_key=auth_header,
@ -2589,13 +2603,12 @@ class ModelResponseIterator:
try:
json_chunk = json.loads(chunk)
except json.JSONDecodeError as e:
if (
self.sent_first_chunk is False
): # only check for accumulated json, on first chunk, else raise error. Prevent real errors from being masked.
self.chunk_type = "accumulated_json"
return self.handle_accumulated_json_chunk(chunk=chunk)
raise e
except json.JSONDecodeError:
# Switch to accumulation mode for partial JSON chunks
# This can happen at any point due to network fragmentation, not just first chunk
# See: https://github.com/BerriAI/litellm/issues/16562
self.chunk_type = "accumulated_json"
return self.handle_accumulated_json_chunk(chunk=chunk)
if self.sent_first_chunk is False:
self.sent_first_chunk = True

6
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -140,7 +140,11 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
# Handle global location differently (no region prefix in URL)
if vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent"

2
litellm/llms/vertex_ai/text_to_speech/transformation.py
View file

@ -220,7 +220,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
Returns:
Tuple of (mapped_voice_str, mapped_params)
"""
mapped_params = {}
mapped_params: Dict[str, Any] = {}
##########################################################
# Map voice using helper

10
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -5,6 +5,7 @@ from litellm.llms.anthropic.experimental_pass_through.messages.transformation im
)
from litellm.types.llms.vertex_ai import VertexPartnerProvider
from litellm.types.router import GenericLiteLLMParams
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS
from ....vertex_llm_base import VertexBase
@ -49,6 +50,15 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
)
headers["content-type"] = "application/json"
# Add web search beta header for Vertex AI only if not already set
if "anthropic-beta" not in headers:
tools = optional_params.get("tools", [])
for tool in tools:
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
break
return headers, api_base
def get_complete_url(

182
litellm/llms/vertex_ai/vertex_embeddings/bge.py Normal file
View file

@ -0,0 +1,182 @@
"""
Vertex AI BGE (BAAI General Embedding) Configuration
BGE models deployed on Vertex AI require different input/output format:
- Request: Use "prompt" instead of "content" as the input field
- Response: Embeddings are returned directly as arrays, not wrapped in objects
Model name handling:
- Model names like "bge/endpoint_id" are automatically transformed in common_utils._get_vertex_url()
- This module focuses on request/response transformation only
"""
from typing import List, Optional, Union
from litellm.types.utils import EmbeddingResponse, Usage
from .types import (
EmbeddingParameters,
TaskType,
TextEmbeddingBGEInput,
VertexEmbeddingRequest,
)
class VertexBGEConfig:
"""
Configuration and transformation logic for BGE models on Vertex AI.
BGE (BAAI General Embedding) models use a different request format
where the input field is named "prompt" instead of "content".
Supported model patterns (after provider split in main.py):
- "bge-small-en-v1.5" (model name)
- "bge/204379420394258432" (endpoint ID pattern)
Note: Model name transformation (bge/ -> numeric ID) is handled automatically
in common_utils._get_vertex_url(). This class focuses on request/response format only.
"""
@staticmethod
def is_bge_model(model: str) -> bool:
"""
Check if the model is a BGE (BAAI General Embedding) model.
After provider split in main.py, supports:
- "bge-small-en-v1.5" (model name)
- "bge/204379420394258432" (endpoint ID pattern)
Args:
model: The model name after provider split
Returns:
bool: True if the model is a BGE model
"""
model_lower = model.lower()
# Check for "bge/" prefix (endpoint pattern) or "bge" in model name
return model_lower.startswith("bge/") or "bge" in model_lower
@staticmethod
def transform_request(
input: Union[list, str], optional_params: dict, model: str
) -> VertexEmbeddingRequest:
"""
Transforms an OpenAI request to a Vertex BGE embedding request.
BGE models use "prompt" instead of "content" as the input field.
Args:
input: The input text(s) to embed
optional_params: Optional parameters for the request
model: The model name
Returns:
VertexEmbeddingRequest: The transformed request
"""
vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest()
vertex_text_embedding_input_list: List[TextEmbeddingBGEInput] = []
task_type: Optional[TaskType] = optional_params.get("task_type")
title = optional_params.get("title")
if isinstance(input, str):
input = [input]
for text in input:
embedding_input = VertexBGEConfig._create_embedding_input(
prompt=text, task_type=task_type, title=title
)
vertex_text_embedding_input_list.append(embedding_input)
vertex_request["instances"] = vertex_text_embedding_input_list
vertex_request["parameters"] = EmbeddingParameters(**optional_params)
return vertex_request
@staticmethod
def _create_embedding_input(
prompt: str,
task_type: Optional[TaskType] = None,
title: Optional[str] = None,
) -> TextEmbeddingBGEInput:
"""
Creates a TextEmbeddingBGEInput object for BGE models.
BGE models use "prompt" instead of "content" as the input field.
Args:
prompt: The prompt to be embedded
task_type: The type of task to be performed
title: The title of the document to be embedded
Returns:
TextEmbeddingBGEInput: A TextEmbeddingBGEInput object
"""
text_embedding_input = TextEmbeddingBGEInput(prompt=prompt)
if task_type is not None:
text_embedding_input["task_type"] = task_type
if title is not None:
text_embedding_input["title"] = title
return text_embedding_input
@staticmethod
def transform_response(
response: dict, model: str, model_response: EmbeddingResponse
) -> EmbeddingResponse:
"""
Transforms a Vertex BGE embedding response to OpenAI format.
BGE models return embeddings directly as arrays in predictions:
{
"predictions": [
[0.002, 0.021, ...],
[0.003, 0.022, ...]
]
}
Args:
response: The raw response from Vertex AI
model: The model name
model_response: The EmbeddingResponse object to populate
Returns:
EmbeddingResponse: The transformed response in OpenAI format
Raises:
KeyError: If response doesn't contain 'predictions'
ValueError: If predictions is not a list or contains invalid data
"""
if "predictions" not in response:
raise KeyError("Response missing 'predictions' field")
_predictions = response["predictions"]
if not isinstance(_predictions, list):
raise ValueError(f"Expected 'predictions' to be a list, got {type(_predictions)}")
embedding_response = []
# BGE models don't return token counts, so we estimate or set to 0
input_tokens = 0
for idx, embedding_values in enumerate(_predictions):
if not isinstance(embedding_values, list):
raise ValueError(
f"Expected embedding at index {idx} to be a list, got {type(embedding_values)}"
)
embedding_response.append(
{
"object": "embedding",
"index": idx,
"embedding": embedding_values,
}
)
model_response.object = "list"
model_response.data = embedding_response
model_response.model = model
usage = Usage(
prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens
)
setattr(model_response, "usage", usage)
return model_response

8
litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py
View file

@ -72,6 +72,9 @@ class VertexEmbedding(VertexBase):
project_id=vertex_project,
custom_llm_provider=custom_llm_provider,
)
# Extract use_psc_endpoint_format from optional_params
use_psc_endpoint_format = optional_params.get("use_psc_endpoint_format", False)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -84,6 +87,7 @@ class VertexEmbedding(VertexBase):
api_base=api_base,
should_use_v1beta1_features=should_use_v1beta1_features,
mode="embedding",
use_psc_endpoint_format=use_psc_endpoint_format,
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = (
@ -164,6 +168,9 @@ class VertexEmbedding(VertexBase):
project_id=vertex_project,
custom_llm_provider=custom_llm_provider,
)
# Extract use_psc_endpoint_format from optional_params
use_psc_endpoint_format = optional_params.get("use_psc_endpoint_format", False)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -176,6 +183,7 @@ class VertexEmbedding(VertexBase):
api_base=api_base,
should_use_v1beta1_features=should_use_v1beta1_features,
mode="embedding",
use_psc_endpoint_format=use_psc_endpoint_format,
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = (

21
litellm/llms/vertex_ai/vertex_embeddings/transformation.py
View file

@ -105,10 +105,16 @@ class VertexAITextEmbeddingConfig(BaseModel):
"""
Transforms an openai request to a vertex embedding request.
"""
# Import here to avoid circular import issues with litellm.__init__
from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig
if model.isdigit():
return self._transform_openai_request_to_fine_tuned_embedding_request(
input, optional_params, model
)
if VertexBGEConfig.is_bge_model(model):
return VertexBGEConfig.transform_request(
input=input, optional_params=optional_params, model=model
)
vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest()
vertex_text_embedding_input_list: List[TextEmbeddingInput] = []
@ -167,6 +173,9 @@ class VertexAITextEmbeddingConfig(BaseModel):
vertex_request["parameters"] = TextEmbeddingFineTunedParameters(
**optional_params
)
# Remove 'shared_session' from parameters if present
if vertex_request["parameters"] is not None and "shared_session" in vertex_request["parameters"]:
del vertex_request["parameters"]["shared_session"] # type: ignore[typeddict-item]
return vertex_request
@ -183,8 +192,8 @@ class VertexAITextEmbeddingConfig(BaseModel):
Args:
content (str): The content to be embedded.
task_type (Optional[TaskType]): The type of task to be performed".
title (Optional[str]): The title of the document to be embedded
task_type (Optional[TaskType]): The type of task to be performed.
title (Optional[str]): The title of the document to be embedded.
Returns:
TextEmbeddingInput: A TextEmbeddingInput object.
@ -206,6 +215,14 @@ class VertexAITextEmbeddingConfig(BaseModel):
return self._transform_vertex_response_to_openai_for_fine_tuned_models(
response, model, model_response
)
# Import here to avoid circular import issues with litellm.__init__
from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig
if VertexBGEConfig.is_bge_model(model):
return VertexBGEConfig.transform_response(
response=response, model=model, model_response=model_response
)
_predictions = response["predictions"]

8
litellm/llms/vertex_ai/vertex_embeddings/types.py
View file

@ -25,6 +25,12 @@ class TextEmbeddingInput(TypedDict, total=False):
title: Optional[str]
class TextEmbeddingBGEInput(TypedDict, total=False):
prompt: str
task_type: Optional[TaskType]
title: Optional[str]
# Fine-tuned models require a different input format
# Ref: https://console.cloud.google.com/vertex-ai/model-garden?hl=en&project=adroit-crow-413218&pageState=(%22galleryStateKey%22:(%22f%22:(%22g%22:%5B%5D,%22o%22:%5B%5D),%22s%22:%22%22))
class TextEmbeddingFineTunedInput(TypedDict, total=False):
@ -44,7 +50,7 @@ class EmbeddingParameters(TypedDict, total=False):
class VertexEmbeddingRequest(TypedDict, total=False):
instances: Union[List[TextEmbeddingInput], List[TextEmbeddingFineTunedInput]]
instances: Union[List[TextEmbeddingInput], List[TextEmbeddingBGEInput], List[TextEmbeddingFineTunedInput]]
parameters: Optional[Union[EmbeddingParameters, TextEmbeddingFineTunedParameters]]

5
litellm/llms/vertex_ai/vertex_gemma_models/main.py
View file

@ -25,7 +25,7 @@ import httpx # type: ignore
from litellm.utils import ModelResponse
from ..common_utils import VertexAIError
from ..common_utils import VertexAIError, get_vertex_base_model_name
from ..vertex_llm_base import VertexBase
@ -82,7 +82,8 @@ class VertexAIGemmaModels(VertexBase):
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
try:
model = model.replace("gemma/", "")
model = get_vertex_base_model_name(model=model)
vertex_httpx_logic = VertexLLM()
access_token, project_id = vertex_httpx_logic._ensure_access_token(

98
litellm/llms/vertex_ai/vertex_llm_base.py
View file

@ -19,6 +19,7 @@ from .common_utils import (
_get_gemini_url,
_get_vertex_url,
all_gemini_url_modes,
get_vertex_base_model_name,
is_global_only_vertex_model,
)
@ -89,9 +90,15 @@ class VertexBase:
else ""
)
if isinstance(environment_id, str) and "aws" in environment_id:
creds = self._credentials_from_identity_pool_with_aws(json_obj)
creds = self._credentials_from_identity_pool_with_aws(
json_obj,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
else:
creds = self._credentials_from_identity_pool(json_obj)
creds = self._credentials_from_identity_pool(
json_obj,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
# Check if the JSON object contains Authorized User configuration (via gcloud auth application-default login)
elif "type" in json_obj and json_obj["type"] == "authorized_user":
creds = self._credentials_from_authorized_user(
@ -130,15 +137,21 @@ class VertexBase:
return creds, project_id
# Google Auth Helpers -- extracted for mocking purposes in tests
def _credentials_from_identity_pool(self, json_obj):
def _credentials_from_identity_pool(self, json_obj, scopes):
from google.auth import identity_pool
return identity_pool.Credentials.from_info(json_obj)
creds = identity_pool.Credentials.from_info(json_obj)
if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
creds = creds.with_scopes(scopes)
return creds
def _credentials_from_identity_pool_with_aws(self, json_obj):
def _credentials_from_identity_pool_with_aws(self, json_obj, scopes):
from google.auth import aws
return aws.Credentials.from_info(json_obj)
creds = aws.Credentials.from_info(json_obj)
if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
creds = creds.with_scopes(scopes)
return creds
def _credentials_from_authorized_user(self, json_obj, scopes):
import google.oauth2.credentials
@ -241,6 +254,9 @@ class VertexBase:
auth_header=None,
url=default_api_base,
model=model,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1", # Partner models typically use v1
)
return api_base
@ -289,10 +305,25 @@ class VertexBase:
auth_header: Optional[str],
url: str,
model: Optional[str] = None,
vertex_project: Optional[str] = None,
vertex_location: Optional[str] = None,
vertex_api_version: Optional[Literal["v1", "v1beta1"]] = None,
use_psc_endpoint_format: bool = False,
) -> Tuple[Optional[str], str]:
"""
for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317
Handles custom api_base for:
1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
2. Vertex AI with standard proxies - constructs {api_base}:{endpoint}
3. Vertex AI with PSC endpoints - constructs full path structure
{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
(only when use_psc_endpoint_format=True)
Args:
use_psc_endpoint_format: If True, constructs PSC endpoint URL format.
If False (default), uses api_base as-is and appends :{endpoint}
## Returns
- (auth_header, url) - Tuple[Optional[str], str]
"""
@ -309,10 +340,31 @@ class VertexBase:
"Missing gemini_api_key, please set `GEMINI_API_KEY`"
)
if gemini_api_key is not None:
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment]
else:
url = "{}:{}".format(api_base, endpoint)
# For Vertex AI
if use_psc_endpoint_format:
# User explicitly specified PSC endpoint format
# Construct full PSC/custom endpoint URL
if not (vertex_project and vertex_location and model):
raise ValueError(
"vertex_project, vertex_location, and model are required when use_psc_endpoint_format=True"
)
# Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction
model_for_url = get_vertex_base_model_name(model=model)
# Format: {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
version = vertex_api_version or "v1"
url = "{}/{}/projects/{}/locations/{}/endpoints/{}:{}".format(
api_base.rstrip("/"),
version,
vertex_project,
vertex_location,
model_for_url,
endpoint,
)
else:
# Fallback to simple format if we don't have all parameters
url = "{}:{}".format(api_base, endpoint)
if stream is True:
url = url + "?alt=sse"
return auth_header, url
@ -330,6 +382,7 @@ class VertexBase:
api_base: Optional[str],
should_use_v1beta1_features: Optional[bool] = False,
mode: all_gemini_url_modes = "chat",
use_psc_endpoint_format: bool = False,
) -> Tuple[Optional[str], str]:
"""
Internal function. Returns the token and url for the call.
@ -339,6 +392,7 @@ class VertexBase:
Returns
token, url
"""
version: Optional[Literal["v1beta1", "v1"]] = None
if custom_llm_provider == "gemini":
url, endpoint = _get_gemini_url(
mode=mode,
@ -354,9 +408,7 @@ class VertexBase:
)
### SET RUNTIME ENDPOINT ###
version: Literal["v1beta1", "v1"] = (
"v1beta1" if should_use_v1beta1_features is True else "v1"
)
version = "v1beta1" if should_use_v1beta1_features is True else "v1"
url, endpoint = _get_vertex_url(
mode=mode,
model=model,
@ -375,6 +427,10 @@ class VertexBase:
stream=stream,
url=url,
model=model,
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_api_version=version,
use_psc_endpoint_format=use_psc_endpoint_format,
)
def _handle_reauthentication(
@ -629,13 +685,13 @@ class VertexBase:
def safe_get_vertex_ai_project(litellm_params: dict) -> Optional[str]:
"""
Safely get Vertex AI project without mutating the litellm_params dict.
Unlike get_vertex_ai_project(), this does NOT pop values from the dict,
making it safe to call multiple times with the same litellm_params.
Args:
litellm_params: Dictionary containing Vertex AI parameters
Returns:
Vertex AI project ID or None
"""
@ -650,13 +706,13 @@ class VertexBase:
def safe_get_vertex_ai_credentials(litellm_params: dict) -> Optional[str]:
"""
Safely get Vertex AI credentials without mutating the litellm_params dict.
Unlike get_vertex_ai_credentials(), this does NOT pop values from the dict,
making it safe to call multiple times with the same litellm_params.
Args:
litellm_params: Dictionary containing Vertex AI parameters
Returns:
Vertex AI credentials or None
"""
@ -670,13 +726,13 @@ class VertexBase:
def safe_get_vertex_ai_location(litellm_params: dict) -> Optional[str]:
"""
Safely get Vertex AI location without mutating the litellm_params dict.
Unlike get_vertex_ai_location(), this does NOT pop values from the dict,
making it safe to call multiple times with the same litellm_params.
Args:
litellm_params: Dictionary containing Vertex AI parameters
Returns:
Vertex AI location/region or None
"""

8
litellm/llms/vertex_ai/vertex_model_garden/main.py
View file

@ -22,7 +22,7 @@ import httpx # type: ignore
from litellm.utils import ModelResponse
from ..common_utils import VertexAIError
from ..common_utils import VertexAIError, get_vertex_base_model_name
from ..vertex_llm_base import VertexBase
@ -89,7 +89,7 @@ class VertexAIModelGardenModels(VertexBase):
message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""",
)
try:
model = model.replace("openai/", "")
model = get_vertex_base_model_name(model=model)
vertex_httpx_logic = VertexLLM()
access_token, project_id = vertex_httpx_logic._ensure_access_token(
@ -123,6 +123,10 @@ class VertexAIModelGardenModels(VertexBase):
stream=stream,
auth_header=None,
url=default_api_base,
model=model,
vertex_project=vertex_project or project_id,
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1beta1",
)
model = ""
return openai_like_chat_completions.completion(

34
litellm/llms/watsonx/audio_transcription/transformation.py
View file

@ -8,7 +8,10 @@ from typing import Any, Dict, List, Optional
import litellm
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.llms.watsonx import WatsonXAudioTranscriptionRequestBody
from litellm.types.utils import FileTypes
@ -32,6 +35,35 @@ class IBMWatsonXAudioTranscriptionConfig(
for authentication and URL construction.
"""
def validate_environment(
self,
headers: Dict,
model: str,
messages: List[AllMessageValues],
optional_params: Dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Dict:
"""
Validate environment for audio transcription.
Removes Content-Type header so httpx can set multipart/form-data automatically.
"""
result = IBMWatsonXMixin.validate_environment(
self,
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=api_key,
api_base=api_base,
)
# Remove Content-Type so httpx sets multipart/form-data automatically
result.pop("Content-Type", None)
return result
def get_supported_openai_params(
self, model: str
) -> List[OpenAIAudioTranscriptionOptionalParams]:

Some files were not shown because too many files have changed in this diff Show more