scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge branch 'main' into teams-on-users-page

Browse source
This commit is contained in:
tanjiro 2025-08-31 07:51:13 +09:00
commit ce8e24fea8
116 changed files with 7785 additions and 3112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -10,4 +10,3 @@ tests
*.tgz
log.txt
docker/Dockerfile.*
*.whl

60
.github/workflows/auto_update_price_and_context_window_file.py vendored
View file

@ -43,8 +43,8 @@ def write_to_file(file_path, data):
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
# Update the existing models and add the missing models
def transform_remote_data(data):
# Update the existing models and add the missing models for OpenRouter
def transform_openrouter_data(data):
transformed = {}
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
@ -81,6 +81,34 @@ def transform_remote_data(data):
return transformed
# Update the existing models and add the missing models for Vercel AI Gateway
def transform_vercel_ai_gateway_data(data):
transformed = {}
for row in data:
obj = {
"max_tokens": row["context_window"],
"input_cost_per_token": float(row["pricing"]["input"]),
"output_cost_per_token": float(row["pricing"]["output"]),
'max_output_tokens': row['max_tokens'],
'max_input_tokens': row["context_window"],
}
# Handle cache pricing if available
if "pricing" in row:
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
return transformed
# Load local data from a specified file
def load_local_data(file_path):
@ -100,22 +128,32 @@ def load_local_data(file_path):
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
# Fetch remote data asynchronously
remote_data = asyncio.run(fetch_data(url))
# Transform the fetched remote data
remote_data = transform_remote_data(remote_data)
# Fetch OpenRouter data
openrouter_data = asyncio.run(fetch_data(openrouter_url))
# Transform the fetched OpenRouter data
openrouter_data = transform_openrouter_data(openrouter_data)
# Fetch Vercel AI Gateway data
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
# Transform the fetched Vercel AI Gateway data
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
# Combine both datasets
all_remote_data = {**openrouter_data, **vercel_data}
# If both local and remote data are available, synchronize and save
if local_data and remote_data:
sync_local_data_with_remote(local_data, remote_data)
# If both local and openrouter data are available, synchronize and save
if local_data and all_remote_data:
sync_local_data_with_remote(local_data, all_remote_data)
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
main()
main()

3
.gitignore vendored
View file

@ -95,5 +95,4 @@ test.py
litellm_config.yaml
.cursor
.vscode/launch.json
*.whl
litellm/proxy/to_delete_loadtest_work/*
litellm/proxy/to_delete_loadtest_work/*

38
deploy/charts/litellm-helm/README.md
View file

@ -36,11 +36,45 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |
| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` |
| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` |
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
#### Example `proxy_config` ConfigMap from values (default):
```
proxyConfigMap:
create: true
key: "config.yaml"
proxy_config:
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: eXaMpLeOnLy
```
#### Example using existing `proxyConfigMap` instead of creating it:
```
proxyConfigMap:
create: false
name: my-litellm-config
key: config.yaml
# proxy_config is ignored in this mode
```
#### Example `environmentSecrets` Secret
```
apiVersion: v1
kind: Secret

4
deploy/charts/litellm-helm/templates/configmap-litellm.yaml
View file

@ -1,7 +1,9 @@
{{- if .Values.proxyConfigMap.create }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
{{ .Values.proxy_config | toYaml | indent 6 }}
{{ .Values.proxy_config | toYaml | indent 6 }}
{{- end }}

8
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -16,7 +16,9 @@ spec:
template:
metadata:
annotations:
{{- if .Values.proxyConfigMap.create }}
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
@ -183,9 +185,13 @@ spec:
{{- end }}
- name: litellm-config
configMap:
{{- if .Values.proxyConfigMap.create }}
name: {{ include "litellm.fullname" . }}-config
{{- else }}
name: {{ .Values.proxyConfigMap.name }}
{{- end }}
items:
- key: "config.yaml"
- key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
path: "config.yaml"
{{- with .Values.volumes }}
{{- toYaml . | nindent 8 }}

22
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -115,3 +115,25 @@ tests:
content:
name: EXTRA_ENV_VAR
value: EXTRA_ENV_VAR_VALUE
- it: should mount existing configmap when create=false
template: deployment.yaml
set:
proxyConfigMap:
create: false
name: my-litellm-config
key: custom.yaml
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: litellm-config
configMap:
name: my-litellm-config
items:
- key: custom.yaml
path: config.yaml
- contains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/

8
deploy/charts/litellm-helm/values.yaml
View file

@ -93,6 +93,14 @@ masterkeySecretName: ""
# if set, use this secret key for the master key; otherwise, use the default key
masterkeySecretKey: ""
proxyConfigMap:
# when true, creates a new configmap
create: true
# if create is false and name is set, use existing ConfigMap
# create: false
# name: ""
# key: "config.yaml"
# The elements within proxy_config are rendered as config.yaml for the proxy
# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
# Reference: https://docs.litellm.ai/docs/proxy/configs

232
docs/my-website/docs/completion/image_generation_chat.md Normal file
View file

@ -0,0 +1,232 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Image Generation in Chat Completions, Responses API
This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
:::info
Requires LiteLLM v1.76.1+
:::
Supported Providers:
- Google AI Studio (`gemini`)
- Vertex AI (`vertex_ai/`)
LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
```python title="Example response from litellm"
"message": {
...
"content": "Here's the image you requested:",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
```
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Image generation with chat completion"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.5-flash-image-preview
api_key: os.environ/GEMINI_API_KEY
```
2. Run proxy server
```bash showLineNumbers title="Start the proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```bash showLineNumbers title="Make request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
]
}'
```
</TabItem>
</Tabs>
**Expected Response**
```bash
{
"id": "chatcmpl-3b66124d79a708e10c603496b363574c",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Here's the image you requested:",
"role": "assistant",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
}
],
"created": 1723323084,
"model": "gemini/gemini-2.5-flash-image-preview",
"object": "chat.completion",
"usage": {
"completion_tokens": 12,
"prompt_tokens": 16,
"total_tokens": 28
}
}
```
## Streaming Support
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Streaming image generation"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
stream=True,
)
for chunk in response:
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
print("Generated image:", chunk.choices[0].delta.image["url"])
break
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash showLineNumbers title="Streaming request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
],
"stream": true
}'
```
</TabItem>
</Tabs>
**Expected Streaming Response**
```bash
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
data: [DONE]
```
## Async Support
```python showLineNumbers title="Async image generation"
from litellm import acompletion
import asyncio
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
async def generate_image():
response = await acompletion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
return response
# Run the async function
asyncio.run(generate_image())
```
## Supported Models
| Provider | Model |
|----------|--------|
| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
## Spec
The `image` field in the response follows this structure:
```python
"image": {
"url": "data:image/png;base64,<base64_encoded_image>",
"detail": "auto"
}
```
- `url` - str: Base64 encoded image data in data URI format
- `detail` - str: Image detail level (always "auto" for generated images)
The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

205
docs/my-website/docs/extras/gemini_img_migration.md Normal file
View file

@ -0,0 +1,205 @@
# Gemini Image Generation Migration Guide
## Who is impacted by this change?
Anyone using the following models with /chat/completions:
- `gemini/gemini-2.0-flash-exp-image-generation`
- `vertex_ai/gemini-2.0-flash-exp-image-generation`
## Key Change
Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.image` with base64 data URLs.
## Before and After
### Before
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
base_64_image_data = response.choices[0].message.content
```
### After
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Image is now available in the response
image_url = response.choices[0].message.image["url"] # "data:image/png;base64,..."
```
### Why the change?
Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
## Usage
### Using the Python SDK
**Key Change:**
```diff
# Before
-- base_64_image_data = response.choices[0].message.content
# After
++ image_url = response.choices[0].message.image["url"]
```
#### Basic Image Generation
```python
from litellm import completion
import os
# Set your API key
os.environ["GEMINI_API_KEY"] = "your-api-key"
# Generate an image
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.image) # Image data
```
#### Response Format
The image is returned in the `message.image` field:
```python
{
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
```
### Using the LiteLLM Proxy Server
**Key Change:**
```diff
# Before
-- "content": "base64-image-data..."
# After
++ "image": {
++ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
++ "detail": "auto"
++ }
```
#### Configuration Setup
1. **Configure your models in `config.yaml`:**
```yaml
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.0-flash-exp-image-generation
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-image-gen
litellm_params:
model: vertex_ai/gemini-2.5-flash-image-preview
vertex_project: your-project-id
vertex_location: us-central1
general_settings:
master_key: sk-1234 # Your proxy API key
```
2. **Start the proxy server:**
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### Making Requests
**Using OpenAI SDK:**
```python
from openai import OpenAI
# Point to your proxy server
client = OpenAI(
api_key="sk-1234", # Your proxy API key
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gemini-image-gen",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
extra_body={"modalities": ["image", "text"]}
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.image) # Image data
```
**Using curl:**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a cat"
}
],
"modalities": ["image", "text"]
}'
```
**Response format from proxy:**
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1704089632,
"model": "gemini-image-gen",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Here's an image of a cat for you!",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 8,
"total_tokens": 18
}
}
```

35
docs/my-website/docs/index.md
View file

@ -226,6 +226,23 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
</Tabs>
### Response Format (OpenAI Format)
@ -446,6 +463,24 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
</Tabs>
### Streaming Response Format (OpenAI Format)

6
docs/my-website/docs/load_test_rpm.md
View file

@ -53,8 +53,8 @@ model_list = [
},
]
router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="simple-shuffle", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="simple-shuffle", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
@ -142,7 +142,7 @@ router_settings:
redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
redis_password: os.environ/REDIS_PASSWORD
redis_port: os.environ/REDIS_PORT
routing_strategy: usage-based-routing-v2
routing_strategy: simple-shuffle # recommended for best performance
```
### 2. Start proxy 2 instances

23
docs/my-website/docs/mcp.md
View file

@ -40,7 +40,28 @@ LiteLLM supports the following MCP transports:
style={{width: '80%', display: 'block', margin: '0'}}
/>
### Adding a stdio MCP Server
<br/>
<br/>
### Add HTTP MCP Server
This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE.
<iframe width="840" height="500" src="https://www.loom.com/embed/e2aebce78e8d46beafeb4bacdde31f14" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
<br/>
<br/>
### Add SSE MCP Server
This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE.
<iframe width="840" height="500" src="https://www.loom.com/embed/07e04e27f5e74475b9cf8ef8247d2c3e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
<br/>
<br/>
### Add STDIO MCP Server
For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format:

23
docs/my-website/docs/observability/langfuse_otel_integration.md
View file

@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs
|----------|----------|-------------|---------|
| `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
| `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
### Endpoint Resolution
The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
- **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
- **EU Region**: `https://cloud.langfuse.com/api/public/otel`
- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
## Usage
@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
# Use EU region
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region (default)
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
# Or use self-hosted instance
# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
litellm.callbacks = ["langfuse_otel"]
```
@ -98,14 +98,16 @@ import litellm
# Get keys for your project from the project settings page: https://cloud.langfuse.com
os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
LANGFUSE_AUTH = base64.b64encode(
f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
).decode()
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
host = os.environ.get("LANGFUSE_OTEL_HOST")
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
litellm.callbacks = ["langfuse_otel"]
@ -120,7 +122,8 @@ Add the integration to your proxy configuration:
```bash
export LANGFUSE_PUBLIC_KEY="pk-lf-..."
export LANGFUSE_SECRET_KEY="sk-lf-..."
export LANGFUSE_HOST="https://us.cloud.langfuse.com" # Default US region
export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region
# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint
```
2. Setup config.yaml

21
docs/my-website/docs/providers/anthropic.md
View file

@ -55,8 +55,29 @@ import os
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
# os.environ["ANTHROPIC_API_BASE"] = "" # [OPTIONAL] or 'ANTHROPIC_BASE_URL'
# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
```
### Custom API Base
When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.
If your custom endpoint already includes the full path or doesn't follow Anthropic's standard URL structure, you can disable this automatic suffix appending:
```python
import os
os.environ["ANTHROPIC_API_BASE"] = "https://my-custom-endpoint.com/custom/path"
os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # Prevents automatic suffix
```
Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
- Base URL `https://my-proxy.com` → `https://my-proxy.com/v1/messages`
- Base URL `https://my-proxy.com/api` → `https://my-proxy.com/api/v1/messages`
With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
- Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
## Usage
```python

43
docs/my-website/docs/providers/datarobot.md Normal file
View file

@ -0,0 +1,43 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DataRobot
LiteLLM supports all models from [DataRobot](https://datarobot.com). Select `datarobot` as the provider to route your request through the `datarobot` OpenAI-compatible endpoint using the upstream [official OpenAI Python API library](https://github.com/openai/openai-python/blob/main/README.md).
## Usage
### Environment variables
```python
import os
from litellm import completion
os.environ["DATAROBOT_API_KEY"] = ""
os.environ["DATAROBOT_API_BASE"] = "" # [OPTIONAL] defaults to https://app.datarobot.com
response = completion(
model="datarobot/openai/gpt-4o-mini",
messages=messages,
)
### Completion
```python
import litellm
import os
response = litellm.completion(
model="datarobot/openai/gpt-4o-mini", # add `datarobot/` prefix to model so litellm knows to route through DataRobot
messages=[
{
"role": "user",
"content": "Hey, how's it going?",
}
],
)
print(response)
```
## DataRobot completion models
🚨 LiteLLM supports _all_ DataRobot LLM gateway models. To get a list for your installation and user account, send the following CURL command:
`curl -X GET -H "Authorization: Bearer $DATAROBOT_API_TOKEN" "$DATAROBOT_ENDPOINT/genai/llmgw/catalog/" | jq | grep 'model":'DATAROBOT_ENDPOINT/genai/llmgw/catalog/`

10
docs/my-website/docs/providers/google_ai_studio/image_gen.md
View file

@ -42,7 +42,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate a single image
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A cute baby sea otter swimming in crystal clear water"
)
@ -64,7 +64,7 @@ async def generate_image():
# Generate image asynchronously
response = await litellm.aimage_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A beautiful sunset over mountains with vibrant colors",
n=1,
)
@ -89,7 +89,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate image with additional parameters
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A futuristic cityscape at night with neon lights",
n=1,
size="1024x1024",
@ -112,7 +112,7 @@ for image in response.data:
model_list:
- model_name: google-imagen
litellm_params:
model: gemini/imagen-4.0-generate-preview-06-06
model: gemini/imagen-4.0-generate-001
api_key: os.environ/GEMINI_API_KEY
model_info:
mode: image_generation
@ -198,7 +198,7 @@ Google AI Studio Image Generation supports the following OpenAI-compatible param
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-preview-06-06"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` |
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |

8
docs/my-website/docs/providers/oci.md
View file

@ -44,7 +44,11 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
# Option 2: pass the private key file path
# oci_key_file="<path/to/oci_key.pem>",
oci_compartment_id=<oci_compartment_id>,
)
print(response)
@ -67,7 +71,11 @@ response = completion(
oci_user=<your_oci_user>,
oci_fingerprint=<your_oci_fingerprint>,
oci_tenancy=<your_oci_tenancy>,
# Provide either the private key string OR the path to the key file:
# Option 1: pass the private key as a string
oci_key=<string_with_content_of_oci_key>,
# Option 2: pass the private key file path
# oci_key_file="<path/to/oci_key.pem>",
oci_compartment_id=<oci_compartment_id>,
)
for chunk in response:

219
docs/my-website/docs/providers/vercel_ai_gateway.md Normal file
View file

@ -0,0 +1,219 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Vercel AI Gateway
## Overview
| Property | Details |
|-------|-------|
| Description | Vercel AI Gateway provides a unified interface to access multiple AI providers through a single endpoint, with built-in caching, rate limiting, and analytics. |
| Provider Route on LiteLLM | `vercel_ai_gateway/` |
| Link to Provider Doc | [Vercel AI Gateway Documentation ↗](https://vercel.com/docs/ai-gateway) |
| Base URL | `https://ai-gateway.vercel.sh/v1` |
| Supported Operations | `/chat/completions`, `/models` |
<br />
<br />
https://vercel.com/docs/ai-gateway
**We support ALL models available through Vercel AI Gateway, just set `vercel_ai_gateway/` as a prefix when sending completion requests**
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key
# OR
os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication
```
## Optional Variables
```python showLineNumbers title="Environment Variables"
os.environ["VERCEL_SITE_URL"] = "" # your site url
# OR
os.environ["VERCEL_APP_NAME"] = "" # your app name
```
Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Vercel AI Gateway Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Vercel AI Gateway Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Vercel AI Gateway call with streaming
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy
Add the following to your LiteLLM Proxy configuration file:
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-4o-gateway
litellm_params:
model: vercel_ai_gateway/openai/gpt-4o
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
- model_name: claude-4-sonnet-gateway
litellm_params:
model: vercel_ai_gateway/anthropic/claude-4-sonnet
api_key: os.environ/VERCEL_AI_GATEWAY_API_KEY
```
Start your LiteLLM Proxy server:
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Vercel AI Gateway via Proxy - Non-streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Non-streaming response
response = client.chat.completions.create(
model="gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}]
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Vercel AI Gateway via Proxy - Streaming"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-proxy-api-key" # Your proxy API key
)
# Streaming response
response = client.chat.completions.create(
model="gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK"
import litellm
# Configure LiteLLM to use your proxy
response = litellm.completion(
model="litellm_proxy/gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
```
```python showLineNumbers title="Vercel AI Gateway via Proxy - LiteLLM SDK Streaming"
import litellm
# Configure LiteLLM to use your proxy with streaming
response = litellm.completion(
model="litellm_proxy/gpt-4o-gateway",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key",
stream=True
)
for chunk in response:
if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
print(chunk.choices[0].delta.content, end="")
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "gpt-4o-gateway",
"messages": [{"role": "user", "content": "Hello, how are you?"}]
}'
```
```bash showLineNumbers title="Vercel AI Gateway via Proxy - cURL Streaming"
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-d '{
"model": "gpt-4o-gateway",
"messages": [{"role": "user", "content": "Hello, how are you?"}],
"stream": true
}'
```
</TabItem>
</Tabs>
For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
## Additional Resources
- [Vercel AI Gateway Documentation](https://vercel.com/docs/ai-gateway)

4
docs/my-website/docs/providers/vertex_image.md
View file

@ -18,7 +18,7 @@ import litellm
# Generate a single image
response = await litellm.aimage_generation(
prompt="An olympic size swimming pool with crystal clear water and modern architecture",
model="vertex_ai/imagen-4.0-generate-preview-06-06",
model="vertex_ai/imagen-4.0-generate-001",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
)
@ -34,7 +34,7 @@ print(response.data[0].url)
model_list:
- model_name: vertex-imagen
litellm_params:
model: vertex_ai/imagen-4.0-generate-preview-06-06
model: vertex_ai/imagen-4.0-generate-001
vertex_ai_project: "your-project-id"
vertex_ai_location: "us-central1"
vertex_ai_credentials: "path/to/service-account.json" # Optional if using environment auth

5
docs/my-website/docs/proxy/config_settings.md
View file

@ -236,7 +236,7 @@ Most values can also be set via `litellm_settings`. If you see overlapping value
```yaml
router_settings:
routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
redis_host: <your-redis-host> # string
redis_password: <your-redis-password> # string
redis_port: <your-redis-port> # string
@ -558,6 +558,7 @@ router_settings:
| LITERAL_API_KEY | API key for Literal integration
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
@ -570,6 +571,7 @@ router_settings:
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@ -580,6 +582,7 @@ router_settings:
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100

5
docs/my-website/docs/proxy/prod.md
View file

@ -90,7 +90,7 @@ Recommended to do this for prod:
```yaml
router_settings:
routing_strategy: usage-based-routing-v2
routing_strategy: simple-shuffle # (default) - recommended for best performance
# redis_url: "os.environ/REDIS_URL"
redis_host: os.environ/REDIS_HOST
redis_port: os.environ/REDIS_PORT
@ -105,6 +105,9 @@ litellm_settings:
password: os.environ/REDIS_PASSWORD
```
> **WARNING**
**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios.
## 5. Disable 'load_dotenv'
Set `export LITELLM_MODE="PRODUCTION"`

10
docs/my-website/docs/proxy/timeout.md
View file

@ -38,9 +38,15 @@ $ litellm --config /path/to/config.yaml
</TabItem>
</Tabs>
### Custom Timeouts, Stream Timeouts - Per Model
For each model you can set `timeout` & `stream_timeout` under `litellm_params`
### Custom Timeouts & Stream Timeouts (Per Model)
For each model, you can set `timeout` and `stream_timeout` under `litellm_params`:
- **`timeout`** → maximum time for the *complete response*.
Use this to cap long-running completions.
- **`stream_timeout`** → maximum time to wait for the *first chunk* (i.e., first token) in a streaming response.
Use this to abort “hanging” providers (e.g., Bedrock slow start) and retry another model.
<Tabs>
<TabItem value="sdk" label="SDK">

284
docs/my-website/docs/routing.md
View file

@ -154,11 +154,153 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
## Advanced - Routing Strategies ⭐️
#### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based, Cost Based
Router provides 4 strategies for routing your calls across multiple deployments:
Router provides multiple strategies for routing your calls across multiple deployments. **We recommend using `simple-shuffle` (default) for best performance in production.**
<Tabs>
<TabItem value="simple-shuffle" label="(Default) Weighted Pick - RECOMMENDED">
**Default and Recommended for Production** - Best performance with minimal latency overhead.
Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
If `rpm` or `tpm` is not provided, it randomly picks a deployment
You can also set a `weight` param, to specify which model should get picked when.
<Tabs>
<TabItem value="rpm" label="RPM-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 900
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 10
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 900, # requests per minute for this API
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 10,
}
},]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
<TabItem value="weight" label="Weight-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 9
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 1
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": {
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 9, # pick this 90% of the time
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 1,
}
}]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="usage-based-v2" label="Rate-Limit Aware v2 (ASYNC)">
> [!WARNING]
**Usage-based routing is not recommended for production due to performance impacts.** Use `simple-shuffle` (default) for optimal performance in high-traffic scenarios. Usage-based routing adds significant latency due to Redis operations for tracking usage across deployments.
**🎉 NEW** This is an async implementation of usage-based-routing.
**Filters out deployment if tpm/rpm limit exceeded** - If you pass in the deployment's tpm/rpm limits.
@ -209,7 +351,7 @@ router = Router(model_list=model_list,
redis_host=os.environ["REDIS_HOST"],
redis_password=os.environ["REDIS_PASSWORD"],
redis_port=os.environ["REDIS_PORT"],
routing_strategy="usage-based-routing-v2" # 👈 KEY CHANGE
routing_strategy="simple-shuffle" # 👈 RECOMMENDED - best performance
enable_pre_call_checks=True, # enables router rate limits for concurrent calls
)
@ -241,7 +383,7 @@ model_list:
rpm: 1000
router_settings:
routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
routing_strategy: simple-shuffle # 👈 RECOMMENDED - best performance
redis_host: <your-redis-host>
redis_password: <your-redis-password>
redis_port: <your-redis-port>
@ -365,143 +507,7 @@ router_settings:
```
</TabItem>
<TabItem value="simple-shuffle" label="(Default) Weighted Pick (Async)">
**Default** Picks a deployment based on the provided **Requests per minute (rpm) or Tokens per minute (tpm)**
If `rpm` or `tpm` is not provided, it randomly picks a deployment
You can also set a `weight` param, to specify which model should get picked when.
<Tabs>
<TabItem value="rpm" label="RPM-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 900
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
rpm: 10
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 900, # requests per minute for this API
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"rpm": 10,
}
},]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
<TabItem value="weight" label="Weight-based shuffling">
##### **LiteLLM Proxy Config.yaml**
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 9
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-functioncalling
api_key: os.environ/AZURE_API_KEY
api_version: os.environ/AZURE_API_VERSION
api_base: os.environ/AZURE_API_BASE
weight: 1
```
##### **Python SDK**
```python
from litellm import Router
import asyncio
model_list = [{
"model_name": "gpt-3.5-turbo", # model alias
"litellm_params": {
"model": "azure/chatgpt-v-2", # actual model name
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 9, # pick this 90% of the time
}
}, {
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"weight": 1,
}
}]
# init router
router = Router(model_list=model_list, routing_strategy="simple-shuffle")
async def router_acompletion():
response = await router.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}]
)
print(response)
return response
asyncio.run(router_acompletion())
```
</TabItem>
</Tabs>
</TabItem>
<TabItem value="usage-based" label="Rate-Limit Aware">
This will route to the deployment with the lowest TPM usage for that minute.

2
docs/my-website/docs/scheduler.md
View file

@ -41,7 +41,7 @@ router = Router(
},
],
timeout=2, # timeout request if takes > 2s
routing_strategy="usage-based-routing-v2",
routing_strategy="simple-shuffle", # recommended for best performance
polling_interval=0.03 # poll queue every 3ms if no healthy deployments
)

6
docs/my-website/release_notes/v1.74.15-stable/index.md
View file

@ -86,9 +86,9 @@ This is great to central AI Platform teams looking to track how they are helping
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Cost per Image |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------------- |
| OpenRouter | `openrouter/x-ai/grok-4` | 256k | $3 | $15 | N/A |
| Google AI Studio | `gemini/imagen-4.0-generate-preview-06-06` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-preview-06-06` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-preview-06-06` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-4.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-001` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-3.0-generate-002` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |

2
docs/my-website/release_notes/v1.75.8/index.md
View file

@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8
ghcr.io/berriai/litellm:v1.75.8-stable
```
</TabItem>

269
docs/my-website/release_notes/v1.76.1-stable/index.md Normal file
View file

@ -0,0 +1,269 @@
---
title: "v1.76.1-stable - Gemini 2.5 Flash Image"
slug: "v1-76-1"
date: 2025-08-30T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.76.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.76.1
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Major Performance Improvements** - 6.5x faster LiteLLM Python SDK completion with fastuuid integration.
- **New Model Support** - Gemini 2.5 Flash Image Preview, Grok Code Fast, and GPT Realtime models
- **Enhanced Provider Support** - DeepSeek-v3.1 pricing on Fireworks AI, Vercel AI Gateway, and improved Anthropic/GitHub Copilot integration
- **MCP Improvements** - Better connection testing and SSE MCP tools bug fixes
## Major Changes
- Added support for using Gemini 2.5 Flash Image Preview with /chat/completions. **🚨 Warning** If you were using `gemini-2.0-flash-exp-image-generation` please follow this migration guide.
[Gemini Image Generation Migration Guide](../../docs/extras/gemini_img_migration)
---
## Performance Improvements
This release includes significant performance optimizations:
- **6.5x faster LiteLLM Python SDK Completion** - Major performance boost for completion operations - [PR #13990](https://github.com/BerriAI/litellm/pull/13990)
- **fastuuid Integration** - 2.1x faster UUID generation with +80 RPS improvement for /chat/completions and other LLM endpoints - [PR #13992](https://github.com/BerriAI/litellm/pull/13992), [PR #14016](https://github.com/BerriAI/litellm/pull/14016)
- **Optimized Request Logging** - Don't print request params by default for +50 RPS improvement - [PR #14015](https://github.com/BerriAI/litellm/pull/14015)
- **Cache Performance** - 21% speedup in InMemoryCache.evict_cache and 45% speedup in `_is_debugging_on` function - [PR #14012](https://github.com/BerriAI/litellm/pull/14012), [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| Google | `gemini-2.5-flash-image-preview` | 1M | $0.30 | $2.50 | Chat completions + image generation ($0.039/image) |
| X.AI | `xai/grok-code-fast` | 256K | $0.20 | $1.50 | Code generation |
| OpenAI | `gpt-realtime` | 32K | $4.00 | $16.00 | Real-time conversation + audio |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3` | 200K | $2.00 | $8.00 | Advanced reasoning |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o3-mini` | 200K | $1.10 | $4.40 | Efficient reasoning |
| Vercel AI Gateway | `vercel_ai_gateway/openai/o4-mini` | 200K | $1.10 | $4.40 | Latest mini model |
| DeepInfra | `deepinfra/zai-org/GLM-4.5` | 131K | $0.55 | $2.00 | Chat completions |
| Perplexity | `perplexity/codellama-34b-instruct` | 16K | $0.35 | $1.40 | Code generation |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/deepseek-v3p1` | 128K | $0.56 | $1.68 | Chat completions |
**Additional Models Added:** Various other Vercel AI Gateway models were added too. See [models.litellm.ai](https://models.litellm.ai) for the full list.
#### Features
- **[Google Gemini](../../docs/providers/gemini)**
- Added support for `gemini-2.5-flash-image-preview` with image return capability - [PR #13979](https://github.com/BerriAI/litellm/pull/13979), [PR #13983](https://github.com/BerriAI/litellm/pull/13983)
- Support for requests with only system prompt - [PR #14010](https://github.com/BerriAI/litellm/pull/14010)
- Fixed invalid model name error for Gemini Imagen models - [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
- **[X.AI](../../docs/providers/xai)**
- Added `xai/grok-code-fast` model family support - [PR #14054](https://github.com/BerriAI/litellm/pull/14054)
- Fixed frequency_penalty parameter for grok-4 models - [PR #14078](https://github.com/BerriAI/litellm/pull/14078)
- **[OpenAI](../../docs/providers/openai)**
- Added support for gpt-realtime models - [PR #14082](https://github.com/BerriAI/litellm/pull/14082)
- Support for reasoning and reasoning_effort parameters by default - [PR #12865](https://github.com/BerriAI/litellm/pull/12865)
- **[Fireworks AI](../../docs/providers/fireworks_ai)**
- Added DeepSeek-v3.1 pricing - [PR #13958](https://github.com/BerriAI/litellm/pull/13958)
- **[DeepInfra](../../docs/providers/deepinfra)**
- Fixed reasoning_effort setting for DeepSeek-V3.1 - [PR #14053](https://github.com/BerriAI/litellm/pull/14053)
- **[GitHub Copilot](../../docs/providers/github_copilot)**
- Added support for thinking and reasoning_effort parameters - [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
- Added image headers support - [PR #13955](https://github.com/BerriAI/litellm/pull/13955)
- **[Anthropic](../../docs/providers/anthropic)**
- Support for custom Anthropic-compatible API endpoints - [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
- Fixed /messages fallback from Anthropic API to Bedrock API - [PR #13946](https://github.com/BerriAI/litellm/pull/13946)
- **[Nebius](../../docs/providers/nebius)**
- Expanded provider models and normalized model IDs - [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
- **[Vertex AI](../../docs/providers/vertex)**
- Fixed Vertex Mistral streaming issues - [PR #13952](https://github.com/BerriAI/litellm/pull/13952)
- Fixed anyOf corner cases for Gemini tool calls - [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
- **[Bedrock](../../docs/providers/bedrock)**
- Fixed structure output issues - [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
- **[OpenRouter](../../docs/providers/openrouter)**
- Added GPT-5 family models pricing - [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
#### New Provider Support
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
- New provider support added - [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
- **[DataRobot](../../docs/providers/datarobot)**
- Added provider documentation - [PR #14038](https://github.com/BerriAI/litellm/pull/14038), [PR #14074](https://github.com/BerriAI/litellm/pull/14074)
---
## LLM API Endpoints
#### Features
- **[Images API](../../docs/image_generation)**
- Support for multiple images in OpenAI images/edits endpoint - [PR #13916](https://github.com/BerriAI/litellm/pull/13916)
- Allow using dynamic `api_key` for image generation requests - [PR #14007](https://github.com/BerriAI/litellm/pull/14007)
- **[Responses API](../../docs/response_api)**
- Fixed `/responses` endpoint ignoring extra_headers in GitHub Copilot - [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
- Added support for new web_search tool - [PR #14083](https://github.com/BerriAI/litellm/pull/14083)
- **[Azure Passthrough](../../docs/providers/azure/azure)**
- Fixed Azure Passthrough request with streaming - [PR #13831](https://github.com/BerriAI/litellm/pull/13831)
#### Bugs
- **General**
- Fixed handling of None metadata in batch requests - [PR #13996](https://github.com/BerriAI/litellm/pull/13996)
- Fixed token_counter with special token input - [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
- Removed incorrect web search support for azure/gpt-4.1 family - [PR #13566](https://github.com/BerriAI/litellm/pull/13566)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- **SSE MCP Tools**
- Bug fix for adding SSE MCP tools - improved connection testing when adding MCPs - [PR #14048](https://github.com/BerriAI/litellm/pull/14048)
[Read More](../../docs/mcp)
---
## Management Endpoints / UI
#### Features
- **Team Management**
- Allow setting Team Member RPM/TPM limits when creating a team - [PR #13943](https://github.com/BerriAI/litellm/pull/13943)
- **UI Improvements**
- Fixed Next.js Security Vulnerabilities in UI Dashboard - [PR #14084](https://github.com/BerriAI/litellm/pull/14084)
- Fixed collapsible navbar design - [PR #14075](https://github.com/BerriAI/litellm/pull/14075)
#### Bugs
- **Authentication**
- Fixed Virtual keys with llm_api type causing Internal Server Error for /anthropic/* and other LLM passthrough routes - [PR #14046](https://github.com/BerriAI/litellm/pull/14046)
---
## Logging / Guardrail Integrations
#### Features
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
- Allow using LANGFUSE_OTEL_HOST for configuring host - [PR #14013](https://github.com/BerriAI/litellm/pull/14013)
- **[Braintrust](../../docs/proxy/logging#braintrust)**
- Added span name metadata feature - [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
- Fixed tests to reference moved attributes in `braintrust_logging` module - [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
- **[OpenMeter](../../docs/proxy/logging#openmeter)**
- Set user from token user_id for OpenMeter integration - [PR #13152](https://github.com/BerriAI/litellm/pull/13152)
#### New Guardrail Support
- **[Noma Security](../../docs/proxy/guardrails)**
- Added Noma Security guardrail support - [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
- **[Pangea](../../docs/proxy/guardrails)**
- Updated Pangea Guardrail to support new AIDR endpoint - [PR #13160](https://github.com/BerriAI/litellm/pull/13160)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **Caching**
- Verify if cache entry has expired prior to serving it to client - [PR #13933](https://github.com/BerriAI/litellm/pull/13933)
- Fixed error saving latency as timedelta on Redis - [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
- **Router**
- Refactored router to choose weights by 'weight', 'rpm', 'tpm' in one loop for simple_shuffle - [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
- **Logging**
- Fixed LoggingWorker graceful shutdown to prevent CancelledError warnings - [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
- Enhanced logging for containers to log on files both with usual format and json format - [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
#### Bugs
- **Dependencies**
- Bumped `orjson` version to "3.11.2" - [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
---
## General Proxy Improvements
#### Features
- **AWS**
- Add support for AWS assume_role with a session token - [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
- **OCI Provider**
- Added oci_key_file as an optional_parameter - [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
- **Configuration**
- Allow configuration to set threshold before request entry in spend log gets truncated - [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
- Enhanced proxy_config configuration: add support for existing configmap in Helm charts - [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
- **Docker**
- Added back supervisor to non-root image - [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
---
## New Contributors
* @ArthurRenault made their first contribution in [PR #13922](https://github.com/BerriAI/litellm/pull/13922)
* @stevenmanton made their first contribution in [PR #13919](https://github.com/BerriAI/litellm/pull/13919)
* @uc4w6c made their first contribution in [PR #13914](https://github.com/BerriAI/litellm/pull/13914)
* @nielsbosma made their first contribution in [PR #13573](https://github.com/BerriAI/litellm/pull/13573)
* @Yuki-Imajuku made their first contribution in [PR #13567](https://github.com/BerriAI/litellm/pull/13567)
* @codeflash-ai[bot] made their first contribution in [PR #13988](https://github.com/BerriAI/litellm/pull/13988)
* @ColeFrench made their first contribution in [PR #13978](https://github.com/BerriAI/litellm/pull/13978)
* @dttran-glo made their first contribution in [PR #13969](https://github.com/BerriAI/litellm/pull/13969)
* @manascb1344 made their first contribution in [PR #13965](https://github.com/BerriAI/litellm/pull/13965)
* @DorZion made their first contribution in [PR #13572](https://github.com/BerriAI/litellm/pull/13572)
* @edwardsamuel made their first contribution in [PR #13536](https://github.com/BerriAI/litellm/pull/13536)
* @blahgeek made their first contribution in [PR #13374](https://github.com/BerriAI/litellm/pull/13374)
* @Deviad made their first contribution in [PR #13394](https://github.com/BerriAI/litellm/pull/13394)
* @XSAM made their first contribution in [PR #13775](https://github.com/BerriAI/litellm/pull/13775)
* @KRRT7 made their first contribution in [PR #14012](https://github.com/BerriAI/litellm/pull/14012)
* @ikaadil made their first contribution in [PR #13991](https://github.com/BerriAI/litellm/pull/13991)
* @timelfrink made their first contribution in [PR #13691](https://github.com/BerriAI/litellm/pull/13691)
* @qidu made their first contribution in [PR #13562](https://github.com/BerriAI/litellm/pull/13562)
* @nagyv made their first contribution in [PR #13243](https://github.com/BerriAI/litellm/pull/13243)
* @xywei made their first contribution in [PR #12885](https://github.com/BerriAI/litellm/pull/12885)
* @ericgtkb made their first contribution in [PR #12797](https://github.com/BerriAI/litellm/pull/12797)
* @NoWall57 made their first contribution in [PR #13945](https://github.com/BerriAI/litellm/pull/13945)
* @lmwang9527 made their first contribution in [PR #14050](https://github.com/BerriAI/litellm/pull/14050)
* @WilsonSunBritten made their first contribution in [PR #14042](https://github.com/BerriAI/litellm/pull/14042)
* @Const-antine made their first contribution in [PR #14041](https://github.com/BerriAI/litellm/pull/14041)
* @dmvieira made their first contribution in [PR #14040](https://github.com/BerriAI/litellm/pull/14040)
* @gotsysdba made their first contribution in [PR #14036](https://github.com/BerriAI/litellm/pull/14036)
* @moshemorad made their first contribution in [PR #14005](https://github.com/BerriAI/litellm/pull/14005)
* @joshualipman123 made their first contribution in [PR #13144](https://github.com/BerriAI/litellm/pull/13144)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.0-nightly...v1.76.1)**

3
docs/my-website/sidebars.js
View file

@ -464,6 +464,7 @@ const sidebars = {
"providers/replicate",
"providers/togetherai",
"providers/v0",
"providers/vercel_ai_gateway",
"providers/morph",
"providers/lambda_ai",
"providers/novita",
@ -482,6 +483,7 @@ const sidebars = {
"providers/dashscope",
"providers/bytez",
"providers/oci",
"providers/datarobot",
],
},
{
@ -493,6 +495,7 @@ const sidebars = {
"guides/finetuned_models",
"guides/security_settings",
"completion/audio",
"completion/image_generation_chat",
"completion/web_search",
"completion/document_understanding",
"completion/vision",

8
litellm/__init__.py
View file

@ -226,6 +226,7 @@ vertex_location: Optional[str] = None
predibase_tenant_id: Optional[str] = None
togetherai_api_key: Optional[str] = None
cloudflare_api_key: Optional[str] = None
vercel_ai_gateway_key: Optional[str] = None
baseten_key: Optional[str] = None
llama_api_key: Optional[str] = None
aleph_alpha_key: Optional[str] = None
@ -542,6 +543,7 @@ hyperbolic_models: Set = set()
recraft_models: Set = set()
cometapi_models: Set = set()
oci_models: Set = set()
vercel_ai_gateway_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -599,6 +601,8 @@ def add_known_models():
empower_models.add(key)
elif value.get("litellm_provider") == "openrouter":
openrouter_models.add(key)
elif value.get("litellm_provider") == "vercel_ai_gateway":
vercel_ai_gateway_models.add(key)
elif value.get("litellm_provider") == "datarobot":
datarobot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
@ -835,6 +839,7 @@ model_list = list(
| recraft_models
| cometapi_models
| oci_models
| vercel_ai_gateway_models
)
model_list_set = set(model_list)
@ -853,6 +858,7 @@ models_by_provider: dict = {
"together_ai": together_ai_models,
"baseten": baseten_models,
"openrouter": openrouter_models,
"vercel_ai_gateway": vercel_ai_gateway_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
"ai21": ai21_models,
@ -1247,6 +1253,7 @@ from .llms.oci.chat.transformation import OCIChatConfig
from .llms.morph.chat.transformation import MorphChatConfig
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
from .main import * # type: ignore
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
@ -1278,7 +1285,6 @@ from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .vector_stores import *
from .batch_completion.main import * # type: ignore
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *

49
litellm/_logging.py
View file

@ -4,41 +4,21 @@ import os
import sys
from datetime import datetime
from logging import Formatter
set_verbose = False
def __strtobool(val: str) -> bool:
"""Convert a string representation of truth to true (1) or false (0).
True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
'val' is anything else.
"""
val = val.lower()
if val in ('y', 'yes', 't', 'true', 'on', '1'):
return True
elif val in ('n', 'no', 'f', 'false', 'off', '0'):
return False
else:
raise ValueError(f"invalid truth value {val!r}")
if set_verbose is True:
logging.warning(
"`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."
)
json_logs = __strtobool(os.getenv("JSON_LOGS", "False"))
json_logs = bool(os.getenv("JSON_LOGS", False))
# Create a handler for the logger (you may need to adapt this based on your needs)
log_level = os.getenv("LITELLM_LOG", "DEBUG")
numeric_level: str = getattr(logging, log_level.upper())
handler = logging.StreamHandler()
handler.setLevel(numeric_level)
log_file = os.getenv("LITELLM_LOG_FILE", "")
file_handler = None
if log_file:
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(numeric_level)
class JsonFormatter(Formatter):
def __init__(self):
super(JsonFormatter, self).__init__()
@ -60,7 +40,6 @@ class JsonFormatter(Formatter):
return json.dumps(json_record)
json_formatter = JsonFormatter()
# Function to set up exception handlers for JSON logging
def _setup_json_exception_handlers(formatter):
@ -110,10 +89,8 @@ def _setup_json_exception_handlers(formatter):
# Create a formatter and set it for the handler
if json_logs:
handler.setFormatter(json_formatter)
if file_handler:
file_handler.setFormatter(json_formatter)
_setup_json_exception_handlers(json_formatter)
handler.setFormatter(JsonFormatter())
_setup_json_exception_handlers(JsonFormatter())
else:
formatter = logging.Formatter(
"\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
@ -121,18 +98,11 @@ else:
)
handler.setFormatter(formatter)
if file_handler:
file_handler.setFormatter(formatter)
verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
verbose_router_logger = logging.getLogger("LiteLLM Router")
verbose_logger = logging.getLogger("LiteLLM")
# Set logger levels
verbose_proxy_logger.setLevel(numeric_level)
verbose_router_logger.setLevel(numeric_level)
verbose_logger.setLevel(numeric_level)
# Add the handler to the logger
verbose_router_logger.addHandler(handler)
verbose_proxy_logger.addHandler(handler)
@ -155,13 +125,6 @@ def _suppress_loggers():
# Call the suppression function
_suppress_loggers()
if file_handler:
verbose_router_logger.addHandler(file_handler)
verbose_proxy_logger.addHandler(file_handler)
verbose_logger.addHandler(file_handler)
ALL_LOGGERS = [
logging.getLogger(),
verbose_logger,
@ -190,10 +153,10 @@ def _turn_on_json():
- Adds a JSON formatter to all loggers
"""
handler = logging.StreamHandler()
handler.setFormatter(json_formatter)
handler.setFormatter(JsonFormatter())
_initialize_loggers_with_handler(handler)
# Set up exception handlers
_setup_json_exception_handlers(json_formatter)
_setup_json_exception_handlers(JsonFormatter())
def _turn_on_debug():

15
litellm/caching/in_memory_cache.py
View file

@ -112,14 +112,15 @@ class InMemoryCache(BaseCache):
- 3. the size of in-memory cache is bounded
"""
for key in list(self.ttl_dict.keys()):
if self._is_key_expired(key):
self._remove_key(key)
current_time = time.time()
expired_keys = [key for key, ttl in self.ttl_dict.items() if current_time > ttl]
for key in expired_keys:
self._remove_key(key)
# de-reference the removed item
# https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
# One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
# This can occur when an object is referenced by another object, but the reference is never removed.
# de-reference the removed item
# https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
# One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
# This can occur when an object is referenced by another object, but the reference is never removed.
def allow_ttl_override(self, key: str) -> bool:
"""

478
litellm/constants.py
View file

@ -157,6 +157,7 @@ NON_LLM_CONNECTION_TIMEOUT = int(
os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15)
) # timeout for adjacent services (e.g. jwt auth)
MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000))
MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 1000))
BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
REPLICATE_POLLING_DELAY_SECONDS = float(
os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
@ -288,6 +289,7 @@ LITELLM_CHAT_PROVIDERS = [
"oci",
"morph",
"lambda_ai",
"vercel_ai_gateway",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@ -420,6 +422,7 @@ openai_compatible_endpoints: List = [
"https://api.morphllm.com/v1",
"https://api.lambda.ai/v1",
"https://api.hyperbolic.xyz/v1",
"https://ai-gateway.vercel.sh/v1",
]
@ -462,6 +465,7 @@ openai_compatible_providers: List = [
"morph",
"lambda_ai",
"hyperbolic",
"vercel_ai_gateway",
"aiml",
]
openai_text_completion_compatible_providers: List = (
@ -486,227 +490,247 @@ _openai_like_providers: List = [
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
replicate_models: set = set([
# llama replicate supported LLMs
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
"meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db",
# Vicuna
"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
"joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe",
# Flan T-5
"daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f",
# Others
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
])
replicate_models: set = set(
[
# llama replicate supported LLMs
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
"meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db",
# Vicuna
"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
"joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe",
# Flan T-5
"daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f",
# Others
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
]
)
clarifai_models: set = set([
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
"clarifai/gcp.generate.gemma-1_1-7b-it",
"clarifai/mistralai.completion.mixtral-8x22B",
"clarifai/cohere.generate.command-r-plus",
"clarifai/databricks.drbx.dbrx-instruct",
"clarifai/mistralai.completion.mistral-large",
"clarifai/mistralai.completion.mistral-medium",
"clarifai/mistralai.completion.mistral-small",
"clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1",
"clarifai/gcp.generate.gemma-2b-it",
"clarifai/gcp.generate.gemma-7b-it",
"clarifai/deci.decilm.deciLM-7B-instruct",
"clarifai/mistralai.completion.mistral-7B-Instruct",
"clarifai/gcp.generate.gemini-pro",
"clarifai/anthropic.completion.claude-v1",
"clarifai/anthropic.completion.claude-instant-1_2",
"clarifai/anthropic.completion.claude-instant",
"clarifai/anthropic.completion.claude-v2",
"clarifai/anthropic.completion.claude-2_1",
"clarifai/meta.Llama-2.codeLlama-70b-Python",
"clarifai/meta.Llama-2.codeLlama-70b-Instruct",
"clarifai/openai.completion.gpt-3_5-turbo-instruct",
"clarifai/meta.Llama-2.llama2-7b-chat",
"clarifai/meta.Llama-2.llama2-13b-chat",
"clarifai/meta.Llama-2.llama2-70b-chat",
"clarifai/openai.chat-completion.gpt-4-turbo",
"clarifai/microsoft.text-generation.phi-2",
"clarifai/meta.Llama-2.llama2-7b-chat-vllm",
"clarifai/upstage.solar.solar-10_7b-instruct",
"clarifai/openchat.openchat.openchat-3_5-1210",
"clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B",
"clarifai/gcp.generate.text-bison",
"clarifai/meta.Llama-2.llamaGuard-7b",
"clarifai/fblgit.una-cybertron.una-cybertron-7b-v2",
"clarifai/openai.chat-completion.GPT-4",
"clarifai/openai.chat-completion.GPT-3_5-turbo",
"clarifai/ai21.complete.Jurassic2-Grande",
"clarifai/ai21.complete.Jurassic2-Grande-Instruct",
"clarifai/ai21.complete.Jurassic2-Jumbo-Instruct",
"clarifai/ai21.complete.Jurassic2-Jumbo",
"clarifai/ai21.complete.Jurassic2-Large",
"clarifai/cohere.generate.cohere-generate-command",
"clarifai/wizardlm.generate.wizardCoder-Python-34B",
"clarifai/wizardlm.generate.wizardLM-70B",
"clarifai/tiiuae.falcon.falcon-40b-instruct",
"clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat",
"clarifai/gcp.generate.code-gecko",
"clarifai/gcp.generate.code-bison",
"clarifai/mistralai.completion.mistral-7B-OpenOrca",
"clarifai/mistralai.completion.openHermes-2-mistral-7B",
"clarifai/wizardlm.generate.wizardLM-13B",
"clarifai/huggingface-research.zephyr.zephyr-7B-alpha",
"clarifai/wizardlm.generate.wizardCoder-15B",
"clarifai/microsoft.text-generation.phi-1_5",
"clarifai/databricks.Dolly-v2.dolly-v2-12b",
"clarifai/bigcode.code.StarCoder",
"clarifai/salesforce.xgen.xgen-7b-8k-instruct",
"clarifai/mosaicml.mpt.mpt-7b-instruct",
"clarifai/anthropic.completion.claude-3-opus",
"clarifai/anthropic.completion.claude-3-sonnet",
"clarifai/gcp.generate.gemini-1_5-pro",
"clarifai/gcp.generate.imagen-2",
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
])
clarifai_models: set = set(
[
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
"clarifai/gcp.generate.gemma-1_1-7b-it",
"clarifai/mistralai.completion.mixtral-8x22B",
"clarifai/cohere.generate.command-r-plus",
"clarifai/databricks.drbx.dbrx-instruct",
"clarifai/mistralai.completion.mistral-large",
"clarifai/mistralai.completion.mistral-medium",
"clarifai/mistralai.completion.mistral-small",
"clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1",
"clarifai/gcp.generate.gemma-2b-it",
"clarifai/gcp.generate.gemma-7b-it",
"clarifai/deci.decilm.deciLM-7B-instruct",
"clarifai/mistralai.completion.mistral-7B-Instruct",
"clarifai/gcp.generate.gemini-pro",
"clarifai/anthropic.completion.claude-v1",
"clarifai/anthropic.completion.claude-instant-1_2",
"clarifai/anthropic.completion.claude-instant",
"clarifai/anthropic.completion.claude-v2",
"clarifai/anthropic.completion.claude-2_1",
"clarifai/meta.Llama-2.codeLlama-70b-Python",
"clarifai/meta.Llama-2.codeLlama-70b-Instruct",
"clarifai/openai.completion.gpt-3_5-turbo-instruct",
"clarifai/meta.Llama-2.llama2-7b-chat",
"clarifai/meta.Llama-2.llama2-13b-chat",
"clarifai/meta.Llama-2.llama2-70b-chat",
"clarifai/openai.chat-completion.gpt-4-turbo",
"clarifai/microsoft.text-generation.phi-2",
"clarifai/meta.Llama-2.llama2-7b-chat-vllm",
"clarifai/upstage.solar.solar-10_7b-instruct",
"clarifai/openchat.openchat.openchat-3_5-1210",
"clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B",
"clarifai/gcp.generate.text-bison",
"clarifai/meta.Llama-2.llamaGuard-7b",
"clarifai/fblgit.una-cybertron.una-cybertron-7b-v2",
"clarifai/openai.chat-completion.GPT-4",
"clarifai/openai.chat-completion.GPT-3_5-turbo",
"clarifai/ai21.complete.Jurassic2-Grande",
"clarifai/ai21.complete.Jurassic2-Grande-Instruct",
"clarifai/ai21.complete.Jurassic2-Jumbo-Instruct",
"clarifai/ai21.complete.Jurassic2-Jumbo",
"clarifai/ai21.complete.Jurassic2-Large",
"clarifai/cohere.generate.cohere-generate-command",
"clarifai/wizardlm.generate.wizardCoder-Python-34B",
"clarifai/wizardlm.generate.wizardLM-70B",
"clarifai/tiiuae.falcon.falcon-40b-instruct",
"clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat",
"clarifai/gcp.generate.code-gecko",
"clarifai/gcp.generate.code-bison",
"clarifai/mistralai.completion.mistral-7B-OpenOrca",
"clarifai/mistralai.completion.openHermes-2-mistral-7B",
"clarifai/wizardlm.generate.wizardLM-13B",
"clarifai/huggingface-research.zephyr.zephyr-7B-alpha",
"clarifai/wizardlm.generate.wizardCoder-15B",
"clarifai/microsoft.text-generation.phi-1_5",
"clarifai/databricks.Dolly-v2.dolly-v2-12b",
"clarifai/bigcode.code.StarCoder",
"clarifai/salesforce.xgen.xgen-7b-8k-instruct",
"clarifai/mosaicml.mpt.mpt-7b-instruct",
"clarifai/anthropic.completion.claude-3-opus",
"clarifai/anthropic.completion.claude-3-sonnet",
"clarifai/gcp.generate.gemini-1_5-pro",
"clarifai/gcp.generate.imagen-2",
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
]
)
huggingface_models: set = set([
"meta-llama/Llama-2-7b-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-2-13b-hf",
"meta-llama/Llama-2-13b-chat-hf",
"meta-llama/Llama-2-70b-hf",
"meta-llama/Llama-2-70b-chat-hf",
"meta-llama/Llama-2-7b",
"meta-llama/Llama-2-7b-chat",
"meta-llama/Llama-2-13b",
"meta-llama/Llama-2-13b-chat",
"meta-llama/Llama-2-70b",
"meta-llama/Llama-2-70b-chat",
]) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = set([
"empower/empower-functions",
"empower/empower-functions-small",
])
huggingface_models: set = set(
[
"meta-llama/Llama-2-7b-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-2-13b-hf",
"meta-llama/Llama-2-13b-chat-hf",
"meta-llama/Llama-2-70b-hf",
"meta-llama/Llama-2-70b-chat-hf",
"meta-llama/Llama-2-7b",
"meta-llama/Llama-2-7b-chat",
"meta-llama/Llama-2-13b",
"meta-llama/Llama-2-13b-chat",
"meta-llama/Llama-2-70b",
"meta-llama/Llama-2-70b-chat",
]
) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = set(
[
"empower/empower-functions",
"empower/empower-functions-small",
]
)
together_ai_models: set = set([
# llama llms - chat
"togethercomputer/llama-2-70b-chat",
# llama llms - language / instruct
"togethercomputer/llama-2-70b",
"togethercomputer/LLaMA-2-7B-32K",
"togethercomputer/Llama-2-7B-32K-Instruct",
"togethercomputer/llama-2-7b",
# falcon llms
"togethercomputer/falcon-40b-instruct",
"togethercomputer/falcon-7b-instruct",
# alpaca
"togethercomputer/alpaca-7b",
# chat llms
"HuggingFaceH4/starchat-alpha",
# code llms
"togethercomputer/CodeLlama-34b",
"togethercomputer/CodeLlama-34b-Instruct",
"togethercomputer/CodeLlama-34b-Python",
"defog/sqlcoder",
"NumbersStation/nsql-llama-2-7B",
"WizardLM/WizardCoder-15B-V1.0",
"WizardLM/WizardCoder-Python-34B-V1.0",
# language llms
"NousResearch/Nous-Hermes-Llama2-13b",
"Austism/chronos-hermes-13b",
"upstage/SOLAR-0-70b-16bit",
"WizardLM/WizardLM-70B-V1.0",
])
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
together_ai_models: set = set(
[
# llama llms - chat
"togethercomputer/llama-2-70b-chat",
# llama llms - language / instruct
"togethercomputer/llama-2-70b",
"togethercomputer/LLaMA-2-7B-32K",
"togethercomputer/Llama-2-7B-32K-Instruct",
"togethercomputer/llama-2-7b",
# falcon llms
"togethercomputer/falcon-40b-instruct",
"togethercomputer/falcon-7b-instruct",
# alpaca
"togethercomputer/alpaca-7b",
# chat llms
"HuggingFaceH4/starchat-alpha",
# code llms
"togethercomputer/CodeLlama-34b",
"togethercomputer/CodeLlama-34b-Instruct",
"togethercomputer/CodeLlama-34b-Python",
"defog/sqlcoder",
"NumbersStation/nsql-llama-2-7B",
"WizardLM/WizardCoder-15B-V1.0",
"WizardLM/WizardCoder-Python-34B-V1.0",
# language llms
"NousResearch/Nous-Hermes-Llama2-13b",
"Austism/chronos-hermes-13b",
"upstage/SOLAR-0-70b-16bit",
"WizardLM/WizardLM-70B-V1.0",
]
)
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
baseten_models: set = set([
"qvv0xeq",
"q841o8w",
"31dxrj3",
]) # FALCON 7B # WizardLM # Mosaic ML
baseten_models: set = set(
[
"qvv0xeq",
"q841o8w",
"31dxrj3",
]
) # FALCON 7B # WizardLM # Mosaic ML
featherless_ai_models: set = set([
"featherless-ai/Qwerky-72B",
"featherless-ai/Qwerky-QwQ-32B",
"Qwen/Qwen2.5-72B-Instruct",
"all-hands/openhands-lm-32b-v0.1",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"deepseek-ai/DeepSeek-V3-0324",
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mistral-Nemo-Instruct-2407",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
])
featherless_ai_models: set = set(
[
"featherless-ai/Qwerky-72B",
"featherless-ai/Qwerky-QwQ-32B",
"Qwen/Qwen2.5-72B-Instruct",
"all-hands/openhands-lm-32b-v0.1",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"deepseek-ai/DeepSeek-V3-0324",
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mistral-Nemo-Instruct-2407",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
]
)
nebius_models: set = set([
# deepseek models
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
"deepseek-ai/DeepSeek-V3",
"deepseek-ai/DeepSeek-R1",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
# google models
"google/gemma-2-2b-it",
"google/gemma-2-9b-it-fast",
# llama models
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Meta-Llama-3.1-70B-Instruct",
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"meta-llama/Meta-Llama-3.1-405B-Instruct",
"NousResearch/Hermes-3-Llama-405B",
# microsoft models
"microsoft/phi-4",
# mistral models
"mistralai/Mistral-Nemo-Instruct-2407",
"mistralai/Devstral-Small-2505",
# moonshot models
"moonshotai/Kimi-K2-Instruct",
# nvidia models
"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
"nvidia/Llama-3_3-Nemotron-Super-49B-v1",
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# qwen models
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B",
"Qwen/Qwen3-32B",
"Qwen/Qwen3-14B",
"Qwen/Qwen3-4B-fast",
"Qwen/Qwen2.5-Coder-7B",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"Qwen/Qwen2.5-72B-Instruct",
"Qwen/QwQ-32B",
"Qwen/Qwen3-30B-A3B-Thinking-2507",
"Qwen/Qwen3-30B-A3B-Instruct-2507",
# zai models
"zai-org/GLM-4.5",
"zai-org/GLM-4.5-Air",
# other models
"aaditya/Llama3-OpenBioLLM-70B",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
"all-hands/openhands-lm-32b-v0.1",
])
nebius_models: set = set(
[
# deepseek models
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
"deepseek-ai/DeepSeek-V3",
"deepseek-ai/DeepSeek-R1",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
# google models
"google/gemma-2-2b-it",
"google/gemma-2-9b-it-fast",
# llama models
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Meta-Llama-3.1-70B-Instruct",
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"meta-llama/Meta-Llama-3.1-405B-Instruct",
"NousResearch/Hermes-3-Llama-405B",
# microsoft models
"microsoft/phi-4",
# mistral models
"mistralai/Mistral-Nemo-Instruct-2407",
"mistralai/Devstral-Small-2505",
# moonshot models
"moonshotai/Kimi-K2-Instruct",
# nvidia models
"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
"nvidia/Llama-3_3-Nemotron-Super-49B-v1",
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# qwen models
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B",
"Qwen/Qwen3-32B",
"Qwen/Qwen3-14B",
"Qwen/Qwen3-4B-fast",
"Qwen/Qwen2.5-Coder-7B",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"Qwen/Qwen2.5-72B-Instruct",
"Qwen/QwQ-32B",
"Qwen/Qwen3-30B-A3B-Thinking-2507",
"Qwen/Qwen3-30B-A3B-Instruct-2507",
# zai models
"zai-org/GLM-4.5",
"zai-org/GLM-4.5-Air",
# other models
"aaditya/Llama3-OpenBioLLM-70B",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
"all-hands/openhands-lm-32b-v0.1",
]
)
dashscope_models: set = set([
"qwen-turbo",
"qwen-plus",
"qwen-max",
"qwen-turbo-latest",
"qwen-plus-latest",
"qwen-max-latest",
"qwq-32b",
"qwen3-235b-a22b",
"qwen3-32b",
"qwen3-30b-a3b",
])
dashscope_models: set = set(
[
"qwen-turbo",
"qwen-plus",
"qwen-max",
"qwen-turbo-latest",
"qwen-plus-latest",
"qwen-max-latest",
"qwq-32b",
"qwen3-235b-a22b",
"qwen3-32b",
"qwen3-30b-a3b",
]
)
nebius_embedding_models: set = set([
"BAAI/bge-en-icl",
"BAAI/bge-multilingual-gemma2",
"intfloat/e5-mistral-7b-instruct",
])
nebius_embedding_models: set = set(
[
"BAAI/bge-en-icl",
"BAAI/bge-multilingual-gemma2",
"intfloat/e5-mistral-7b-instruct",
]
)
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
@ -721,20 +745,24 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
]
open_ai_embedding_models: set = set(["text-embedding-ada-002"])
cohere_embedding_models: set = set([
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
"embed-multilingual-v3.0",
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
])
bedrock_embedding_models: set = set([
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
])
cohere_embedding_models: set = set(
[
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
"embed-multilingual-v3.0",
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
]
)
bedrock_embedding_models: set = set(
[
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
]
)
known_tokenizer_config = {
"mistralai/Mistral-7B-Instruct-v0.1": {

3
litellm/images/main.py
View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast, overload, List
from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
import httpx
@ -347,6 +347,7 @@ def image_generation( # noqa: PLR0915
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
return llm_http_handler.image_generation_handler(
api_key=api_key,
model=model,
prompt=prompt,
image_generation_provider_config=image_generation_config,

13
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -141,6 +141,17 @@ class LangfuseOtelLogger(OpenTelemetry):
value = str(value)
safe_set_attribute(span, enum_attr.value, value)
@staticmethod
def _get_langfuse_otel_host() -> Optional[str]:
"""
Returns the Langfuse OTEL host based on environment variables.
Returned in the following order of precedence:
1. LANGFUSE_OTEL_HOST
2. LANGFUSE_HOST
"""
return os.environ.get("LANGFUSE_OTEL_HOST") or os.environ.get("LANGFUSE_HOST")
@staticmethod
def get_langfuse_otel_config() -> LangfuseOtelConfig:
"""
@ -166,7 +177,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
# Determine endpoint - default to US cloud
langfuse_host = os.environ.get("LANGFUSE_HOST", None)
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it

12
litellm/integrations/openmeter.py
View file

@ -66,8 +66,18 @@ class OpenMeterLogger(CustomLogger):
}
user_param = kwargs.get("user", None) # end-user passed in via 'user' param
# If no user provided directly, try to get it from token user_id
if user_param is None:
raise Exception("OpenMeter: user is required")
# Check if user_id is available from the API key metadata
litellm_params = kwargs.get("litellm_params", {})
metadata = litellm_params.get("metadata", {})
user_api_key_user_id = metadata.get("user_api_key_user_id", None)
if user_api_key_user_id is not None:
user_param = user_api_key_user_id
else:
raise Exception("OpenMeter: user is required")
# Ensure subject is always a string for OpenMeter API
subject = str(user_param)

3
litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py
View file

@ -8,6 +8,7 @@ It searches the vector store for relevant context and appends it to the messages
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast
import litellm
import litellm.vector_stores
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
@ -192,4 +193,4 @@ class VectorStorePreCallHook(CustomLogger):
modified_messages.insert(-1, cast(AllMessageValues, context_message))
return modified_messages
return messages
return messages

10
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -249,6 +249,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.hyperbolic.xyz/v1":
custom_llm_provider = "hyperbolic"
dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
elif endpoint == "https://ai-gateway.vercel.sh/v1":
custom_llm_provider = "vercel_ai_gateway"
dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(
@ -742,6 +745,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "vercel_ai_gateway":
(
api_base,
dynamic_api_key,
) = litellm.VercelAIGatewayConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "aiml":
(
api_base,

9
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -131,6 +131,8 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.AzureOpenAIConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "openrouter":
return litellm.OpenrouterConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "vercel_ai_gateway":
return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral":
# mistal and codestral api have the exact same params
if request_type == "chat_completion":
@ -268,10 +270,9 @@ def get_supported_openai_params( # noqa: PLR0915
from litellm.llms.elevenlabs.audio_transcription.transformation import (
ElevenLabsAudioTranscriptionConfig,
)
return (
ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
model=model
)
return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
model=model
)
elif custom_llm_provider in litellm._custom_providers:
if request_type == "chat_completion":

6
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -580,7 +580,9 @@ class StandardBuiltInToolCostTracking:
return WebSearchOptions(**kwargs.get("web_search_options", {}))
tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs, "web_search_preview"
kwargs=kwargs, tool_type="web_search_preview"
) or StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search"
)
if tools:
# Look for web search tool in the tools array
@ -612,6 +614,8 @@ class StandardBuiltInToolCostTracking:
def _is_web_search_tool_call(tool: Dict) -> bool:
if tool.get("type", None) == "web_search_preview":
return True
if tool.get("type", None) == "web_search":
return True
if "search_context_size" in tool:
return True
return False

7
litellm/litellm_core_utils/logging_worker.py
View file

@ -57,9 +57,10 @@ class LoggingWorker:
finally:
self._queue.task_done()
except asyncio.CancelledError as e:
verbose_logger.exception(f"LoggingWorker cancelled: {e}")
pass
except asyncio.CancelledError:
verbose_logger.debug("LoggingWorker cancelled during shutdown")
# Attempt to clear remaining items to prevent "never awaited" warnings
await self.clear_queue()
def enqueue(self, coroutine: Coroutine) -> None:
"""

8
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3846,7 +3846,13 @@ def function_call_prompt(messages: list, functions: list):
function_added_to_prompt = False
for message in messages:
if "system" in message["role"]:
message["content"] += f""" {function_prompt}"""
if isinstance(message["content"], str):
message["content"] += f""" {function_prompt}"""
else:
message["content"].append({
"type": "text",
"text": f""" {function_prompt}"""
})
function_added_to_prompt = True
if function_added_to_prompt is False:

92
litellm/litellm_core_utils/streaming_handler.py
View file

@ -20,7 +20,9 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.types.llms.openai import ChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import Delta
from litellm.types.utils import (
Delta,
)
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import (
ModelResponse,
@ -35,6 +37,12 @@ from .exception_mapping_utils import exception_type
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
# Constants for special delta attribute names
AUDIO_ATTRIBUTE = "audio"
IMAGE_ATTRIBUTE = "image"
TOOL_CALLS_ATTRIBUTE = "tool_calls"
FUNCTION_CALL_ATTRIBUTE = "function_call"
def is_async_iterable(obj: Any) -> bool:
"""
@ -766,6 +774,66 @@ class CustomStreamWrapper:
model_response.choices[0].delta = Delta(**_initial_delta)
return model_response
def _has_special_delta_content(self, model_response: ModelResponseStream) -> bool:
"""
Check if the delta contains special content types (tool_calls, function_call, audio, or image).
"""
if len(model_response.choices) == 0:
return False
delta = model_response.choices[0].delta
# Check for tool_calls or function_call
if getattr(delta, TOOL_CALLS_ATTRIBUTE, None) is not None or getattr(delta, FUNCTION_CALL_ATTRIBUTE, None) is not None:
return True
# Check for audio
if hasattr(delta, AUDIO_ATTRIBUTE) and getattr(delta, AUDIO_ATTRIBUTE, None) is not None:
return True
# Check for image
if hasattr(delta, IMAGE_ATTRIBUTE) and getattr(delta, IMAGE_ATTRIBUTE, None) is not None:
return True
return False
def _handle_special_delta_content(self, model_response: ModelResponseStream) -> ModelResponseStream:
"""
Handle special delta content types by stripping role and returning the response.
"""
return self.strip_role_from_delta(model_response)
def _has_special_delta_attribute(self, delta, attribute_name: str) -> bool:
"""
Check if delta has a specific attribute and it's not None.
"""
return delta is not None and getattr(delta, attribute_name, None) is not None
def _copy_delta_attribute(self, source_delta, target_delta, attribute_name: str) -> None:
"""
Copy a specific attribute from source delta to target delta.
"""
setattr(target_delta, attribute_name, getattr(source_delta, attribute_name))
def _has_any_special_delta_attributes(self, delta) -> bool:
"""
Check if delta has any special attributes (audio, image).
"""
special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
for attribute in special_attributes:
if self._has_special_delta_attribute(delta, attribute):
return True
return False
def _handle_special_delta_attributes(self, delta, model_response: "ModelResponseStream") -> None:
"""
Handle special delta attributes (audio, image) by copying them to model_response.
"""
special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
for attribute in special_attributes:
if self._has_special_delta_attribute(delta, attribute):
self._copy_delta_attribute(delta, model_response.choices[0].delta, attribute)
def return_processed_chunk_logic( # noqa
self,
completion_obj: Dict[str, Any],
@ -888,20 +956,8 @@ class CustomStreamWrapper:
self.sent_last_chunk = True
return model_response
elif (
model_response.choices[0].delta.tool_calls is not None
or model_response.choices[0].delta.function_call is not None
):
model_response = self.strip_role_from_delta(model_response)
return model_response
elif (
len(model_response.choices) > 0
and hasattr(model_response.choices[0].delta, "audio")
and model_response.choices[0].delta.audio is not None
):
model_response = self.strip_role_from_delta(model_response)
return model_response
elif self._has_special_delta_content(model_response):
return self._handle_special_delta_content(model_response)
else:
if hasattr(model_response, "usage"):
self.chunks.append(model_response)
@ -1374,10 +1430,8 @@ class CustomStreamWrapper:
)
)
model_response.choices[0].delta = Delta()
elif (
delta is not None and getattr(delta, "audio", None) is not None
):
model_response.choices[0].delta.audio = delta.audio
elif self._has_any_special_delta_attributes(delta):
self._handle_special_delta_attributes(delta, model_response)
else:
try:
delta = (

7
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2681,6 +2681,7 @@ class BaseLLMHTTPHandler:
_is_async: bool = False,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
api_key: Optional[str] = None,
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
@ -2705,6 +2706,7 @@ class BaseLLMHTTPHandler:
client=client if isinstance(client, AsyncHTTPHandler) else None,
fake_stream=fake_stream,
litellm_metadata=litellm_metadata,
api_key=api_key,
)
if client is None or not isinstance(client, HTTPHandler):
@ -2715,7 +2717,7 @@ class BaseLLMHTTPHandler:
sync_httpx_client = client
headers = image_generation_provider_config.validate_environment(
api_key=litellm_params.get("api_key", None),
api_key=api_key,
headers=image_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,
@ -2798,6 +2800,7 @@ class BaseLLMHTTPHandler:
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
api_key: Optional[str] = None,
) -> ImageResponse:
"""
Async version of the image generation handler.
@ -2812,7 +2815,7 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
headers = image_generation_provider_config.validate_environment(
api_key=litellm_params.get("api_key", None),
api_key=api_key,
headers=image_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,

14
litellm/llms/deepinfra/chat/transformation.py
View file

@ -12,6 +12,9 @@ class DeepInfraConfig(OpenAIGPTConfig):
The class `DeepInfra` provides configuration for the DeepInfra's Chat Completions API interface. Below are the parameters:
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "deepinfra"
frequency_penalty: Optional[int] = None
function_call: Optional[Union[str, dict]] = None
@ -53,7 +56,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
return super().get_config()
def get_supported_openai_params(self, model: str):
return [
supported_openai_params = [
"stream",
"frequency_penalty",
"function_call",
@ -68,9 +71,16 @@ class DeepInfraConfig(OpenAIGPTConfig):
"top_p",
"response_format",
"tools",
"tool_choice",
"tool_choice"
]
if litellm.supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
):
supported_openai_params.append("reasoning_effort")
return supported_openai_params
def map_openai_params(
self,
non_default_params: dict,

24
litellm/llms/github_copilot/chat/transformation.py
View file

@ -81,6 +81,30 @@ class GithubCopilotConfig(OpenAIConfig):
return validated_headers
def get_supported_openai_params(self, model: str) -> list:
"""
Get supported OpenAI parameters for GitHub Copilot.
For Claude models that support extended thinking (Claude 4 family and Claude 3-7), includes thinking and reasoning_effort parameters.
For other models, returns standard OpenAI parameters (which may include reasoning_effort for o-series models).
"""
from litellm.utils import supports_reasoning
# Get base OpenAI parameters
base_params = super().get_supported_openai_params(model)
# Add Claude-specific parameters for models that support extended thinking
if "claude" in model.lower() and supports_reasoning(
model=model.lower(),
):
if "thinking" not in base_params:
base_params.append("thinking")
# reasoning_effort is not included by parent for Claude models, so add it
if "reasoning_effort" not in base_params:
base_params.append("reasoning_effort")
return base_params
def _determine_initiator(self, messages: List[AllMessageValues]) -> str:
"""
Determine if request is user or agent initiated based on message roles.

1
litellm/llms/github_copilot/common_utils.py
View file

@ -28,7 +28,6 @@ class GithubCopilotError(BaseLLMException):
)
class GetDeviceCodeError(GithubCopilotError):
pass

45
litellm/llms/oci/chat/transformation.py
View file

@ -92,6 +92,22 @@ def load_private_key_from_str(key_str: str):
return key
def load_private_key_from_file(file_path: str):
"""Loads a private key from a file path"""
try:
with open(file_path, "r", encoding="utf-8") as f:
key_str = f.read().strip()
except FileNotFoundError:
raise FileNotFoundError(f"Private key file not found: {file_path}")
except OSError as e:
raise OSError(f"Failed to read private key file '{file_path}': {e}") from e
if not key_str:
raise ValueError(f"Private key file is empty: {file_path}")
return load_private_key_from_str(key_str)
def get_vendor_from_model(model: str) -> OCIVendors:
"""
Extracts the vendor from the model name.
@ -237,10 +253,17 @@ class OCIChatConfig(BaseConfig):
oci_fingerprint = optional_params.get("oci_fingerprint")
oci_tenancy = optional_params.get("oci_tenancy")
oci_key = optional_params.get("oci_key")
oci_key_file = optional_params.get("oci_key_file")
if not oci_user or not oci_fingerprint or not oci_tenancy or not oci_key:
if (
not oci_user
or not oci_fingerprint
or not oci_tenancy
or not (oci_key or oci_key_file)
):
raise Exception(
"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key"
"Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, "
"and at least one of oci_key or oci_key_file."
)
method = str(optional_params.get("method", "POST")).upper()
@ -283,7 +306,17 @@ class OCIChatConfig(BaseConfig):
"Please install it with: pip install cryptography"
) from e
private_key = load_private_key_from_str(oci_key)
private_key = (
load_private_key_from_str(oci_key)
if oci_key
else load_private_key_from_file(oci_key_file) if oci_key_file else None
)
if private_key is None:
raise Exception(
"Private key is required for OCI authentication. Please provide either oci_key or oci_key_file."
)
signature = private_key.sign(
signing_string.encode("utf-8"),
padding.PKCS1v15(),
@ -334,17 +367,19 @@ class OCIChatConfig(BaseConfig):
oci_fingerprint = optional_params.get("oci_fingerprint")
oci_tenancy = optional_params.get("oci_tenancy")
oci_key = optional_params.get("oci_key")
oci_key_file = optional_params.get("oci_key_file")
oci_compartment_id = optional_params.get("oci_compartment_id")
if (
not oci_user
or not oci_fingerprint
or not oci_tenancy
or not oci_key
or not (oci_key or oci_key_file)
or not oci_compartment_id
):
raise Exception(
"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id"
"Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, "
"and at least one of oci_key or oci_key_file."
)
if not api_base:

112
litellm/llms/vercel_ai_gateway/chat/transformation.py Normal file
View file

@ -0,0 +1,112 @@
"""
Support for OpenAI's `/v1/chat/completions` endpoint.
Calls done in OpenAI/openai.py as Vercel AI Gateway is openai-compatible.
Docs: https://vercel.com/docs/ai-gateway
"""
from typing import List, Optional, Tuple, Union
import httpx
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues
from litellm.secret_managers.main import get_secret_str
import litellm
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import VercelAIGatewayException
class VercelAIGatewayConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> Optional[str]:
return "vercel_ai_gateway"
def get_supported_openai_params(self, model: str) -> list:
base_params = super().get_supported_openai_params(model)
if "extra_body" not in base_params:
base_params.append("extra_body")
return base_params
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
api_base = (
api_base
or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
or "https://ai-gateway.vercel.sh/v1"
)
user_api_key = (
api_key
or get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
or get_secret_str("VERCEL_OIDC_TOKEN")
)
return api_base, user_api_key
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
mapped_openai_params = super().map_openai_params(
non_default_params, optional_params, model, drop_params
)
# Vercel AI Gateway-only parameters
extra_body = {}
provider_options = non_default_params.pop("providerOptions", None)
if provider_options is not None:
extra_body["providerOptions"] = provider_options
mapped_openai_params["extra_body"] = extra_body # openai client supports `extra_body` param
return mapped_openai_params
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the overall request to be sent to the API.
Returns:
dict: The transformed request. Sent as the body of the API call.
"""
return super().transform_request(
model, messages, optional_params, litellm_params, headers
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:
return VercelAIGatewayException(
message=error_message,
status_code=status_code,
headers=headers,
)
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> List[str]:
api_base, _ = self._get_openai_compatible_provider_info(api_base, api_key)
if api_base is None:
api_base = "https://ai-gateway.vercel.sh/v1"
models_url = f"{api_base}/models"
response = litellm.module_level_client.get(url=models_url)
if response.status_code != 200:
raise Exception(f"Failed to get models: {response.text}")
models = response.json()["data"]
return [model["id"] for model in models]

5
litellm/llms/vercel_ai_gateway/common_utils.py Normal file
View file

@ -0,0 +1,5 @@
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class VercelAIGatewayException(BaseLLMException):
pass

4
litellm/llms/vertex_ai/common_utils.py
View file

@ -254,9 +254,7 @@ def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
item["title"] = title
if description:
item["description"] = description
return {"anyOf": any_of}
else:
return schema_dict
return {"anyOf": any_of}
return schema_dict

15
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -35,6 +35,7 @@ from litellm.types.llms.openai import (
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionTextObject,
ChatCompletionUserMessage,
)
from litellm.types.llms.vertex_ai import *
from litellm.types.llms.vertex_ai import (
@ -475,6 +476,13 @@ async def async_transform_request_body(
optional_params=optional_params,
)
def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
"""
Returns a default user message when a "system" message is passed in gemini fails.
This adds a blank user message to the messages list, to ensure that gemini doesn't fail the request.
"""
return ChatCompletionUserMessage(content=".", role="user")
def _transform_system_message(
supports_system_message: bool, messages: List[AllMessageValues]
@ -510,6 +518,13 @@ def _transform_system_message(
messages.pop(idx)
if len(system_content_blocks) > 0:
#########################################################
# If no messages are passed in, add a blank user message
# Relevant Issue - https://github.com/BerriAI/litellm/issues/13769
#########################################################
if len(messages) == 0:
messages.append(_default_user_message_when_system_message_passed())
#########################################################
return SystemInstructions(parts=system_content_blocks), messages
return None, messages

161
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -46,6 +46,7 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParamFunctionChunk,
ImageURLObject,
OpenAIChatCompletionFinishReason,
)
from litellm.types.llms.vertex_ai import (
@ -89,11 +90,12 @@ from .transformation import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponseStream
from litellm.types.utils import ModelResponseStream, StreamingChoices
LoggingClass = LiteLLMLoggingObj
else:
LoggingClass = Any
StreamingChoices = Any
class VertexAIBaseConfig:
@ -461,7 +463,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = True
if thinking_budget is not None and isinstance(thinking_budget, int):
params["thinkingBudget"] = thinking_budget
return params
def map_response_modalities(self, value: list) -> list:
@ -774,8 +775,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif "inlineData" in part:
mime_type = part["inlineData"]["mimeType"]
data = part["inlineData"]["data"]
# Check if inline data is audio - if so, exclude from text content
if mime_type.startswith("audio/"):
# Check if inline data is audio or image - if so, exclude from text content
# Images and audio are now handled separately in their respective response fields
if mime_type.startswith("audio/") or mime_type.startswith("image/"):
continue
_content_str += "data:{};base64,{}".format(mime_type, data)
@ -790,6 +792,23 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
content_str += _content_str
return content_str, reasoning_content_str
def _extract_image_response_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[ImageURLObject]:
"""Extract image response from parts if present"""
for part in parts:
if "inlineData" in part:
mime_type = part["inlineData"]["mimeType"]
data = part["inlineData"]["data"]
if mime_type.startswith("image/"):
# Convert base64 data to data URI format
data_uri = f"data:{mime_type};base64,{data}"
return ImageURLObject(
url=data_uri,
detail="auto"
)
return None
def _extract_audio_response_from_parts(
self, parts: List[HttpxPartType]
@ -1108,6 +1127,75 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif web_search_queries:
web_search_requests = len(grounding_metadata)
return web_search_requests
@staticmethod
def _create_streaming_choice(
chat_completion_message: ChatCompletionResponseMessage,
candidate: Candidates,
idx: int,
tools: Optional[List[ChatCompletionToolCallChunk]],
functions: Optional[ChatCompletionToolCallFunctionChunk],
chat_completion_logprobs: Optional[ChoiceLogprobs],
image_response: Optional[ImageURLObject],
) -> StreamingChoices:
"""
Helper method to create a streaming choice object for Vertex AI
"""
from litellm.types.utils import Delta, StreamingChoices
# create a streaming choice object
choice = StreamingChoices(
finish_reason=VertexGeminiConfig._check_finish_reason(
chat_completion_message, candidate.get("finishReason")
),
index=candidate.get("index", idx),
delta=Delta(
content=chat_completion_message.get("content"),
reasoning_content=chat_completion_message.get(
"reasoning_content"
),
tool_calls=tools,
image=image_response,
function_call=functions,
),
logprobs=chat_completion_logprobs,
enhancements=None,
)
return choice
@staticmethod
def _extract_candidate_metadata(candidate: Candidates) -> Tuple[List[dict], List[dict], List, List]:
"""
Extract metadata from a single candidate response.
Returns:
grounding_metadata: List[dict]
url_context_metadata: List[dict]
safety_ratings: List
citation_metadata: List
"""
grounding_metadata: List[dict] = []
url_context_metadata: List[dict] = []
safety_ratings: List = []
citation_metadata: List = []
if "groundingMetadata" in candidate:
if isinstance(candidate["groundingMetadata"], list):
grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore
else:
grounding_metadata.append(candidate["groundingMetadata"]) # type: ignore
if "safetyRatings" in candidate:
safety_ratings.append(candidate["safetyRatings"])
if "citationMetadata" in candidate:
citation_metadata.append(candidate["citationMetadata"])
if "urlContextMetadata" in candidate:
# Add URL context metadata to grounding metadata
url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata
@staticmethod
def _process_candidates(
@ -1131,6 +1219,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
grounding_metadata: List[dict] = []
url_context_metadata: List[dict] = []
image_response: Optional[ImageURLObject] = None
safety_ratings: List = []
citation_metadata: List = []
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
@ -1143,21 +1232,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if "content" not in candidate:
continue
if "groundingMetadata" in candidate:
if isinstance(candidate["groundingMetadata"], list):
grounding_metadata.extend(candidate["groundingMetadata"]) # type: ignore
else:
grounding_metadata.append(candidate["groundingMetadata"]) # type: ignore
if "safetyRatings" in candidate:
safety_ratings.append(candidate["safetyRatings"])
if "citationMetadata" in candidate:
citation_metadata.append(candidate["citationMetadata"])
if "urlContextMetadata" in candidate:
# Add URL context metadata to grounding metadata
url_context_metadata.append(cast(dict, candidate["urlContextMetadata"]))
# Extract metadata using helper function
(
candidate_grounding_metadata,
candidate_url_context_metadata,
candidate_safety_ratings,
candidate_citation_metadata,
) = VertexGeminiConfig._extract_candidate_metadata(candidate)
grounding_metadata.extend(candidate_grounding_metadata)
url_context_metadata.extend(candidate_url_context_metadata)
safety_ratings.extend(candidate_safety_ratings)
citation_metadata.extend(candidate_citation_metadata)
if "parts" in candidate["content"]:
(
@ -1172,18 +1258,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts=candidate["content"]["parts"]
)
)
image_response = (
VertexGeminiConfig()._extract_image_response_from_parts(
parts=candidate["content"]["parts"]
)
)
if audio_response is not None:
cast(Dict[str, Any], chat_completion_message)[
"audio"
] = audio_response
chat_completion_message["content"] = None # OpenAI spec
elif content is not None:
if image_response is not None:
# Handle image response - combine with text content into structured format
cast(Dict[str, Any], chat_completion_message)["image"] = image_response
if content is not None:
chat_completion_message["content"] = content
if reasoning_content is not None:
chat_completion_message["reasoning_content"] = reasoning_content
(
functions,
tools,
@ -1206,24 +1299,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
chat_completion_message["function_call"] = functions
if isinstance(model_response, ModelResponseStream):
from litellm.types.utils import Delta, StreamingChoices
# create a streaming choice object
choice = StreamingChoices(
finish_reason=VertexGeminiConfig._check_finish_reason(
chat_completion_message, candidate.get("finishReason")
),
index=candidate.get("index", idx),
delta=Delta(
content=chat_completion_message.get("content"),
reasoning_content=chat_completion_message.get(
"reasoning_content"
),
tool_calls=tools,
function_call=functions,
),
logprobs=chat_completion_logprobs,
enhancements=None,
choice = VertexGeminiConfig._create_streaming_choice(
chat_completion_message=chat_completion_message,
candidate=candidate,
idx=idx,
tools=tools,
functions=functions,
chat_completion_logprobs=chat_completion_logprobs,
image_response=image_response
)
model_response.choices.append(choice)
elif isinstance(model_response, ModelResponse):

27
litellm/llms/xai/chat/transformation.py
View file

@ -31,7 +31,6 @@ class XAIChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list:
base_openai_params = [
"frequency_penalty",
"logit_bias",
"logprobs",
"max_tokens",
@ -50,8 +49,22 @@ class XAIChatConfig(OpenAIGPTConfig):
"web_search_options",
]
# for some reason, grok-3-mini does not support stop tokens
#########################################################
# stop tokens check
#########################################################
if self._supports_stop_reason(model):
base_openai_params.append("stop")
#########################################################
# frequency penalty check
#########################################################
if self._supports_frequency_penalty(model):
base_openai_params.append("frequency_penalty")
#########################################################
# reasoning check
#########################################################
try:
if litellm.supports_reasoning(
model=model, custom_llm_provider=self.custom_llm_provider
@ -68,6 +81,18 @@ class XAIChatConfig(OpenAIGPTConfig):
elif "grok-4" in model:
return False
return True
def _supports_frequency_penalty(self, model: str) -> bool:
"""
From manual testing grok-4 does not support `frequency_penalty`
When sent the model fails from xAI API
"""
if "grok-4" in model:
return False
if "grok-code-fast" in model:
return False
return True
def map_openai_params(
self,

88
litellm/main.py
View file

@ -2173,8 +2173,18 @@ def completion( # type: ignore # noqa: PLR0915
or "https://api.anthropic.com/v1/complete"
)
if api_base is not None and not api_base.endswith("/v1/complete"):
# Check if we should disable automatic URL suffix appending
disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
if (
api_base is not None
and not disable_url_suffix
and not api_base.endswith("/v1/complete")
):
api_base += "/v1/complete"
elif disable_url_suffix:
verbose_logger.debug(
"LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix"
)
response = base_llm_http_handler.completion(
model=model,
@ -2210,8 +2220,18 @@ def completion( # type: ignore # noqa: PLR0915
or "https://api.anthropic.com/v1/messages"
)
if api_base is not None and not api_base.endswith("/v1/messages"):
# Check if we should disable automatic URL suffix appending
disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX")
if (
api_base is not None
and not disable_url_suffix
and not api_base.endswith("/v1/messages")
):
api_base += "/v1/messages"
elif disable_url_suffix:
verbose_logger.debug(
"LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix"
)
response = anthropic_chat_completions.completion(
model=model,
@ -2648,6 +2668,70 @@ def completion( # type: ignore # noqa: PLR0915
logging.post_call(
input=messages, api_key=openai.api_key, original_response=response
)
elif custom_llm_provider == "vercel_ai_gateway":
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERCEL_AI_GATEWAY_API_BASE")
or "https://ai-gateway.vercel.sh/v1"
)
api_key = (
api_key
or litellm.api_key
or get_secret("VERCEL_AI_GATEWAY_API_KEY")
)
vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM"
vercel_headers = {
"http-referer": vercel_site_url,
"x-title": vercel_app_name,
}
_headers = headers or litellm.headers
if _headers:
vercel_headers.update(_headers)
headers = vercel_headers
## Load Config
config = litellm.VercelAIGatewayConfig.get_config()
for k, v in config.items():
if k == "extra_body":
# we use openai 'extra_body' to pass vercel specific params - providerOptions
if "extra_body" in optional_params:
optional_params[k].update(v)
else:
optional_params[k] = v
elif k not in optional_params:
optional_params[k] = v
data = {"model": model, "messages": messages, **optional_params}
## COMPLETION CALL
response = base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
acompletion=acompletion,
api_base=api_base,
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
custom_llm_provider="vercel_ai_gateway",
timeout=timeout,
headers=headers,
encoding=encoding,
api_key=api_key,
logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements
client=client,
)
## LOGGING
logging.post_call(
input=messages, api_key=openai.api_key, original_response=response
)
elif (
custom_llm_provider == "together_ai"
or ("togethercomputer" in model)

2764
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

89
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,5 +1,5 @@
import importlib
from typing import Optional, Dict
from typing import Dict, List, Optional
from fastapi import APIRouter, Depends, Query, Request
@ -21,9 +21,10 @@ router = APIRouter(
)
if MCP_AVAILABLE:
from litellm.experimental_mcp_client.client import MCPTool
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
_convert_protocol_version_to_enum,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.server import (
ListMCPToolsRestAPIResponseObject,
@ -100,7 +101,9 @@ if MCP_AVAILABLE:
"message": "Successfully retrieved tools"
}
"""
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import MCPRequestHandler
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
try:
# Extract auth headers from request
@ -172,10 +175,11 @@ if MCP_AVAILABLE:
"""
REST API to call a specific MCP tool with the provided arguments
"""
from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from fastapi import HTTPException
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
try:
data = await request.json()
data = await add_litellm_data_to_request(
@ -230,13 +234,19 @@ if MCP_AVAILABLE:
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
)
@router.post("/test/connection")
async def test_connection(
request: NewMCPServerRequest,
):
async def _execute_with_mcp_client(request: NewMCPServerRequest, operation):
"""
Test if we can connect to the provided MCP server before adding it
Common helper to create MCP client, execute operation, and ensure proper cleanup.
Args:
request: MCP server configuration
operation: Async function that takes a client and returns the operation result
Returns:
Operation result or error response
"""
client = None
try:
client = global_mcp_server_manager._create_mcp_client(
server=MCPServer(
@ -250,12 +260,31 @@ if MCP_AVAILABLE:
),
mcp_auth_header=None,
)
await client.connect()
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in test_connection: {e}", exc_info=True)
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
return {"status": "error", "message": "An internal error has occurred."}
return {"status": "ok"}
finally:
# Ensure client is properly disconnected before response is sent
if client is not None:
try:
await client.disconnect()
except Exception as e:
verbose_logger.warning(f"Error disconnecting MCP client: {e}")
@router.post("/test/connection")
async def test_connection(
request: NewMCPServerRequest,
):
"""
Test if we can connect to the provided MCP server before adding it
"""
async def _test_connection_operation(client):
await client.connect()
return {"status": "ok"}
return await _execute_with_mcp_client(request, _test_connection_operation)
@router.post("/test/tools/list")
@ -266,25 +295,13 @@ if MCP_AVAILABLE:
"""
Preview tools available from MCP server before adding it
"""
try:
client = global_mcp_server_manager._create_mcp_client(
server=MCPServer(
server_id=request.server_id or "",
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
spec_version=_convert_protocol_version_to_enum(request.spec_version),
auth_type=request.auth_type,
mcp_info=request.mcp_info,
),
mcp_auth_header=None,
)
list_tools_result = await client.list_tools()
except Exception as e:
verbose_logger.error(f"Error in test_tools_list: {e}", exc_info=True)
return {"status": "error", "message": "An internal error has occurred."}
return {
"tools": list_tools_result,
"error": None,
"message": "Successfully retrieved tools"
}
async def _list_tools_operation(client):
list_tools_result: List[MCPTool] = await client.list_tools()
model_dumped_tools: List[dict] = [tool.model_dump() for tool in list_tools_result]
return {
"tools": model_dumped_tools,
"error": None,
"message": "Successfully retrieved tools"
}
return await _execute_with_mcp_client(request, _list_tools_operation)

7
litellm/proxy/_types.py
View file

@ -326,6 +326,12 @@ class LiteLLMRoutes(enum.Enum):
"/mistral",
]
#########################################################
# e.g /vllm/*, anthropic/*, etc.
# allows using /anthropic/v1/messages, /vllm/v1/chat/completions, etc.
#########################################################
passthrough_routes_wildcard = [f"{route}/*" for route in mapped_pass_through_routes]
anthropic_routes = [
"/v1/messages",
]
@ -356,6 +362,7 @@ class LiteLLMRoutes(enum.Enum):
openai_routes
+ anthropic_routes
+ mapped_pass_through_routes
+ passthrough_routes_wildcard
+ apply_guardrail_routes
+ mcp_routes
)

43
litellm/proxy/auth/handle_jwt.py
View file

@ -414,7 +414,16 @@ class JWTHandler:
if cached_keys is None:
response = await self.http_handler.get(key_url)
response_json = response.json()
try:
response_json = response.json()
except Exception as e:
verbose_proxy_logger.error(
f"Error parsing response: {e}. Original Response: {response.text}"
)
raise Exception(
f"Error parsing response: {e}. Check server logs for original response."
)
if "keys" in response_json:
keys: JWKKeyValue = response.json()["keys"]
else:
@ -904,7 +913,7 @@ class JWTAuthManager:
if end_user_id
else None
)
team_membership_object: Optional[LiteLLM_TeamMembership] = None
if user_id and team_id:
team_membership_object = (
@ -1145,19 +1154,21 @@ class JWTAuthManager:
)
# Get other objects
user_object, org_object, end_user_object, team_membership_object = await JWTAuthManager.get_objects(
user_id=user_id,
user_email=user_email,
org_id=org_id,
end_user_id=end_user_id,
team_id=team_id,
valid_user_email=valid_user_email,
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
route=route,
user_object, org_object, end_user_object, team_membership_object = (
await JWTAuthManager.get_objects(
user_id=user_id,
user_email=user_email,
org_id=org_id,
end_user_id=end_user_id,
team_id=team_id,
valid_user_email=valid_user_email,
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
route=route,
)
)
await JWTAuthManager.sync_user_role_and_teams(
@ -1186,8 +1197,6 @@ class JWTAuthManager:
is_proxy_admin = True
else:
is_proxy_admin = False
return JWTAuthBuilderResult(
is_proxy_admin=is_proxy_admin,

35
litellm/proxy/auth/route_checks.py
View file

@ -300,6 +300,13 @@ class RouteChecks:
if re.match(pattern, route):
return True
return False
@staticmethod
def _is_wildcard_pattern(pattern: str) -> bool:
"""
Check if pattern is a wildcard pattern
"""
return pattern.endswith("*")
@staticmethod
def _route_matches_wildcard_pattern(route: str, pattern: str) -> bool:
@ -342,10 +349,34 @@ class RouteChecks:
Returns:
bool: True if route is allowed, False otherwise
"""
return route in allowed_routes or any( # Check exact match
#########################################################
# exact match route is in allowed_routes
#########################################################
if route in allowed_routes:
return True
#########################################################
# wildcard match route is in allowed_routes
# e.g calling /anthropic/v1/messages is allowed if allowed_routes has /anthropic/*
#########################################################
wildcard_allowed_routes = [route for route in allowed_routes if RouteChecks._is_wildcard_pattern(pattern=route)]
for allowed_route in wildcard_allowed_routes:
if RouteChecks._route_matches_wildcard_pattern(route=route, pattern=allowed_route):
return True
#########################################################
# pattern match route is in allowed_routes
# pattern: "/threads/{thread_id}"
# route: "/threads/thread_49EIN5QF32s4mH20M7GFKdlZ"
# returns: True
#########################################################
if any( # Check pattern match
RouteChecks._route_matches_pattern(route=route, pattern=allowed_route)
for allowed_route in allowed_routes
) # Check pattern match
):
return True
return False
@staticmethod
def _is_assistants_api_request(request: Request) -> bool:

14
litellm/proxy/common_request_processing.py
View file

@ -1,7 +1,7 @@
import asyncio
import json
import logging
import traceback
import uuid
from datetime import datetime
from typing import (
TYPE_CHECKING,
@ -14,6 +14,7 @@ from typing import (
Union,
)
import fastuuid as uuid
import httpx
import orjson
from fastapi import HTTPException, Request, status
@ -385,11 +386,12 @@ class ProxyBaseLLMRequestProcessing:
"""
Common request processing logic for both chat completions and responses API endpoints
"""
verbose_proxy_logger.debug(
"Request received by LiteLLM:\n{}".format(
json.dumps(self.data, indent=4, default=str)
),
)
if verbose_proxy_logger.isEnabledFor(logging.DEBUG):
verbose_proxy_logger.debug(
"Request received by LiteLLM:\n{}".format(
json.dumps(self.data, indent=4, default=str)
),
)
self.data, logging_obj = await self.common_processing_pre_call_logic(
request=request,

361
litellm/proxy/guardrails/guardrail_hooks/pangea/pangea.py
View file

@ -1,6 +1,6 @@
# litellm/proxy/guardrails/guardrail_hooks/pangea.py
import os
from typing import TYPE_CHECKING, Any, Optional, Protocol, Type
from typing import TYPE_CHECKING, Any, Optional, Type
from fastapi import HTTPException
@ -19,7 +19,7 @@ from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import LLMResponseTypes, ModelResponse, TextCompletionResponse
from litellm.types.utils import Choices, LLMResponseTypes, ModelResponse, TextCompletionResponse
if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
@ -31,14 +31,6 @@ class PangeaGuardrailMissingSecrets(Exception):
pass
class _Transformer(Protocol):
def get_messages(self) -> list[dict]: # noqa: E704
...
def update_original_body(self, prompt_messages: list[dict]) -> Any: # noqa: E704
...
class _TextCompletionRequest:
def __init__(self, body):
self.body = body
@ -53,109 +45,6 @@ class _TextCompletionRequest:
return self.body
class _TextCompletionResponse:
def __init__(self, body):
self.body = body
def get_messages(self) -> list[dict]:
messages = []
for choice in self.body["choices"]:
messages.append({"role": "assistant", "content": choice["text"]})
return messages
def update_original_body(self, prompt_messages: list[dict]) -> Any:
assert len(prompt_messages) == len(self.body["choices"])
for choice, prompt_message in zip(self.body["choices"], prompt_messages):
choice["text"] = prompt_message["content"]
return self.body
class _ChatCompletionRequest:
def __init__(self, body):
self.body = body
def get_messages(self) -> list[dict]:
messages = []
for message in self.body["messages"]:
role = message["role"]
content = message["content"]
if isinstance(content, str):
messages.append({"role": role, "content": content})
if isinstance(content, list):
for content_part in content:
if content_part["type"] == "text":
messages.append({"role": role, "content": content_part["text"]})
return messages
def update_original_body(self, prompt_messages: list[dict]) -> Any:
count = 0
for message in self.body["messages"]:
content = message["content"]
if isinstance(content, str):
message["content"] = prompt_messages[count]["content"]
count += 1
if isinstance(content, list):
for content_part in content:
if content_part["type"] == "text":
content_part["text"] = prompt_messages[count]["content"]
count += 1
assert len(prompt_messages) == count
return self.body
class _ChatCompletionResponse:
def __init__(self, body):
self.body = body
def get_messages(self) -> list[dict]:
messages = []
for choice in self.body["choices"]:
messages.append(
{
"role": choice["message"]["role"],
"content": choice["message"]["content"],
}
)
return messages
def update_original_body(self, prompt_messages: list[dict]) -> Any:
assert len(prompt_messages) == len(self.body["choices"])
for choice, prompt_message in zip(self.body["choices"], prompt_messages):
choice["message"]["content"] = prompt_message["content"]
return self.body
def _get_transformer_for_request(body, call_type) -> Optional[_Transformer]:
match call_type:
case "text_completion" | "atext_completion":
return _TextCompletionRequest(body)
case "completion" | "acompletion":
return _ChatCompletionRequest(body)
return None
def _get_transformer_for_response(body) -> Optional[_Transformer]:
match body:
case TextCompletionResponse():
return _TextCompletionResponse(body)
case ModelResponse():
return _ChatCompletionResponse(body)
return None
class PangeaHandler(CustomGuardrail):
"""
Pangea AI Guardrail handler to interact with the Pangea AI Guard service.
@ -200,7 +89,6 @@ class PangeaHandler(CustomGuardrail):
)
self.pangea_input_recipe = pangea_input_recipe
self.pangea_output_recipe = pangea_output_recipe
self.guardrail_endpoint = f"{self.api_base}/v1/text/guard"
# Pass relevant kwargs to the parent class
super().__init__(guardrail_name=guardrail_name, **kwargs)
@ -208,7 +96,9 @@ class PangeaHandler(CustomGuardrail):
f"Initialized Pangea Guardrail: name={guardrail_name}, recipe={pangea_input_recipe}, api_base={self.api_base}"
)
async def _call_pangea_guard(self, payload: dict, hook_name: str) -> dict:
async def _call_pangea_ai_guard(
self, api: str, payload: dict, hook_name: str
) -> dict:
"""
Makes the API call to the Pangea AI Guard endpoint.
The function itself will raise an error in the case that a response
@ -216,6 +106,7 @@ class PangeaHandler(CustomGuardrail):
should act on.
Args:
api (str): Which API to use (text/guard or v1beta/guard)
payload (dict): The request payload.
request_data (dict): Original request data (used for logging/headers).
hook_name (str): Name of the hook calling this function (for logging).
@ -227,62 +118,84 @@ class PangeaHandler(CustomGuardrail):
Returns:
list[dict]: The original response body
"""
endpoint = f"{self.api_base}/{api}"
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
try:
verbose_proxy_logger.debug(
f"Pangea Guardrail ({hook_name}): Calling endpoint {self.guardrail_endpoint} with payload: {payload}"
)
response = await self.async_handler.post(
url=self.guardrail_endpoint, json=payload, headers=headers
)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
result = response.json()
verbose_proxy_logger.debug(
f"Pangea Guardrail ({hook_name}): Received response: {result}"
verbose_proxy_logger.debug(
f"Pangea Guardrail ({hook_name}): Calling endpoint {endpoint} with payload: {payload}"
)
response = await self.async_handler.post(
url=endpoint, json=payload, headers=headers
)
response.raise_for_status()
result = response.json()
if result.get("result", {}).get("blocked"):
verbose_proxy_logger.warning(
f"Pangea Guardrail ({hook_name}): Request blocked. Response: {result}"
)
# Check if the request was blocked
if result.get("result", {}).get("blocked") is True:
verbose_proxy_logger.warning(
f"Pangea Guardrail ({hook_name}): Request blocked. Response: {result}"
)
raise HTTPException(
status_code=400, # Bad Request, indicating violation
detail={
"error": "Violated Pangea guardrail policy",
"guardrail_name": self.guardrail_name,
"pangea_response": result.get("result"),
},
)
else:
verbose_proxy_logger.info(
f"Pangea Guardrail ({hook_name}): Request passed. Response: {result.get('result', {}).get('detectors')}"
)
return result
except HTTPException as e:
# Re-raise HTTPException if it's the one we raised for blocking
raise e
except Exception as e:
verbose_proxy_logger.error(
f"Pangea Guardrail ({hook_name}): Error calling API: {e}. Response text: {getattr(e, 'response', None) and getattr(e.response, 'text', None)}" # type: ignore
)
# Decide if you want to block by default on error, or allow through
# Raising an exception here will block the request.
# To allow through on error, you might just log and return.
raise HTTPException(
status_code=500,
status_code=400, # Bad Request, indicating violation
detail={
"error": "Error communicating with Pangea Guardrail",
"error": "Violated Pangea guardrail policy",
"guardrail_name": self.guardrail_name,
"exception": str(e),
},
) from e
)
verbose_proxy_logger.info(
f"Pangea Guardrail ({hook_name}): Request passed. Response: {result.get('result', {}).get('detectors')}"
)
return result
async def _async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: str
):
transformer = None
messages: Any = None
if call_type == "text_completion" or call_type == "atext_completion":
transformer = _TextCompletionRequest(data)
messages = transformer.get_messages()
else:
messages = data.get("messages")
ai_guard_payload = {
"debug": False,
"input": {
"messages": messages, # type: ignore
"tools": data.get("tools")
},
"event_type": "input",
}
if self.pangea_input_recipe:
ai_guard_payload["recipe"] = self.pangea_input_recipe
ai_guard_response = await self._call_pangea_ai_guard(
"v1beta/guard", ai_guard_payload, "async_pre_call_hook"
)
add_guardrail_to_applied_guardrails_header(
request_data=data, guardrail_name=self.guardrail_name
)
if not ai_guard_response.get("result", {}).get("transformed"):
return
output = ai_guard_response.get("result", {}).get("output", {})
if call_type == "text_completion" or call_type == "atext_completion":
data = transformer.update_original_body(output["messages"]) # type: ignore
else:
data["messages"] = output["messages"]
return data
@log_guardrail_information
async def async_pre_call_hook(
@ -299,50 +212,75 @@ class PangeaHandler(CustomGuardrail):
)
return data
transformer = _get_transformer_for_request(data, call_type)
if not transformer:
verbose_proxy_logger.warning(
f"Pangea Guardrail (async_pre_call_hook): Skipping guardrail {self.guardrail_name}"
f" because we cannot determine type of request: call_type '{call_type}'"
)
return
messages = transformer.get_messages()
if not messages:
verbose_proxy_logger.warning(
f"Pangea Guardrail (async_pre_call_hook): Skipping guardrail {self.guardrail_name}"
" because messages is empty."
)
return
ai_guard_payload = {
"debug": False, # Or make this configurable if needed
"messages": messages,
}
if self.pangea_input_recipe:
ai_guard_payload["recipe"] = self.pangea_input_recipe
ai_guard_response = await self._call_pangea_guard(
ai_guard_payload, "async_pre_call_hook"
)
# Add guardrail name to header if passed
add_guardrail_to_applied_guardrails_header(
request_data=data, guardrail_name=self.guardrail_name
)
prompt_messages = ai_guard_response.get("result", {}).get("prompt_messages", [])
try:
return transformer.update_original_body(prompt_messages)
return await self._async_pre_call_hook(user_api_key_dict, cache, data, call_type)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail={
"error": "Failed to update original request body",
"error": "Error in Pangea Guardrail",
"guardrail_name": self.guardrail_name,
"exceptions": str(e),
},
}
) from e
async def _async_post_call_success_hook(
self,
data: dict,
user_api_key_dict: UserAPIKeyAuth,
# This union isn't actually correct -- it can get other response types depending on the API called
response: LLMResponseTypes,
):
if isinstance(response, TextCompletionResponse):
# Assume the earlier call type as well
input_messages = _TextCompletionRequest(data).get_messages()
if not isinstance(response, ModelResponse):
return
else:
input_messages = data.get("messages")
if choices := response.get("choices"):
if isinstance(choices, list):
serialized_choices = []
for c in choices:
if isinstance(c, Choices):
try:
serialized_choices.append(c.model_dump())
except Exception:
serialized_choices.append(c.dict())
else:
serialized_choices.append(c)
choices = serialized_choices
ai_guard_payload = {
"debug": False,
"input": {
"messages": input_messages,
"tools": data.get("tools"),
"choices": choices,
},
"event_type": "output",
}
if self.pangea_output_recipe:
ai_guard_payload["recipe"] = self.pangea_output_recipe
ai_guard_response = await self._call_pangea_ai_guard(
"v1beta/guard", ai_guard_payload, "async_pre_call_hook"
)
add_guardrail_to_applied_guardrails_header(
request_data=data, guardrail_name=self.guardrail_name
)
if not ai_guard_response.get("result", {}).get("transformed"):
return
output = ai_guard_response.get("result", {}).get("output", {})
response.choices = output["choices"]
return response
@log_guardrail_information
async def async_post_call_success_hook(
self,
@ -365,39 +303,18 @@ class PangeaHandler(CustomGuardrail):
f"Pangea Guardrail (async_pre_call_hook): Guardrail is disabled {self.guardrail_name}."
)
return data
transformer = _get_transformer_for_response(response)
if not transformer:
verbose_proxy_logger.warning(
f"Pangea Guardrail (async_post_call_success_hook): Skipping guardrail {self.guardrail_name}"
" because we cannot determine type of request"
)
return
messages = transformer.get_messages()
verbose_proxy_logger.warning(f"GOT MESSAGES: {messages}")
ai_guard_payload = {
"debug": False, # Or make this configurable if needed
"messages": messages,
}
if self.pangea_output_recipe:
ai_guard_payload["recipe"] = self.pangea_output_recipe
ai_guard_response = await self._call_pangea_guard(
ai_guard_payload, "post_call_success_hook"
)
prompt_messages = ai_guard_response.get("result", {}).get("prompt_messages", [])
try:
return transformer.update_original_body(prompt_messages)
return await self._async_post_call_success_hook(data, user_api_key_dict, response)
except HTTPException:
raise
except Exception as e:
raise HTTPException(
status_code=500,
detail={
"error": "Failed to update original response body",
"error": "Error in Pangea Guardrail",
"guardrail_name": self.guardrail_name,
"exceptions": str(e),
},
}
) from e
@staticmethod

4
litellm/proxy/litellm_pre_call_utils.py
View file

@ -489,8 +489,10 @@ class LiteLLMProxyRequestSetup:
@staticmethod
def add_key_level_controls(
key_metadata: dict, data: dict, _metadata_variable_name: str
key_metadata: Optional[dict], data: dict, _metadata_variable_name: str
):
if key_metadata is None:
return data
if "cache" in key_metadata:
data["cache"] = {}
if isinstance(key_metadata["cache"], dict):

23
litellm/proxy/proxy_config.yaml
View file

@ -1,23 +1,4 @@
model_list:
- model_name: anthropic/*
- model_name: xai/*
litellm_params:
model: anthropic/*
api_key: os.environ/OPENAI_API_KEY_IJ
- model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
- model_name: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
litellm_params:
model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
router_settings:
fallbacks: [
{"anthropic/claude-opus-4-20250514":
{
"bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
}
}
]
litellm_settings:
callbacks: ["datadog_llm_observability"]
model: xai/*

13
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -10,7 +10,7 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELM_STRING
from litellm.constants import REDACTED_BY_LITELM_STRING, MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
@ -53,7 +53,7 @@ def _get_spend_logs_metadata(
guardrail_information: Optional[StandardLoggingGuardrailInformation] = None,
usage_object: Optional[dict] = None,
model_map_information: Optional[StandardLoggingModelInformation] = None,
cold_storage_object_key: Optional[str] = None
cold_storage_object_key: Optional[str] = None,
) -> SpendLogsMetadata:
if metadata is None:
return SpendLogsMetadata(
@ -101,7 +101,7 @@ def _get_spend_logs_metadata(
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
return clean_metadata
@ -481,10 +481,9 @@ def _sanitize_request_body_for_spend_logs_payload(
) -> dict:
"""
Recursively sanitize request body to prevent logging large base64 strings or other large values.
Truncates strings longer than 1000 characters and handles nested dictionaries.
Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries.
"""
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
MAX_STRING_LENGTH = 1000
if visited is None:
visited = set()
@ -501,8 +500,8 @@ def _sanitize_request_body_for_spend_logs_payload(
elif isinstance(value, list):
return [_sanitize_value(item) for item in value]
elif isinstance(value, str):
if len(value) > MAX_STRING_LENGTH:
return f"{value[:MAX_STRING_LENGTH]}... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} {len(value) - MAX_STRING_LENGTH} chars)"
if len(value) > MAX_STRING_LENGTH_PROMPT_IN_DB:
return f"{value[:MAX_STRING_LENGTH_PROMPT_IN_DB]}... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} {len(value) - MAX_STRING_LENGTH_PROMPT_IN_DB} chars)"
return value
return value

4
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -2,7 +2,7 @@
Handler for transforming responses api requests to litellm.completion requests
"""
from typing import Any, Coroutine, Optional, Union
from typing import Any, Coroutine, Dict, Optional, Union
import litellm
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
@ -30,6 +30,7 @@ class LiteLLMCompletionTransformationHandler:
custom_llm_provider: Optional[str] = None,
_is_async: bool = False,
stream: Optional[bool] = None,
extra_headers: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[
ResponsesAPIResponse,
@ -45,6 +46,7 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=responses_api_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
**kwargs,
)
)

2
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -99,6 +99,7 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: Optional[str] = None,
stream: Optional[bool] = None,
extra_headers: Optional[Dict[str, Any]] = None,
**kwargs,
) -> dict:
"""
@ -126,6 +127,7 @@ class LiteLLMCompletionResponsesConfig:
"web_search_options": web_search_options,
# litellm specific params
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
}
# Responses API `Completed` events require usage, we pass `stream_options` to litellm.completion to include usage

1
litellm/responses/main.py
View file

@ -455,6 +455,7 @@ def responses(
custom_llm_provider=custom_llm_provider,
_is_async=_is_async,
stream=stream,
extra_headers=extra_headers,
**kwargs,
)

2
litellm/router_strategy/lowest_latency.py
View file

@ -108,7 +108,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
if final_value is not None:
final_value = float(final_value)
else:
final_value = response_ms
final_value = response_seconds
if time_to_first_token_response_time is not None:
if isinstance(time_to_first_token_response_time, timedelta):

70
litellm/router_strategy/simple_shuffle.py
View file

@ -9,7 +9,6 @@ import random
from typing import TYPE_CHECKING, Any, Dict, List, Union
from litellm._logging import verbose_router_logger
from litellm.litellm_core_utils.core_helpers import safe_divide
if TYPE_CHECKING:
from litellm.router import Router as _Router
@ -40,57 +39,24 @@ def simple_shuffle(
Dict: A single healthy deployment
"""
############## Check if 'weight' param set for a weighted pick #################
weight = healthy_deployments[0].get("litellm_params").get("weight", None)
if weight is not None:
# use weight-random pick if rpms provided
weights = [m["litellm_params"].get("weight", 0) for m in healthy_deployments]
verbose_router_logger.debug(f"\nweight {weights}")
total_weight = sum(weights)
weights = [safe_divide(weight, total_weight, 0) for weight in weights]
verbose_router_logger.debug(f"\n weights {weights}")
# Perform weighted random pick
selected_index = random.choices(range(len(weights)), weights=weights)[0]
verbose_router_logger.debug(f"\n selected index, {selected_index}")
deployment = healthy_deployments[selected_index]
verbose_router_logger.info(
f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}"
)
return deployment or deployment[0]
############## Check if we can do a RPM/TPM based weighted pick #################
rpm = healthy_deployments[0].get("litellm_params").get("rpm", None)
if rpm is not None:
# use weight-random pick if rpms provided
rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
verbose_router_logger.debug(f"\nrpms {rpms}")
total_rpm = sum(rpms)
weights = [safe_divide(rpm, total_rpm, 0) for rpm in rpms]
verbose_router_logger.debug(f"\n weights {weights}")
# Perform weighted random pick
selected_index = random.choices(range(len(rpms)), weights=weights)[0]
verbose_router_logger.debug(f"\n selected index, {selected_index}")
deployment = healthy_deployments[selected_index]
verbose_router_logger.info(
f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}"
)
return deployment or deployment[0]
############## Check if we can do a RPM/TPM based weighted pick #################
tpm = healthy_deployments[0].get("litellm_params").get("tpm", None)
if tpm is not None:
# use weight-random pick if rpms provided
tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
verbose_router_logger.debug(f"\ntpms {tpms}")
total_tpm = sum(tpms)
weights = [safe_divide(tpm, total_tpm, 0) for tpm in tpms]
verbose_router_logger.debug(f"\n weights {weights}")
# Perform weighted random pick
selected_index = random.choices(range(len(tpms)), weights=weights)[0]
verbose_router_logger.debug(f"\n selected index, {selected_index}")
deployment = healthy_deployments[selected_index]
verbose_router_logger.info(
f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}"
)
return deployment or deployment[0]
############## Check if 'weight' or 'rpm' or 'tpm' param set for a weighted pick #################
for weight_by in ["weight", "rpm", "tpm"]:
weight = healthy_deployments[0].get("litellm_params").get(weight_by, None)
if weight is not None:
weights = [m["litellm_params"].get(weight_by, 0) for m in healthy_deployments]
verbose_router_logger.debug(f"\nweight {weights}")
total_weight = sum(weights)
weights = [weight / total_weight for weight in weights]
verbose_router_logger.debug(f"\n weights {weights} by {weight_by}")
# Perform weighted random pick
selected_index = random.choices(range(len(weights)), weights=weights)[0]
verbose_router_logger.debug(f"\n selected index, {selected_index}")
deployment = healthy_deployments[selected_index]
verbose_router_logger.info(
f"get_available_deployment for model: {model}, Selected deployment: {llm_router_instance.print_deployment(deployment) or deployment[0]} for model: {model}"
)
return deployment or deployment[0]
############## No RPM/TPM passed, we do a random pick #################
item = random.choice(healthy_deployments)

2
litellm/types/llms/openai.py
View file

@ -1038,7 +1038,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
parallel_tool_calls: bool
temperature: Optional[float]
tool_choice: ToolChoice
tools: Union[List[Tool], List[ResponseFunctionToolCall]]
tools: Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]]
top_p: Optional[float]
max_output_tokens: Optional[int]
previous_response_id: Optional[str]

19
litellm/types/utils.py
View file

@ -51,6 +51,7 @@ from .llms.openai import (
ChatCompletionUsageBlock,
FileSearchTool,
FineTuningJob,
ImageURLObject,
OpenAIChatCompletionChunk,
OpenAIFileObject,
OpenAIRealtimeStreamList,
@ -572,6 +573,7 @@ class Message(OpenAIObject):
tool_calls: Optional[List[ChatCompletionMessageToolCall]]
function_call: Optional[FunctionCall]
audio: Optional[ChatCompletionAudioResponse] = None
image: Optional[ImageURLObject] = None
reasoning_content: Optional[str] = None
thinking_blocks: Optional[
List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
@ -588,6 +590,7 @@ class Message(OpenAIObject):
function_call=None,
tool_calls: Optional[list] = None,
audio: Optional[ChatCompletionAudioResponse] = None,
image: Optional[ImageURLObject] = None,
provider_specific_fields: Optional[Dict[str, Any]] = None,
reasoning_content: Optional[str] = None,
thinking_blocks: Optional[
@ -621,6 +624,9 @@ class Message(OpenAIObject):
if audio is not None:
init_values["audio"] = audio
if image is not None:
init_values["image"] = image
if thinking_blocks is not None:
init_values["thinking_blocks"] = thinking_blocks
@ -640,6 +646,10 @@ class Message(OpenAIObject):
# OpenAI compatible APIs like mistral API will raise an error if audio is passed in
if hasattr(self, "audio"):
del self.audio
if image is None:
if hasattr(self, "image"):
del self.image
if annotations is None:
# ensure default response matches OpenAI spec
@ -693,6 +703,7 @@ class Delta(OpenAIObject):
function_call=None,
tool_calls=None,
audio: Optional[ChatCompletionAudioResponse] = None,
image: Optional[ImageURLObject] = None,
reasoning_content: Optional[str] = None,
thinking_blocks: Optional[
List[
@ -710,6 +721,7 @@ class Delta(OpenAIObject):
self.function_call: Optional[Union[FunctionCall, Any]] = None
self.tool_calls: Optional[List[Union[ChatCompletionDeltaToolCall, Any]]] = None
self.audio: Optional[ChatCompletionAudioResponse] = None
self.image: Optional[ImageURLObject] = None
self.annotations: Optional[List[ChatCompletionAnnotation]] = None
if reasoning_content is not None:
@ -729,6 +741,11 @@ class Delta(OpenAIObject):
self.annotations = annotations
else:
del self.annotations
if image is not None:
self.image = image
else:
del self.image
if function_call is not None and isinstance(function_call, dict):
self.function_call = FunctionCall(**function_call)
@ -2334,9 +2351,9 @@ class LlmProviders(str, Enum):
COMETAPI = "cometapi"
OCI = "oci"
AUTO_ROUTER = "auto_router"
VERCEL_AI_GATEWAY = "vercel_ai_gateway"
DOTPROMPT = "dotprompt"
# Create a set of all provider values for quick lookup
LlmProvidersSet = {provider.value for provider in LlmProviders}

20
litellm/utils.py
View file

@ -32,7 +32,6 @@ import textwrap
import threading
import time
import traceback
import uuid
from dataclasses import dataclass, field
from functools import lru_cache, wraps
from importlib import resources
@ -41,6 +40,7 @@ from os.path import abspath, dirname, join
import aiohttp
import dotenv
import fastuuid as uuid
import httpx
import openai
import tiktoken
@ -2352,6 +2352,9 @@ def register_model(model_cost: Union[str, dict]): # noqa: PLR0915
split_string = key.split("/", 1)
if key not in litellm.openrouter_models:
litellm.openrouter_models.add(split_string[1])
elif value.get("litellm_provider") == "vercel_ai_gateway":
if key not in litellm.vercel_ai_gateway_models:
litellm.vercel_ai_gateway_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
if key not in litellm.vertex_text_models:
litellm.vertex_text_models.add(key)
@ -3226,6 +3229,7 @@ def pre_process_optional_params(
and custom_llm_provider != "bedrock"
and custom_llm_provider != "ollama_chat"
and custom_llm_provider != "openrouter"
and custom_llm_provider != "vercel_ai_gateway"
and custom_llm_provider != "nebius"
and custom_llm_provider not in litellm.openai_compatible_providers
):
@ -3902,7 +3906,6 @@ def get_optional_params( # noqa: PLR0915
else False
),
)
elif custom_llm_provider == "watsonx":
optional_params = litellm.IBMWatsonXChatConfig().map_openai_params(
non_default_params=non_default_params,
@ -5297,6 +5300,11 @@ def validate_environment( # noqa: PLR0915
keys_in_environment = True
else:
missing_keys.append("OPENROUTER_API_KEY")
elif custom_llm_provider == "vercel_ai_gateway":
if "VERCEL_AI_GATEWAY_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("VERCEL_AI_GATEWAY_API_KEY")
elif custom_llm_provider == "datarobot":
if "DATAROBOT_API_TOKEN" in os.environ:
keys_in_environment = True
@ -5520,6 +5528,12 @@ def validate_environment( # noqa: PLR0915
keys_in_environment = True
else:
missing_keys.append("OPENROUTER_API_KEY")
## vercel_ai_gateway
elif model in litellm.vercel_ai_gateway_models:
if "VERCEL_AI_GATEWAY_API_KEY" in os.environ:
keys_in_environment = True
else:
missing_keys.append("VERCEL_AI_GATEWAY_API_KEY")
## datarobot
elif model in litellm.datarobot_models:
if "DATAROBOT_API_TOKEN" in os.environ:
@ -6904,6 +6918,8 @@ class ProviderConfigManager:
return litellm.TogetherAIConfig()
elif litellm.LlmProviders.OPENROUTER == provider:
return litellm.OpenrouterConfig()
elif litellm.LlmProviders.VERCEL_AI_GATEWAY == provider:
return litellm.VercelAIGatewayConfig()
elif litellm.LlmProviders.COMETAPI == provider:
return litellm.CometAPIConfig()
elif litellm.LlmProviders.DATAROBOT == provider:

1023
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

123
tests/image_gen_tests/test_image_edits.py
View file

@ -651,14 +651,7 @@ async def test_image_edit_array_handling():
image=TEST_IMAGES,
)
# Test 3: Empty list (should fail validation)
with pytest.raises(Exception):
await aimage_edit(
prompt=prompt,
model="gpt-image-1",
image=[],
)
# Both valid calls should succeed
ImageResponse.model_validate(result1)
ImageResponse.model_validate(result2)
@ -667,117 +660,3 @@ async def test_image_edit_array_handling():
assert mock_post.call_count == 2
@pytest.mark.asyncio
async def test_openai_transformation_handles_multiple_images():
"""Test that OpenAI transformation correctly handles multiple images in request"""
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.types.router import GenericLiteLLMParams
config = OpenAIImageEditConfig()
# Test with multiple images
prompt = "Edit these images"
images = [b"fake_image_1", b"fake_image_2", b"fake_image_3"]
litellm_params = GenericLiteLLMParams(api_key="test_key")
data, files = config.transform_image_edit_request(
model="gpt-image-1",
prompt=prompt,
image=images,
image_edit_optional_request_params={"n": 1},
litellm_params=litellm_params,
headers={}
)
# Check that data contains the prompt and parameters
assert data["prompt"] == prompt
assert data["model"] == "gpt-image-1"
assert data["n"] == 1
# Check that files contains all images with correct field names
assert len(files) == len(images)
for i, file_entry in enumerate(files):
assert file_entry[0] == "image[]" # OpenAI uses image[] for multiple files
assert file_entry[1][1] == images[i] # Image data
assert file_entry[1][2] == "image/png" # Content type
print(f"Successfully processed {len(images)} images in transformation")
@pytest.mark.asyncio
async def test_multiple_image_edit_parameter_validation():
"""Test parameter validation with multiple images"""
from litellm import aimage_edit
# Mock response
mock_response = {
"created": 1589478378,
"data": [
{
"b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
}
]
}
class MockResponse:
def __init__(self, json_data, status_code):
self._json_data = json_data
self.status_code = status_code
self.text = json.dumps(json_data)
def json(self):
return self._json_data
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new_callable=AsyncMock,
) as mock_post:
mock_post.return_value = MockResponse(mock_response, 200)
# Test with valid parameters
result = await aimage_edit(
prompt="Test prompt",
model="gpt-image-1",
image=TEST_IMAGES,
n=1,
size="1024x1024",
response_format="b64_json"
)
ImageResponse.model_validate(result)
# Verify the request was made with correct parameters
mock_post.assert_called_once()
call_args = mock_post.call_args
# Check that the request contains the expected data
if 'data' in call_args.kwargs:
form_data = call_args.kwargs['data']
assert 'model' in form_data
assert 'prompt' in form_data
assert 'n' in form_data
assert form_data['n'] == 1 # Could be int or string depending on implementation print("Parameter validation passed for multiple image edit")
@pytest.mark.asyncio
async def test_multiple_image_edit_error_handling():
"""Test error handling with multiple images"""
from litellm import aimage_edit
# Test with None image (should raise error)
with pytest.raises(Exception):
await aimage_edit(
prompt="Test prompt",
model="gpt-image-1",
image=None,
)
# Test with invalid model (should raise error)
with pytest.raises(Exception):
await aimage_edit(
prompt="Test prompt",
model="invalid-model",
image=TEST_IMAGES,
)
print("Error handling tests passed for multiple image edit")

77
tests/image_gen_tests/test_image_generation.py
View file

@ -175,7 +175,7 @@ class TestAimlImageGeneration(BaseImageGenTest):
class TestGoogleImageGen(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
return {"model": "gemini/imagen-4.0-generate-preview-06-06"}
return {"model": "gemini/imagen-4.0-generate-001"}
class TestAzureOpenAIDalle3(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
@ -330,3 +330,78 @@ async def test_gpt_image_1_with_input_fidelity():
assert captured_kwargs["quality"] == "medium"
assert captured_kwargs["size"] == "1024x1024"
@pytest.mark.asyncio
async def test_aiml_image_generation_with_dynamic_api_key():
"""
Test that when api_key is passed as a dynamic parameter to aimage_generation,
it gets properly used for AIML provider authentication instead of falling back
to environment variables.
This test validates the fix for ensuring dynamic API keys are respected
when making image generation requests to the AIML provider.
"""
from unittest.mock import AsyncMock, patch, MagicMock
import httpx
# Mock AIML response
mock_aiml_response = {
"created": 1703658209,
"data": [
{
"url": "https://example.com/generated_image.png"
}
]
}
# Track captured arguments
captured_headers = None
captured_url = None
captured_json_data = None
def capture_post_call(*args, **kwargs):
nonlocal captured_headers, captured_url, captured_json_data
captured_url = kwargs.get('url') or (args[0] if args else None)
captured_headers = kwargs.get('headers', {})
captured_json_data = kwargs.get('json', {})
# Create a mock response
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = mock_aiml_response
mock_response.text = json.dumps(mock_aiml_response)
return mock_response
# Mock the HTTP client that actually makes the request (sync version for image generation)
with patch('litellm.llms.custom_httpx.http_handler.HTTPHandler.post') as mock_post:
mock_post.side_effect = capture_post_call
# Test with dynamic api_key
test_api_key = "test-dynamic-api-key-12345"
response = await litellm.aimage_generation(
prompt="A cute baby sea otter",
model="aiml/flux-pro/v1.1",
api_key=test_api_key, # This should be used instead of env vars
)
# Validate the response (mocked response processing might not populate data correctly)
assert response is not None
# The most important validations: API key and endpoint usage
# These prove that the dynamic API key was properly used
assert captured_headers is not None
assert "Authorization" in captured_headers
assert captured_headers["Authorization"] == f"Bearer {test_api_key}"
print("TESTCAPTURED HEADERS", captured_headers)
# Validate the correct AIML endpoint was called
assert captured_url is not None
assert "api.aimlapi.com" in captured_url
assert "/v1/images/generations" in captured_url
# Validate the request data
assert captured_json_data is not None
assert captured_json_data["prompt"] == "A cute baby sea otter"
assert captured_json_data["model"] == "flux-pro/v1.1"

25
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -1483,3 +1483,28 @@ async def test_openai_gpt5_reasoning_effort_parameter():
# Validate the response
print("Response:", json.dumps(response, indent=4, default=str))
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [True, False])
async def test_basic_openai_responses_with_websearch(stream):
litellm._turn_on_debug()
request_model = "gpt-4o"
response = await litellm.aresponses(
model=request_model,
stream=stream,
input="hi",
tools=[
{
"type": "web_search",
"search_context_size": "low"
}
]
)
if stream:
async for chunk in response:
print("chunk=", json.dumps(chunk, indent=4, default=str))
else:
print("response=", json.dumps(response, indent=4, default=str))

2
tests/llm_translation/base_llm_unit_tests.py
View file

@ -119,7 +119,7 @@ class BaseLLMChatTest(ABC):
pytest.skip("Model is overloaded")
assert response.choices[0].message.content is not None
def test_content_list_handling(self):
"""Check if content list is supported by LLM API"""
base_completion_call_args = self.get_base_completion_call_args()

76
tests/llm_translation/test_gemini.py
View file

@ -261,7 +261,13 @@ def test_gemini_image_generation():
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
assert response.choices[0].message.content is not None
#########################################################
# Important: Validate we did get an image in the response
#########################################################
assert response.choices[0].message.image is not None
assert response.choices[0].message.image["url"] is not None
assert response.choices[0].message.image["url"].startswith("data:image/png;base64,")
def test_gemini_thinking():
@ -323,7 +329,7 @@ def test_gemini_thinking_budget_0():
"thinking": {"type": "enabled", "budget_tokens": 0},
},
)
print(raw_request)
print(json.dumps(raw_request, indent=4, default=str))
assert "0" in json.dumps(raw_request["raw_request_body"])
@ -571,3 +577,69 @@ def test_gemini_tool_use():
stop_reason = chunk.choices[0].finish_reason
assert stop_reason is not None
assert stop_reason == "tool_calls"
@pytest.mark.asyncio
async def test_gemini_image_generation_async():
litellm._turn_on_debug()
response = await litellm.acompletion(
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
model="gemini/gemini-2.5-flash-image-preview",
)
CONTENT = response.choices[0].message.content
IMAGE_URL = response.choices[0].message.image
print("IMAGE_URL: ", IMAGE_URL)
assert CONTENT is not None, "CONTENT is not None"
assert IMAGE_URL is not None, "IMAGE_URL is not None"
assert IMAGE_URL["url"] is not None, "IMAGE_URL['url'] is not None"
assert IMAGE_URL["url"].startswith("data:image/png;base64,")
@pytest.mark.asyncio
async def test_gemini_image_generation_async_stream():
#litellm._turn_on_debug()
response = await litellm.acompletion(
messages=[{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}],
model="gemini/gemini-2.5-flash-image-preview",
stream=True,
)
print("RESPONSE: ", response)
model_response_image = None
async for chunk in response:
print("CHUNK: ", chunk)
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
model_response_image = chunk.choices[0].delta.image
print("MODEL_RESPONSE_IMAGE: ", model_response_image)
assert model_response_image is not None
assert model_response_image["url"].startswith("data:image/png;base64,")
break
#########################################################
# Important: Validate we did get an image in the response
#########################################################
assert model_response_image is not None
assert model_response_image["url"].startswith("data:image/png;base64,")
def test_system_message_with_no_user_message():
"""
Test that the system message is translated correctly for non-OpenAI providers.
"""
messages = [
{
"role": "system",
"content": "Be a good bot!",
},
]
response = litellm.completion(
model="gemini/gemini-2.5-flash",
messages=messages,
)
assert response is not None
assert response.choices[0].message.content is not None

18
tests/llm_translation/test_openrouter.py Normal file
View file

@ -0,0 +1,18 @@
import os
import sys
import pytest
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system paths
import litellm
def test_completion_openrouter_reasoning_content():
litellm._turn_on_debug()
resp = litellm.completion(
model="openrouter/anthropic/claude-3.7-sonnet",
messages=[{"role": "user", "content": "Hello world"}],
reasoning={"effort": "high"},
)
print(resp)
assert resp.choices[0].message.reasoning_content is not None

2
tests/llm_translation/test_optional_params.py
View file

@ -1557,7 +1557,7 @@ def test_azure_ai_cohere_embed_input_type_param():
def test_optional_params_image_gen_with_aspect_ratio():
optional_params = get_optional_params_image_gen(
model="imagen-4.0-ultra-generate-preview-06-06",
model="imagen-4.0-ultra-generate-001",
custom_llm_provider="vertex_ai",
aspect_ratio="16:9",
)

9
tests/llm_translation/test_xai.py
View file

@ -130,6 +130,15 @@ def test_xai_grok_4_stop_not_supported(model):
assert "stop" not in supported_params
@pytest.mark.parametrize("model", ["xai/grok-4", "xai/grok-4-0709", "xai/grok-4-latest", "xai/grok-code-fast", "xai/grok-code-fast-1"])
def test_xai_grok_4_frequency_penalty_not_supported(model):
"""
Test that grok-4 models do not support the frequency_penalty parameter
"""
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
assert "frequency_penalty" not in supported_params
def test_xai_message_name_filtering():
messages = [

2
tests/local_testing/test_caching.py
View file

@ -314,7 +314,7 @@ async def test_caching_with_cache_controls(sync_flag):
# test_caching_with_cache_controls()
@pytest.mark.flaky(retries=3, delay=1)
def test_caching_with_models_v2():
messages = [
{"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"}

11
tests/local_testing/test_completion.py
View file

@ -4592,14 +4592,3 @@ def test_completion_gpt_4o_empty_str():
messages=[{"role": "user", "content": ""}],
)
assert resp.choices[0].message.content is not None
def test_completion_openrouter_reasoning_content():
litellm._turn_on_debug()
resp = litellm.completion(
model="openrouter/anthropic/claude-3.7-sonnet",
messages=[{"role": "user", "content": "Hello world"}],
reasoning={"effort": "high"},
)
print(resp)
assert resp.choices[0].message.reasoning_content is not None

90
tests/logging_callback_tests/test_spend_logs.py
View file

@ -25,7 +25,7 @@ from typing import Optional
import pytest
import litellm
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload, _sanitize_request_body_for_spend_logs_payload
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
@ -396,3 +396,91 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch):
payload_disabled: SpendLogsPayload = get_logging_payload(**input_args)
assert payload_disabled["messages"] == "{}"
assert payload_disabled["response"] == "{}"
def test_large_request_no_truncation_threshold():
"""
Test that MAX_STRING_LENGTH_PROMPT_IN_DB constant is used for request body sanitization
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, LITELLM_TRUNCATED_PAYLOAD_FIELD
# Create a large string that exceeds the threshold
large_content = "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
request_body = {
"messages": [
{"role": "user", "content": large_content}
],
"model": "gpt-4"
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify the content was truncated
truncated_content = sanitized["messages"][0]["content"]
assert len(truncated_content) > MAX_STRING_LENGTH_PROMPT_IN_DB # includes truncation message
assert truncated_content.startswith("x" * MAX_STRING_LENGTH_PROMPT_IN_DB)
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_content
assert "500 chars" in truncated_content
def test_small_request_no_truncation():
"""
Test that small strings are not truncated by MAX_STRING_LENGTH_PROMPT_IN_DB
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a small string that's under the threshold
small_content = "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB - 100)
request_body = {
"messages": [
{"role": "user", "content": small_content}
],
"model": "gpt-4"
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify the content was NOT truncated
assert sanitized["messages"][0]["content"] == small_content
assert len(sanitized["messages"][0]["content"]) == MAX_STRING_LENGTH_PROMPT_IN_DB - 100
def test_configurable_string_length_env_var(monkeypatch):
"""
Test that MAX_STRING_LENGTH_PROMPT_IN_DB can be configured via environment variable
"""
# Set environment variable to a custom value
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "500")
# Import after setting env var to ensure it picks up the new value
import importlib
import litellm.constants
import litellm.proxy.spend_tracking.spend_tracking_utils
importlib.reload(litellm.constants)
importlib.reload(litellm.proxy.spend_tracking.spend_tracking_utils)
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, LITELLM_TRUNCATED_PAYLOAD_FIELD
from litellm.proxy.spend_tracking.spend_tracking_utils import _sanitize_request_body_for_spend_logs_payload
# Verify the constant was set to the env var value
assert MAX_STRING_LENGTH_PROMPT_IN_DB == 500
# Test truncation with the custom value
large_content = "y" * 750 # 250 chars over the custom limit
request_body = {
"messages": [
{"role": "user", "content": large_content}
],
"model": "gpt-4"
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify truncation occurred at the custom threshold
truncated_content = sanitized["messages"][0]["content"]
assert truncated_content.startswith("y" * 500)
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_content
assert "250 chars" in truncated_content

2
tests/openai_endpoints_tests/input_azure.jsonl
View file

@ -1 +1 @@
{"custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-mini", "temperature": 0, "max_tokens": 1024, "response_format": {"type": "json_object"}, "messages": [{"role": "user", "content": "# Instruction \n\nYou are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models. \nWe will provide you with the user query and an AI-generated responses.\nYo must respond in json"}]}}
{"custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "method": "POST", "url": "/chat/completions", "body": {"model": "gpt-4o-mini", "temperature": 0, "max_tokens": 1024, "response_format": {"type": "json_object"}, "messages": [{"role": "user", "content": "# Instruction \n\nYou are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models. \nWe will provide you with the user query and an AI-generated responses.\nYo must respond in json"}]}}

71
tests/test_litellm/conftest.py
View file

@ -3,23 +3,9 @@
import importlib
import os
import sys
import tempfile
import random
import string
import pytest
# Set up a temporary log directory and file BEFORE importing litellm
temp_dir = tempfile.mkdtemp(prefix="litellm_test_")
test_log_file = os.path.join(temp_dir, "test_litellm.log")
# Store original log file for cleanup
orig_log_file = os.getenv("LITELLM_LOG_FILE")
# Set environment variables to use temporary files BEFORE importing litellm
os.environ["LITELLM_LOG_FILE"] = test_log_file
# Import litellm after setting up the environment
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
@ -27,61 +13,6 @@ import asyncio
import litellm
@pytest.fixture(scope="function")
def temp_log_file():
"""
Creates a temporary log file in /tmp/litellm<random_number>.log for testing.
Returns the path to the temporary log file and cleans it up after the test.
"""
# Generate a random number for the log file
random_number = ''.join(random.choices(string.digits, k=8))
log_file_path = f"/tmp/litellm{random_number}.log"
# Set the environment variable for litellm to use this temporary log file
original_log_file = os.environ.get("LITELLM_LOG_FILE")
os.environ["LITELLM_LOG_FILE"] = log_file_path
yield log_file_path
# Cleanup: Restore original environment variable and remove the temporary file
if original_log_file is not None:
os.environ["LITELLM_LOG_FILE"] = original_log_file
else:
os.environ.pop("LITELLM_LOG_FILE", None)
# Remove the temporary log file if it exists
if os.path.exists(log_file_path):
try:
os.remove(log_file_path)
except OSError:
pass # Ignore errors if file can't be removed
@pytest.fixture(scope="session", autouse=True)
def cleanup_temp_log_dir():
"""
Cleans up the temporary log directory created at module import time.
This runs once per test session after all tests are complete.
"""
yield
if orig_log_file is not None:
os.environ["LITELLM_LOG_FILE"] = orig_log_file
else:
os.environ.pop("LITELLM_LOG_FILE", None)
# Cleanup: Remove the temporary directory created at module import time
if os.path.exists(temp_dir):
try:
# Remove the test log file first
if os.path.exists(test_log_file):
os.remove(test_log_file)
# Remove the temporary directory
import shutil
shutil.rmtree(temp_dir, ignore_errors=True)
except OSError:
pass # Ignore errors if cleanup fails
@pytest.fixture(scope="session")
def event_loop():
@ -94,6 +25,7 @@ def event_loop():
@pytest.fixture(scope="function", autouse=True)
def setup_and_teardown():
"""
@ -145,4 +77,3 @@ def pytest_collection_modifyitems(config, items):
# Reorder the items list
items[:] = custom_logger_tests + other_tests

13
tests/test_litellm/integrations/test_langfuse_otel.py
View file

@ -229,6 +229,19 @@ class TestLangfuseOtelIntegration:
# Should return an empty dict
assert result == {}
def test_get_langfuse_otel_config_with_otel_host_priority(self):
"""LANGFUSE_OTEL_HOST should take priority over LANGFUSE_HOST."""
with patch.dict(os.environ, {
'LANGFUSE_PUBLIC_KEY': 'test_public_key',
'LANGFUSE_SECRET_KEY': 'test_secret_key',
'LANGFUSE_HOST': 'https://should-not-be-used.com',
'LANGFUSE_OTEL_HOST': 'https://otel-host.com'
}, clear=False):
_ = LangfuseOtelLogger.get_langfuse_otel_config()
assert os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") == "https://otel-host.com/api/public/otel"
if __name__ == "__main__":

200
tests/test_litellm/integrations/test_openmeter.py
View file

@ -1,7 +1,5 @@
import asyncio
import json
import os
import sys
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@ -282,3 +280,201 @@ class TestOpenMeterIntegration:
result = logger._common_logic(kwargs, response_obj)
assert result["type"] == "custom_event_type"
def test_common_logic_user_from_token_user_id(self):
"""Test that _common_logic uses user_api_key_user_id when no user provided"""
logger = OpenMeterLogger()
kwargs = {
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key_user_id": "token-user-123"
}
}
# No "user" parameter - should use token user_id
}
response_obj = {
"id": "test-response-id",
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15
}
}
result = logger._common_logic(kwargs, response_obj)
# Verify user was set from token user_id
assert isinstance(result["subject"], str)
assert result["subject"] == "token-user-123"
assert result["data"]["model"] == "gpt-3.5-turbo"
def test_common_logic_direct_user_takes_priority_over_token(self):
"""Test that direct user parameter takes priority over token user_id"""
logger = OpenMeterLogger()
kwargs = {
"user": "direct-user-456", # Direct user should take priority
"model": "gpt-4",
"response_cost": 0.002,
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key_user_id": "token-user-123" # This should be ignored
}
}
}
response_obj = {
"id": "test-response-id",
"usage": {
"prompt_tokens": 20,
"completion_tokens": 10,
"total_tokens": 30
}
}
result = logger._common_logic(kwargs, response_obj)
# Verify direct user takes priority
assert isinstance(result["subject"], str)
assert result["subject"] == "direct-user-456"
assert result["subject"] != "token-user-123"
def test_common_logic_missing_user_and_token_user_id(self):
"""Test that exception is raised when neither user nor token user_id available"""
logger = OpenMeterLogger()
kwargs = {
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
# No user_api_key_user_id
}
}
# No "user" parameter
}
response_obj = {"id": "test-response-id"}
with pytest.raises(Exception, match="OpenMeter: user is required"):
logger._common_logic(kwargs, response_obj)
def test_common_logic_token_user_id_none(self):
"""Test that exception is raised when token user_id is None"""
logger = OpenMeterLogger()
kwargs = {
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key_user_id": None # Explicitly None
}
}
}
response_obj = {"id": "test-response-id"}
with pytest.raises(Exception, match="OpenMeter: user is required"):
logger._common_logic(kwargs, response_obj)
def test_common_logic_no_metadata(self):
"""Test that exception is raised when no metadata is available"""
logger = OpenMeterLogger()
kwargs = {
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
"litellm_call_id": "test-call-id",
# No litellm_params at all
}
response_obj = {"id": "test-response-id"}
with pytest.raises(Exception, match="OpenMeter: user is required"):
logger._common_logic(kwargs, response_obj)
def test_common_logic_integer_token_user_id(self):
"""Test that integer token user_id is converted to string"""
logger = OpenMeterLogger()
kwargs = {
"model": "gpt-4",
"response_cost": 0.003,
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key_user_id": 12345 # Integer user_id
}
}
}
response_obj = {
"id": "test-response-id",
"usage": {
"prompt_tokens": 25,
"completion_tokens": 12,
"total_tokens": 37
}
}
result = logger._common_logic(kwargs, response_obj)
# Verify integer user_id is converted to string
assert isinstance(result["subject"], str)
assert result["subject"] == "12345"
@patch('litellm.integrations.openmeter.HTTPHandler')
def test_integration_token_user_id_scenario(self, mock_http_handler):
"""Integration test simulating the exact scenario that was failing"""
mock_post = MagicMock()
mock_http_handler.return_value.post = mock_post
logger = OpenMeterLogger()
# Simulate the exact scenario: request with token that has user_id but no direct user param
kwargs = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"response_cost": 0.001,
"litellm_call_id": "test-integration-call-id",
"litellm_params": {
"metadata": {
"user_api_key_user_id": "user123-from-token",
"user_api_key": "hashed-key-abc",
"user_api_key_metadata": {}
}
}
# No "user" parameter - this was causing "OpenMeter: user is required" error
}
response_obj = {
"id": "chatcmpl-test123",
"usage": {
"prompt_tokens": 15,
"completion_tokens": 10,
"total_tokens": 25
}
}
# This should NOT raise "OpenMeter: user is required" anymore
logger.log_success_event(kwargs, response_obj, None, None)
# Verify HTTP call was made
mock_post.assert_called_once()
# Verify the data structure contains user from token
call_args = mock_post.call_args
data = json.loads(call_args[1]['data'])
assert data["subject"] == "user123-from-token"
assert isinstance(data["subject"], str)
assert data["data"]["model"] == "gpt-3.5-turbo"

141
tests/test_litellm/litellm_core_utils/test_logging_worker.py Normal file
View file

@ -0,0 +1,141 @@
"""
Tests for the LoggingWorker class to ensure graceful shutdown handling.
"""
import asyncio
import pytest
from unittest.mock import AsyncMock, patch
from litellm.litellm_core_utils.logging_worker import LoggingWorker
class TestLoggingWorker:
"""Test cases for LoggingWorker functionality."""
@pytest.fixture
def logging_worker(self):
"""Create a LoggingWorker instance for testing."""
return LoggingWorker(timeout=1.0, max_queue_size=10)
@pytest.mark.asyncio
async def test_graceful_shutdown_with_clear_queue(self, logging_worker):
"""Test that cancellation triggers clear_queue to prevent 'never awaited' warnings."""
# Mock the clear_queue method to verify it's called during cancellation
with patch.object(logging_worker, "clear_queue", new_callable=AsyncMock) as mock_clear_queue:
# Start the worker
logging_worker.start()
# Give it a moment to start
await asyncio.sleep(0.1)
# Cancel the worker task to simulate shutdown
if logging_worker._worker_task:
logging_worker._worker_task.cancel()
# Wait for the task to handle the cancellation
try:
await logging_worker._worker_task
except asyncio.CancelledError:
# Expected during cancellation
pass
# Verify that clear_queue was called during cancellation
mock_clear_queue.assert_called_once()
@pytest.mark.asyncio
async def test_clear_queue_processes_remaining_items(self, logging_worker):
"""Test that clear_queue processes remaining coroutines to prevent warnings."""
# Create mock coroutines
mock_coro1 = AsyncMock()
mock_coro2 = AsyncMock()
# Initialize the worker and add items to queue
logging_worker._ensure_queue()
logging_worker.enqueue(mock_coro1())
logging_worker.enqueue(mock_coro2())
# Clear the queue
await logging_worker.clear_queue()
# Verify the queue is empty after clearing
assert logging_worker._queue.empty()
@pytest.mark.asyncio
async def test_worker_handles_cancellation_gracefully(self, logging_worker):
"""Test that the worker handles cancellation without throwing exceptions."""
# Mock verbose_logger to capture debug messages
with patch("litellm.litellm_core_utils.logging_worker.verbose_logger") as mock_logger:
# Start the worker
logging_worker.start()
# Give it a moment to start
await asyncio.sleep(0.1)
# Cancel and wait for completion
await logging_worker.stop()
# Verify debug message was logged instead of exception
debug_calls = [
call
for call in mock_logger.debug.call_args_list
if "LoggingWorker cancelled during shutdown" in str(call)
]
assert len(debug_calls) >= 0 # May be 0 if no cancellation occurred
@pytest.mark.asyncio
async def test_enqueue_and_process_single_item(self, logging_worker):
"""Test basic enqueue and process functionality."""
# Create a mock coroutine that we can track
mock_coro = AsyncMock()
# Start the worker
logging_worker.start()
# Enqueue a coroutine
logging_worker.enqueue(mock_coro())
# Give the worker time to process the item
await asyncio.sleep(0.2)
# Stop the worker
await logging_worker.stop()
# The mock should have been awaited (processed)
assert mock_coro.called
@pytest.mark.asyncio
async def test_clear_queue_with_time_limit(self, logging_worker):
"""Test that clear_queue respects the time limit."""
# Create several mock coroutines that take time to complete
slow_coro = AsyncMock()
slow_coro.return_value = asyncio.sleep(0.5) # Takes 500ms
# Initialize the worker and add items
logging_worker._ensure_queue()
for _ in range(5):
logging_worker.enqueue(slow_coro())
# Clear the queue - should timeout based on MAX_TIME_TO_CLEAR_QUEUE
start_time = asyncio.get_event_loop().time()
await logging_worker.clear_queue()
elapsed_time = asyncio.get_event_loop().time() - start_time
# Should complete within reasonable time (allowing for some processing)
assert elapsed_time < 10.0 # Much less than if it processed all slow items
@pytest.mark.asyncio
async def test_queue_full_handling(self, logging_worker):
"""Test that queue full condition is handled gracefully."""
# Create a worker with very small queue size
small_worker = LoggingWorker(timeout=1.0, max_queue_size=2)
small_worker._ensure_queue()
# Mock verbose_logger to capture exception messages
with patch("litellm.litellm_core_utils.logging_worker.verbose_logger") as mock_logger:
# Fill the queue beyond capacity
mock_coro = AsyncMock()
for _ in range(5): # More than max_queue_size of 2
small_worker.enqueue(mock_coro())
# Should have logged queue full exceptions
exception_calls = [call for call in mock_logger.exception.call_args_list if "queue is full" in str(call)]
assert len(exception_calls) > 0

282
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -15,7 +15,10 @@ from typing import Optional
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.streaming_handler import (
AUDIO_ATTRIBUTE,
CustomStreamWrapper,
)
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
Delta,
@ -754,7 +757,7 @@ def test_optional_combine_thinking_block_with_none_content(
# Second chunk with reasoning_content and None content
second_chunk = {
"id": "chunk2",
"id": "chunk2",
"object": "chat.completion.chunk",
"created": 1741037891,
"model": "deepseek-reasoner",
@ -773,16 +776,13 @@ def test_optional_combine_thinking_block_with_none_content(
# Final chunk with actual content - should add </think> tag
final_chunk = {
"id": "chunk3",
"object": "chat.completion.chunk",
"object": "chat.completion.chunk",
"created": 1741037892,
"model": "deepseek-reasoner",
"choices": [
{
"index": 0,
"delta": {
"content": "The answer is 42",
"reasoning_content": None
},
"delta": {"content": "The answer is 42", "reasoning_content": None},
"finish_reason": None,
}
],
@ -793,12 +793,15 @@ def test_optional_combine_thinking_block_with_none_content(
initialized_custom_stream_wrapper._optional_combine_thinking_block_in_choices(
first_response
)
assert first_response.choices[0].delta.content == "<think>Let me think about this problem"
assert (
first_response.choices[0].delta.content
== "<think>Let me think about this problem"
)
assert not hasattr(first_response.choices[0].delta, "reasoning_content")
assert initialized_custom_stream_wrapper.sent_first_thinking_block is True
# Process second chunk - should work with continued reasoning
second_response = ModelResponseStream(**second_chunk)
second_response = ModelResponseStream(**second_chunk)
initialized_custom_stream_wrapper._optional_combine_thinking_block_in_choices(
second_response
)
@ -813,3 +816,264 @@ def test_optional_combine_thinking_block_with_none_content(
assert final_response.choices[0].delta.content == "</think>The answer is 42"
assert initialized_custom_stream_wrapper.sent_last_thinking_block is True
assert not hasattr(final_response.choices[0].delta, "reasoning_content")
def test_has_special_delta_content(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Test the _has_special_delta_content helper method"""
# Test empty choices
empty_response = ModelResponseStream(
id="test", created=1742056047, model=None, choices=[]
)
assert not initialized_custom_stream_wrapper._has_special_delta_content(
empty_response
)
# Test with tool_calls (simulate with mock object)
tool_call_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(
content=None,
tool_calls=[
{
"id": "test",
"function": {"arguments": "{}", "name": "test_func"},
}
],
),
)
],
)
assert initialized_custom_stream_wrapper._has_special_delta_content(
tool_call_response
)
# Test with function_call (simulate with mock object)
function_call_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(
content=None, function_call={"name": "test_func", "arguments": "{}"}
),
)
],
)
assert initialized_custom_stream_wrapper._has_special_delta_content(
function_call_response
)
# Test with audio (simulate by adding audio attribute)
audio_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content=None))
],
)
# Manually add audio attribute to delta
audio_response.choices[0].delta.audio = {"transcript": "test"}
assert initialized_custom_stream_wrapper._has_special_delta_content(audio_response)
# Test with image (simulate by adding image attribute)
image_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content=None))
],
)
# Manually add image attribute to delta
image_response.choices[0].delta.image = {"url": "test.jpg"}
assert initialized_custom_stream_wrapper._has_special_delta_content(image_response)
# Test with regular content (should return False)
regular_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(
finish_reason=None, index=0, delta=Delta(content="Hello world")
)
],
)
assert not initialized_custom_stream_wrapper._has_special_delta_content(
regular_response
)
def test_handle_special_delta_content(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Test the _handle_special_delta_content helper method"""
test_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(content="test", role="assistant"),
)
],
)
# The method should call strip_role_from_delta
result = initialized_custom_stream_wrapper._handle_special_delta_content(
test_response
)
# Should return the same response object (modified)
assert result is test_response
# Should have set sent_first_chunk to True
assert initialized_custom_stream_wrapper.sent_first_chunk is True
def test_has_any_special_delta_attributes(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Test the _has_any_special_delta_attributes helper method"""
# Test with delta that has audio attribute
class MockDelta:
def __init__(self):
self.audio = {"transcript": "Hello world"}
audio_delta = MockDelta()
result = initialized_custom_stream_wrapper._has_any_special_delta_attributes(
audio_delta
)
assert result is True
# Test with delta that has image attribute
class MockDeltaImage:
def __init__(self):
self.image = {"url": "test.jpg"}
image_delta = MockDeltaImage()
result = initialized_custom_stream_wrapper._has_any_special_delta_attributes(
image_delta
)
assert result is True
# Test with delta that has no special attributes
class MockDeltaRegular:
def __init__(self):
self.content = "regular content"
regular_delta = MockDeltaRegular()
result = initialized_custom_stream_wrapper._has_any_special_delta_attributes(
regular_delta
)
assert result is False
def test_handle_special_delta_attributes(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Test the _handle_special_delta_attributes helper method"""
# Create a model response
model_response = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="test"))
],
)
# Test with delta that has audio attribute
class MockDelta:
def __init__(self):
self.audio = {"transcript": "Hello world"}
audio_delta = MockDelta()
initialized_custom_stream_wrapper._handle_special_delta_attributes(
audio_delta, model_response
)
# Should copy the audio attribute
assert hasattr(model_response.choices[0].delta, "audio")
assert model_response.choices[0].delta.audio == {"transcript": "Hello world"}
# Test with delta that has image attribute
class MockDeltaImage:
def __init__(self):
self.image = {"url": "test.jpg"}
image_delta = MockDeltaImage()
model_response2 = ModelResponseStream(
id="test",
created=1742056047,
model=None,
choices=[
StreamingChoices(finish_reason=None, index=0, delta=Delta(content="test"))
],
)
initialized_custom_stream_wrapper._handle_special_delta_attributes(
image_delta, model_response2
)
# Should copy the image attribute
assert hasattr(model_response2.choices[0].delta, "image")
assert model_response2.choices[0].delta.image == {"url": "test.jpg"}
def test_has_special_delta_attribute(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Test the _has_special_delta_attribute helper method"""
# Test with None delta
assert not initialized_custom_stream_wrapper._has_special_delta_attribute(
None, "audio"
)
# Test with delta that has the attribute
class MockDelta:
def __init__(self):
self.audio = {"transcript": "test"}
delta_with_audio = MockDelta()
assert initialized_custom_stream_wrapper._has_special_delta_attribute(
delta_with_audio, "audio"
)
# Test with delta that doesn't have the attribute
class MockDeltaNoAudio:
def __init__(self):
self.content = "test"
delta_without_audio = MockDeltaNoAudio()
assert not initialized_custom_stream_wrapper._has_special_delta_attribute(
delta_without_audio, "audio"
)
# Test with delta that has the attribute but it's None
class MockDeltaNone:
def __init__(self):
self.audio = None
delta_with_none = MockDeltaNone()
assert not initialized_custom_stream_wrapper._has_special_delta_attribute(
delta_with_none, "audio"
)

22
tests/test_litellm/llms/deepinfra/test_deepinfra_chat_transformation.py Normal file
View file

@ -0,0 +1,22 @@
import asyncio
import json
import os
import sys
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
# Add litellm to path
sys.path.insert(0, os.path.abspath("../../../.."))
import litellm
def test_deepseek_supported_openai_params():
"""
Test "reasoning_effort" is an openai param supported for the DeepSeek model on deepinfra
"""
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
supported_openai_params = DeepInfraConfig().get_supported_openai_params(model="deepinfra/deepseek-ai/DeepSeek-V3.1")
print(supported_openai_params)
assert "reasoning_effort" in supported_openai_params

51
tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py
View file

@ -364,6 +364,57 @@ def test_x_initiator_header_system_only_messages():
assert headers["X-Initiator"] == "user"
def test_get_supported_openai_params_claude_model():
"""Test that Claude models with extended thinking support have thinking and reasoning parameters."""
config = GithubCopilotConfig()
# Test Claude 4 model supports thinking and reasoning_effort parameters
supported_params = config.get_supported_openai_params("claude-sonnet-4-20250514")
assert "thinking" in supported_params
assert "reasoning_effort" in supported_params
# Test Claude 3-7 model supports thinking and reasoning_effort parameters
supported_params_claude37 = config.get_supported_openai_params("claude-3-7-sonnet-20250219")
assert "thinking" in supported_params_claude37
assert "reasoning_effort" in supported_params_claude37
# Test Claude 3.5 model does NOT support thinking parameters (no extended thinking)
supported_params_claude35 = config.get_supported_openai_params("claude-3.5-sonnet")
assert "thinking" not in supported_params_claude35
assert "reasoning_effort" not in supported_params_claude35
# Test non-Claude model doesn't include thinking parameters but may include reasoning_effort
supported_params_gpt = config.get_supported_openai_params("gpt-4o")
assert "thinking" not in supported_params_gpt
# gpt-4o should NOT have reasoning_effort (not a reasoning model)
assert "reasoning_effort" not in supported_params_gpt
# Test O-series reasoning models include reasoning_effort but not thinking
supported_params_o3 = config.get_supported_openai_params("o3-mini")
assert "thinking" not in supported_params_o3
# o3-mini should have reasoning_effort (it's an O-series reasoning model)
assert "reasoning_effort" in supported_params_o3
def test_get_supported_openai_params_case_insensitive():
"""Test that Claude model detection is case-insensitive for models with extended thinking."""
config = GithubCopilotConfig()
# Test uppercase Claude 4 model with full model name
supported_params_upper = config.get_supported_openai_params("CLAUDE-SONNET-4-20250514")
assert "thinking" in supported_params_upper
assert "reasoning_effort" in supported_params_upper
# Test mixed case Claude 3-7 model (has extended thinking) with full model name
supported_params_mixed = config.get_supported_openai_params("Claude-3-7-Sonnet-20250219")
assert "thinking" in supported_params_mixed
assert "reasoning_effort" in supported_params_mixed
# Test that Claude 3.5 models don't have thinking support (case insensitive)
supported_params_35 = config.get_supported_openai_params("CLAUDE-3.5-SONNET")
assert "thinking" not in supported_params_35
assert "reasoning_effort" not in supported_params_35
def test_copilot_vision_request_header_with_image():
"""Test that Copilot-Vision-Request header is added when messages contain images"""
config = GithubCopilotConfig()

48
tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py
View file

@ -17,45 +17,56 @@ TEST_MODEL_NAME = "xai.grok-4"
TEST_MODEL = f"oci/{TEST_MODEL_NAME}"
TEST_MESSAGES = [{"role": "user", "content": "Hello, how are you?"}]
TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx"
TEST_OCI_PARAMS = {
BASE_OCI_PARAMS = {
"oci_region": "us-ashburn-1",
"oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx",
"oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90",
"oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx",
"oci_compartment_id": TEST_COMPARTMENT_ID,
"oci_key": "<private_key.pem as string>"
}
TEST_OCI_PARAMS_KEY = {
**BASE_OCI_PARAMS,
"oci_key": "<private_key.pem as string>",
}
TEST_OCI_PARAMS_KEY_FILE = {
**BASE_OCI_PARAMS,
"oci_key_file": "<private_key.pem as a Path>",
}
@pytest.fixture(params=[TEST_OCI_PARAMS_KEY, TEST_OCI_PARAMS_KEY_FILE])
def supplied_params(request):
"""Fixture for passing in optional_parameters"""
return request.param
class TestOCIChatConfig:
def test_validate_environment_with_oci_region(self):
def test_validate_environment_with_oci_region(self, supplied_params):
config = OCIChatConfig()
headers = {}
result = config.validate_environment(
headers=headers,
model=TEST_MODEL,
messages=TEST_MESSAGES, # type: ignore
optional_params=TEST_OCI_PARAMS,
optional_params=supplied_params,
litellm_params={},
)
assert result["content-type"] == "application/json"
assert result["user-agent"] == f"litellm/{version}"
def test_missing_oci_auth_parameters(self):
optional_params = TEST_OCI_PARAMS.copy()
optional_params.pop("oci_region")
def test_missing_oci_auth_parameters(self, supplied_params):
params = supplied_params.copy() # safely copy, no reassignment
params.pop("oci_region")
# Remove optional_params one by one and verify that an exception is raised
for key in optional_params.keys():
modified_params = optional_params.copy()
for key in list(params.keys()):
modified_params = params.copy()
del modified_params[key]
with pytest.raises(Exception) as excinfo:
config = OCIChatConfig()
headers = {}
config.validate_environment(
@ -66,8 +77,7 @@ class TestOCIChatConfig:
api_base="https://api.oci.example.com",
litellm_params={},
)
assert f"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" in str(excinfo.value)
assert ("Missing required parameters:") in str(excinfo.value)
def test_transform_request_simple(self):
"""
@ -264,22 +274,22 @@ class TestOCIChatConfig:
litellm_params={},
encoding={},
)
# General assertions
assert isinstance(result, ModelResponse)
assert len(result.choices) == 1
choice = result.choices[0]
assert isinstance(choice, litellm.Choices)
assert choice.finish_reason == "stop"
# Message and tool_calls assertions
message = choice.message
assert isinstance(message, litellm.Message)
assert hasattr(message, "tool_calls")
assert isinstance(message.tool_calls, list)
assert len(message.tool_calls) == 1
# Specific tool_call assertions
tool_call = message.tool_calls[0]
assert isinstance(tool_call, litellm.utils.ChatCompletionMessageToolCall)
@ -287,7 +297,7 @@ class TestOCIChatConfig:
assert tool_call.type == "function"
assert tool_call.function["name"] == "get_weather"
assert tool_call.function["arguments"] == '{"location": "Vila Velha, BR"}'
# Usage assertions
assert hasattr(result, "usage")
usage = result.usage # type: ignore

112
tests/test_litellm/llms/vercel_ai_gateway/chat/test_vercel_ai_gateway_transformation.py Normal file
View file

@ -0,0 +1,112 @@
import os
import sys
from unittest.mock import patch
import pytest
sys.path.insert(
0, os.path.abspath("../../../../..")
) # Adds the parent directory to the system path
from litellm.llms.vercel_ai_gateway.chat.transformation import (
VercelAIGatewayConfig,
)
from litellm.llms.vercel_ai_gateway.common_utils import VercelAIGatewayException
def test_vercel_ai_gateway_extra_body_transformation():
"""Test that providerOptions is correctly moved to extra_body"""
transformed_request = VercelAIGatewayConfig().transform_request(
model="vercel_ai_gateway/openai/gpt-4o",
messages=[{"role": "user", "content": "Hello, world!"}],
optional_params={
"extra_body": {
"providerOptions": {
"gateway": {"order": ["azure", "openai"]}
}
}
},
litellm_params={},
headers={},
)
assert transformed_request["extra_body"]["providerOptions"]["gateway"]["order"] == ["azure", "openai"]
assert transformed_request["messages"] == [
{"role": "user", "content": "Hello, world!"}
]
def test_vercel_ai_gateway_provider_options_mapping():
"""Test that providerOptions from non_default_params is moved to extra_body"""
config = VercelAIGatewayConfig()
non_default_params = {
"providerOptions": {
"gateway": {"order": ["azure", "openai"]}
}
}
optional_params = {}
model = "vercel_ai_gateway/openai/gpt-4o"
result = config.map_openai_params(
non_default_params, optional_params, model, drop_params=False
)
assert result["extra_body"]["providerOptions"]["gateway"]["order"] == ["azure", "openai"]
assert "providerOptions" not in result
def test_vercel_ai_gateway_get_supported_openai_params():
"""Test that extra_body is included in supported params"""
config = VercelAIGatewayConfig()
supported_params = config.get_supported_openai_params("vercel_ai_gateway/openai/gpt-4o")
assert "extra_body" in supported_params
assert "temperature" in supported_params
assert "max_tokens" in supported_params
assert "stream" in supported_params
def test_vercel_ai_gateway_get_openai_compatible_provider_info():
"""Test provider info retrieval with environment variables"""
config = VercelAIGatewayConfig()
with patch.dict(
"os.environ",
{
"VERCEL_AI_GATEWAY_API_BASE": "https://env.vercel.sh/v1",
"VERCEL_AI_GATEWAY_API_KEY": "env_api_key",
},
):
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
assert api_base == "https://env.vercel.sh/v1"
assert api_key == "env_api_key"
def test_vercel_ai_gateway_error_class():
"""Test error class creation"""
config = VercelAIGatewayConfig()
error_message = "Test error"
status_code = 400
headers = {"Content-Type": "application/json"}
error_class = config.get_error_class(error_message, status_code, headers)
assert isinstance(error_class, VercelAIGatewayException)
assert error_class.message == error_message
assert error_class.status_code == status_code
assert error_class.headers == headers
def test_vercel_ai_gateway_exception_inheritance():
"""Test that VercelAIGatewayException inherits from BaseLLMException"""
from litellm.llms.base_llm.chat.transformation import BaseLLMException
exception = VercelAIGatewayException(
message="test",
status_code=500,
headers={}
)
assert isinstance(exception, BaseLLMException)

228
tests/test_litellm/llms/vercel_ai_gateway/test_vercel_ai_gateway.py Executable file
View file

@ -0,0 +1,228 @@
"""
Mock tests for vercel_ai_gateway provider
"""
import json
from unittest.mock import MagicMock, patch
import pytest
import respx
import litellm
from litellm import completion
from litellm.llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
@pytest.fixture
def vercel_ai_gateway_response():
"""Mock response from Vercel AI Gateway API"""
return {
"id": "chatcmpl-vercel-123",
"object": "chat.completion",
"created": 1677652288,
"model": "openai/gpt-3.5-turbo",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello! This is a test response from Vercel AI Gateway."},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 15, "total_tokens": 25},
}
def test_vercel_ai_gateway_config_initialization():
"""Test VercelAIGatewayConfig initializes correctly"""
config = VercelAIGatewayConfig()
assert config.custom_llm_provider == "vercel_ai_gateway"
def test_get_llm_provider_vercel_ai_gateway():
"""Test that get_llm_provider correctly identifies vercel_ai_gateway"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
# Test with vercel_ai_gateway/provider/model-name format
model, provider, api_key, api_base = get_llm_provider("vercel_ai_gateway/openai/gpt-4o")
assert model == "openai/gpt-4o"
assert provider == "vercel_ai_gateway"
# Test with api_base containing vercel ai gateway endpoint
model, provider, api_key, api_base = get_llm_provider("gpt-4o", api_base="https://ai-gateway.vercel.sh/v1")
assert model == "gpt-4o"
assert provider == "vercel_ai_gateway"
assert api_base == "https://ai-gateway.vercel.sh/v1"
def test_vercel_ai_gateway_in_provider_lists():
"""Test that vercel_ai_gateway is registered in all necessary provider lists"""
assert "vercel_ai_gateway" in litellm.openai_compatible_providers
assert "vercel_ai_gateway" in litellm.provider_list
assert "https://ai-gateway.vercel.sh/v1" in litellm.openai_compatible_endpoints
@pytest.mark.asyncio
async def test_vercel_ai_gateway_completion_call(respx_mock, vercel_ai_gateway_response, monkeypatch):
"""Test completion call with vercel_ai_gateway provider using mocked response"""
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "test-api-key")
litellm.disable_aiohttp_transport = True
respx_mock.post("https://ai-gateway.vercel.sh/v1/chat/completions").respond(json=vercel_ai_gateway_response)
response = await litellm.acompletion(
model="vercel_ai_gateway/openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello, this is a test"}],
max_tokens=20,
)
assert response.choices[0].message.content == "Hello! This is a test response from Vercel AI Gateway."
assert response.model == "vercel_ai_gateway/openai/gpt-3.5-turbo"
assert response.usage.total_tokens == 25
assert len(respx_mock.calls) == 1
request = respx_mock.calls[0].request
assert request.method == "POST"
assert "ai-gateway.vercel.sh" in str(request.url)
assert "Authorization" in request.headers
assert request.headers["Authorization"] == "Bearer test-api-key"
@pytest.mark.asyncio
async def test_vercel_ai_gateway_with_oidc_token(respx_mock, vercel_ai_gateway_response, monkeypatch):
"""Test completion call with vercel_ai_gateway provider using VERCEL_OIDC_TOKEN"""
monkeypatch.setenv("VERCEL_OIDC_TOKEN", "test-oidc-token")
litellm.disable_aiohttp_transport = True
respx_mock.post("https://ai-gateway.vercel.sh/v1/chat/completions").respond(json=vercel_ai_gateway_response)
response = await litellm.acompletion(
model="vercel_ai_gateway/openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello, this is a test"}],
max_tokens=20,
)
assert response.choices[0].message.content == "Hello! This is a test response from Vercel AI Gateway."
assert response.model == "vercel_ai_gateway/openai/gpt-3.5-turbo"
assert response.usage.total_tokens == 25
assert len(respx_mock.calls) == 1
request = respx_mock.calls[0].request
assert "Authorization" in request.headers
assert request.headers["Authorization"] == "Bearer test-oidc-token"
def test_vercel_ai_gateway_supported_params():
"""Test that vercel_ai_gateway returns the supported parameters"""
config = VercelAIGatewayConfig()
supported_params = config.get_supported_openai_params("vercel_ai_gateway/openai/gpt-3.5-turbo")
# vercel_ai_gateway should include all base OpenAI params plus extra_body
expected_base_params = [
"frequency_penalty",
"logit_bias",
"logprobs",
"top_logprobs",
"max_tokens",
"max_completion_tokens",
"modalities",
"prediction",
"n",
"presence_penalty",
"seed",
"stop",
"stream",
"stream_options",
"temperature",
"top_p",
"tools",
"tool_choice",
"function_call",
"functions",
"max_retries",
"extra_headers",
"parallel_tool_calls",
"audio",
"web_search_options",
"extra_body",
]
for param in expected_base_params:
assert param in supported_params, f"Expected parameter '{param}' not found in supported params"
assert "extra_body" in supported_params
def test_vercel_ai_gateway_sync_completion(respx_mock, vercel_ai_gateway_response, monkeypatch):
"""Test synchronous completion call"""
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "test-api-key")
litellm.disable_aiohttp_transport = True
respx_mock.post("https://ai-gateway.vercel.sh/v1/chat/completions").respond(json=vercel_ai_gateway_response)
response = completion(
model="vercel_ai_gateway/openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello"}],
max_tokens=20,
)
assert response.choices[0].message.content == "Hello! This is a test response from Vercel AI Gateway."
assert response.model == "vercel_ai_gateway/openai/gpt-3.5-turbo"
assert response.usage.total_tokens == 25
def test_vercel_ai_gateway_with_provider_options(respx_mock, vercel_ai_gateway_response, monkeypatch):
"""Test vercel_ai_gateway with providerOptions parameter"""
monkeypatch.setenv("VERCEL_AI_GATEWAY_API_KEY", "test-api-key")
litellm.disable_aiohttp_transport = True
respx_mock.post("https://ai-gateway.vercel.sh/v1/chat/completions").respond(json=vercel_ai_gateway_response)
response = completion(
model="vercel_ai_gateway/openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello"}],
providerOptions={"gateway": {"order": ["azure", "openai"]}},
max_tokens=20,
)
assert response.choices[0].message.content == "Hello! This is a test response from Vercel AI Gateway."
assert response.model == "vercel_ai_gateway/openai/gpt-3.5-turbo"
assert response.usage.total_tokens == 25
assert len(respx_mock.calls) == 1
request = respx_mock.calls[0].request
request_data = json.loads(request.content.decode("utf-8"))
assert "providerOptions" in request_data
assert request_data["providerOptions"]["gateway"]["order"] == ["azure", "openai"]
def test_vercel_ai_gateway_models_endpoint():
"""Test the get_models functionality"""
config = VercelAIGatewayConfig()
with patch("litellm.module_level_client.get") as mock_get:
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"data": [{"id": "openai/gpt-4o"}, {"id": "openai/gpt-3.5-turbo"}, {"id": "anthropic/claude-3-sonnet"}]
}
mock_get.return_value = mock_response
models = config.get_models()
assert models == ["openai/gpt-4o", "openai/gpt-3.5-turbo", "anthropic/claude-3-sonnet"]
mock_get.assert_called_once_with(url="https://ai-gateway.vercel.sh/v1/models")
def test_vercel_ai_gateway_models_endpoint_failure():
"""Test the get_models functionality with failure"""
config = VercelAIGatewayConfig()
with patch("litellm.module_level_client.get") as mock_get:
mock_response = MagicMock()
mock_response.status_code = 404
mock_response.text = "Not found"
mock_get.return_value = mock_response
with pytest.raises(Exception, match="Failed to get models: Not found"):
config.get_models()

33
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -496,6 +496,36 @@ def test_vertex_ai_map_tool_with_anyof():
"anyOf": [{"type": "string", "nullable": True, "title": "Base Branch"}]
}, f"Expected only anyOf field and its contents to be kept, but got {tools[0]['function_declarations'][0]['parameters']['properties']['base_branch']}"
new_value = [
{
"type": "function",
"function": {
"name": "git_create_branch",
"description": "Creates a new branch from an optional base branch",
"parameters": {
"type": "object",
"properties": {
"repo_path": {"title": "Repo Path", "type": "string"},
"branch_name": {"title": "Branch Name", "type": "string"},
"base_branch": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"default": None,
},
},
"required": ["repo_path", "branch_name"],
"title": "GitCreateBranch",
},
},
}
]
new_tools = v._map_function(value=new_value)
assert new_tools[0]["function_declarations"][0]["parameters"]["properties"][
"base_branch"
] == {
"anyOf": [{"type": "string", "nullable": True}]
}, f"Expected only anyOf field and its contents to be kept, but got {new_tools[0]['function_declarations'][0]['parameters']['properties']['base_branch']}"
def test_vertex_ai_streaming_usage_calculation():
"""
@ -1003,10 +1033,11 @@ def test_vertex_ai_code_line_length():
This is a meta-test to ensure the code change meets the 40-character requirement.
"""
import inspect
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
# Get the source code of the _transform_parts method
source_lines = inspect.getsource(VertexGeminiConfig._transform_parts).split('\n')

34
tests/test_litellm/llms/vertex_ai/test_vertex.py
View file

@ -1469,3 +1469,37 @@ def test_vertex_parallel_tool_calls_false_single_tool():
parallel_tool_calls=False,
)
assert "tools" in optional_params
from litellm.llms.vertex_ai.gemini.transformation import _transform_request_body
def test_system_prompt_only_adds_blank_user_message():
"""
Test that the system prompt only adds a blank user message when a system message is passed in.
Relevant Issue - https://github.com/BerriAI/litellm/issues/13769
"""
SYSTEM_INSTRUCTION = "System instructions for the model"
data = _transform_request_body(
messages=[{"role": "system", "content": SYSTEM_INSTRUCTION}],
model="gemini-2.5-flash",
optional_params={},
custom_llm_provider="vertex_ai",
litellm_params={},
cached_content=None,
)
print("Final data: ", data)
# validate that a blank user message is added when a system message is passed in
assert len(data["contents"]) == 1
first_content = data["contents"][0]
assert first_content["role"] == "user"
assert len(first_content["parts"]) == 1
#########################################################
# system message was passed in
#########################################################
assert len(data["system_instruction"]) == 1
assert data["system_instruction"]["parts"][0]["text"] == SYSTEM_INSTRUCTION

23
tests/test_litellm/proxy/auth/test_route_checks.py
View file

@ -130,6 +130,29 @@ def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied():
assert "Only allowed to call routes: ['info_routes']" in str(exc_info.value)
assert "Tried to call route: /chat/completions" in str(exc_info.value)
@pytest.mark.parametrize("route", [
"/anthropic/v1/messages",
"/anthropic/v1/count_tokens",
"/gemini/v1/models",
"/gemini/countTokens",
])
def test_virtual_key_llm_api_route_includes_passthrough_prefix(route):
"""
Virtual key with llm_api_routes should allow passthrough routes like /anthropic/v1/messages
Relevant issue: https://github.com/BerriAI/litellm/issues/14017
"""
valid_token = UserAPIKeyAuth(
user_id="test_user", allowed_routes=["llm_api_routes"]
)
result = RouteChecks.is_virtual_key_allowed_to_call_route(
route=route, valid_token=valid_token
)
assert result is True
def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names():
"""Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes"""

138
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_pangea.py
View file

@ -75,6 +75,7 @@ async def test_pangea_ai_guard_request_blocked(pangea_guardrail):
},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with pytest.raises(HTTPException, match="Violated Pangea guardrail policy"):
with patch(
@ -82,9 +83,9 @@ async def test_pangea_ai_guard_request_blocked(pangea_guardrail):
return_value=httpx.Response(
status_code=200,
# Mock only tested part of response
json={"result": {"blocked": True, "prompt_messages": data["messages"]}},
json={"result": {"blocked": True, "transformed": False}},
request=httpx.Request(
method="POST", url=pangea_guardrail.guardrail_endpoint
method="POST", url=guardrail_endpoint,
),
),
) as mock_method:
@ -94,7 +95,52 @@ async def test_pangea_ai_guard_request_blocked(pangea_guardrail):
called_kwargs = mock_method.call_args.kwargs
assert called_kwargs["json"]["recipe"] == "guard_llm_request"
assert called_kwargs["json"]["messages"] == data["messages"]
assert called_kwargs["json"]["input"]["messages"] == data["messages"]
@pytest.mark.asyncio
async def test_pangea_ai_guard_request_transformed(pangea_guardrail):
data = {
"messages": [
{"role": "system", "content": "You are a helpful assistant"},
{
"role": "user",
"content": "Here is an SSN for one my employees: 078-05-1120",
},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=httpx.Response(
status_code=200,
# Mock only tested part of response
json={
"result": {
"blocked": False,
"transformed": True,
"output": {
"messages": [
{"role": "system", "content": "You are a helpful assistant"},
{
"role": "user",
"content": "Here is an SSN for one my employees: <US_SSN>",
},
]
},
},
},
request=httpx.Request(
method="POST", url=guardrail_endpoint,
),
),
):
request = await pangea_guardrail.async_pre_call_hook(
user_api_key_dict=None, cache=None, data=data, call_type="completion"
)
assert request["messages"][1]["content"] == "Here is an SSN for one my employees: <US_SSN>"
@pytest.mark.asyncio
@ -109,15 +155,16 @@ async def test_pangea_ai_guard_request_ok(pangea_guardrail):
},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=httpx.Response(
status_code=200,
# Mock only tested part of response
json={"result": {"blocked": False, "prompt_messages": data["messages"]}},
json={"result": {"blocked": False, "transformed": False}},
request=httpx.Request(
method="POST", url=pangea_guardrail.guardrail_endpoint
method="POST", url=guardrail_endpoint,
),
),
) as mock_method:
@ -127,7 +174,7 @@ async def test_pangea_ai_guard_request_ok(pangea_guardrail):
called_kwargs = mock_method.call_args.kwargs
assert called_kwargs["json"]["recipe"] == "guard_llm_request"
assert called_kwargs["json"]["messages"] == data["messages"]
assert called_kwargs["json"]["input"]["messages"] == data["messages"]
@pytest.mark.asyncio
@ -139,6 +186,7 @@ async def test_pangea_ai_guard_response_blocked(pangea_guardrail):
{"role": "user", "content": "Hello"},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with pytest.raises(HTTPException, match="Violated Pangea guardrail policy"):
with patch(
@ -149,16 +197,11 @@ async def test_pangea_ai_guard_response_blocked(pangea_guardrail):
json={
"result": {
"blocked": True,
"prompt_messages": [
{
"role": "assistant",
"content": "Yes, I will leak all my PII for you",
}
],
"transformed": False,
}
},
request=httpx.Request(
method="POST", url=pangea_guardrail.guardrail_endpoint
method="POST", url=guardrail_endpoint,
),
),
) as mock_method:
@ -180,7 +223,7 @@ async def test_pangea_ai_guard_response_blocked(pangea_guardrail):
called_kwargs = mock_method.call_args.kwargs
assert called_kwargs["json"]["recipe"] == "guard_llm_response"
assert (
called_kwargs["json"]["messages"][0]["content"]
called_kwargs["json"]["input"]["choices"][0]["message"]["content"]
== "Yes, I will leak all my PII for you"
)
@ -194,6 +237,7 @@ async def test_pangea_ai_guard_response_ok(pangea_guardrail):
{"role": "user", "content": "Hello"},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
@ -203,16 +247,11 @@ async def test_pangea_ai_guard_response_ok(pangea_guardrail):
json={
"result": {
"blocked": False,
"prompt_messages": [
{
"role": "assistant",
"content": "Yes, I will leak all my PII for you",
}
],
"transformed": False,
}
},
request=httpx.Request(
method="POST", url=pangea_guardrail.guardrail_endpoint
method="POST", url=guardrail_endpoint,
),
),
) as mock_method:
@ -234,6 +273,61 @@ async def test_pangea_ai_guard_response_ok(pangea_guardrail):
called_kwargs = mock_method.call_args.kwargs
assert called_kwargs["json"]["recipe"] == "guard_llm_response"
assert (
called_kwargs["json"]["messages"][0]["content"]
called_kwargs["json"]["input"]["choices"][0]["message"]["content"]
== "Yes, I will leak all my PII for you"
)
@pytest.mark.asyncio
async def test_pangea_ai_guard_response_transformed(pangea_guardrail):
# Content of data isn't that import since its mocked
data = {
"messages": [
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"},
]
}
guardrail_endpoint = f"{pangea_guardrail.api_base}/v1beta/guard"
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
return_value=httpx.Response(
status_code=200,
# Mock only tested part of response
json={
"result": {
"blocked": False,
"transformed": True,
"output": {
"messages": data["messages"],
"choices": [
{
"message": {
"role": "assistant",
"content": "Yes, here is an SSN: <US_SSN>",
},
},
],
},
},
},
request=httpx.Request(
method="POST", url=guardrail_endpoint,
),
),
):
response = await pangea_guardrail.async_post_call_success_hook(
data=data,
user_api_key_dict=None,
response=ModelResponse(
choices=[
{
"message": {
"role": "assistant",
"content": "Yes, here is an SSN: 078-05-1120",
}
}
]
),
)
assert response.choices[0]["message"]["content"] == "Yes, here is an SSN: <US_SSN>"

Some files were not shown because too many files have changed in this diff Show more