scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into litellm_refactor_01

Browse source
This commit is contained in:
Alexsander Hamir 2025-11-26 08:56:40 -08:00 • committed by GitHub
commit eacf20fddb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
197 changed files with 17684 additions and 1060 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -129,6 +129,10 @@ spec:
args:
- --config
- /etc/litellm/config.yaml
{{ if .Values.numWorkers }}
- --num_workers
- {{ .Values.numWorkers | quote }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
@ -208,3 +212,8 @@ spec:
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }}
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
{{- end }}

39
deploy/charts/litellm-helm/templates/servicemonitor.yaml Normal file
View file

@ -0,0 +1,39 @@
{{- with .Values.serviceMonitor }}
{{- if and (eq .enabled true) }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "litellm.fullname" $ }}
labels:
{{- include "litellm.labels" $ | nindent 4 }}
{{- if .labels }}
{{- toYaml .labels | nindent 4 }}
{{- end }}
{{- if .annotations }}
annotations:
{{- toYaml .annotations | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "litellm.selectorLabels" $ | nindent 6 }}
namespaceSelector:
matchNames:
# if not set, use the release namespace
{{- if not .namespaceSelector.matchNames }}
- {{ $.Release.Namespace | quote }}
{{- else }}
{{- toYaml .namespaceSelector.matchNames | nindent 4 }}
{{- end }}
endpoints:
- port: http
path: /metrics/
interval: {{ .interval }}
scrapeTimeout: {{ .scrapeTimeout }}
scheme: http
{{- if .relabelings }}
relabelings:
{{- toYaml .relabelings | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

152
deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml Normal file
View file

@ -0,0 +1,152 @@
{{- if .Values.serviceMonitor.enabled }}
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "litellm.fullname" . }}-test-servicemonitor"
labels:
{{- include "litellm.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test
image: bitnami/kubectl:latest
command: ['sh', '-c']
args:
- |
set -e
echo "🔍 Testing ServiceMonitor configuration..."
# Check if ServiceMonitor exists
if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then
echo "❌ ServiceMonitor not found"
exit 1
fi
echo "✅ ServiceMonitor exists"
# Get ServiceMonitor YAML
SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml)
# Test endpoint configuration
ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}')
if [ "$ENDPOINT_PORT" != "http" ]; then
echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT"
exit 1
fi
echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT"
# Test endpoint path
ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}')
if [ "$ENDPOINT_PATH" != "/metrics/" ]; then
echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH"
exit 1
fi
echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH"
# Test interval
INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}')
if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then
echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL"
exit 1
fi
echo "✅ Interval is correctly set to: $INTERVAL"
# Test scrapeTimeout
TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}')
if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then
echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT"
exit 1
fi
echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT"
# Test scheme
SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}')
if [ "$SCHEME" != "http" ]; then
echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME"
exit 1
fi
echo "✅ Scheme is correctly set to: $SCHEME"
{{- if .Values.serviceMonitor.labels }}
# Test custom labels
echo "🔍 Checking custom labels..."
{{- range $key, $value := .Values.serviceMonitor.labels }}
LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}')
if [ "$LABEL_VALUE" != "{{ $value }}" ]; then
echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE"
exit 1
fi
echo "✅ Label {{ $key }} is correctly set to: {{ $value }}"
{{- end }}
{{- end }}
{{- if .Values.serviceMonitor.annotations }}
# Test annotations
echo "🔍 Checking annotations..."
{{- range $key, $value := .Values.serviceMonitor.annotations }}
ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}')
if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then
echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE"
exit 1
fi
echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}"
{{- end }}
{{- end }}
{{- if .Values.serviceMonitor.namespaceSelector.matchNames }}
# Test namespace selector
echo "🔍 Checking namespace selector..."
{{- range .Values.serviceMonitor.namespaceSelector.matchNames }}
if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then
echo "❌ Namespace {{ . }} not found in namespaceSelector"
exit 1
fi
echo "✅ Namespace {{ . }} found in namespaceSelector"
{{- end }}
{{- else }}
# Test default namespace selector (should be release namespace)
if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then
echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector"
exit 1
fi
echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}"
{{- end }}
{{- if .Values.serviceMonitor.relabelings }}
# Test relabelings
echo "🔍 Checking relabelings configuration..."
if ! echo "$SM" | grep -q "relabelings:"; then
echo "❌ Relabelings section not found"
exit 1
fi
echo "✅ Relabelings section exists"
{{- range .Values.serviceMonitor.relabelings }}
{{- if .targetLabel }}
if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then
echo "❌ Relabeling targetLabel {{ .targetLabel }} not found"
exit 1
fi
echo "✅ Relabeling targetLabel {{ .targetLabel }} found"
{{- end }}
{{- if .action }}
if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then
echo "❌ Relabeling action {{ .action }} not found"
exit 1
fi
echo "✅ Relabeling action {{ .action }} found"
{{- end }}
{{- end }}
{{- end }}
# Test selector labels match the service
echo "🔍 Checking selector labels match service..."
SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}')
echo "Service labels: $SVC_LABELS"
echo "✅ Selector labels validation passed"
echo ""
echo "🎉 All ServiceMonitor tests passed successfully!"
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
restartPolicy: Never
{{- end }}

26
deploy/charts/litellm-helm/values.yaml
View file

@ -3,6 +3,7 @@
# Declare variables to be passed into your templates.
replicaCount: 1
# numWorkers: 2
image:
# Use "ghcr.io/berriai/litellm-database" for optimized image with database
@ -33,6 +34,15 @@ deploymentAnnotations: {}
podAnnotations: {}
podLabels: {}
terminationGracePeriodSeconds: 90
topologySpreadConstraints: []
# - maxSkew: 1
# topologyKey: kubernetes.io/hostname
# whenUnsatisfiable: DoNotSchedule
# labelSelector:
# matchLabels:
# app: litellm
# At the time of writing, the litellm docker image requires write access to the
# filesystem on startup so that prisma can install some dependencies.
podSecurityContext: {}
@ -248,3 +258,19 @@ pdb:
maxUnavailable: null # e.g. 1 or "20%"
annotations: {}
labels: {}
serviceMonitor:
enabled: false
labels: {}
# test: test
annotations: {}
# kubernetes.io/test: test
interval: 15s
scrapeTimeout: 10s
relabelings: []
# - targetLabel: __meta_kubernetes_pod_node_name
# replacement: $1
# action: replace
namespaceSelector:
matchNames: []
# - test-namespace

36
docker/Dockerfile.non_root
View file

@ -20,27 +20,33 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI
RUN mkdir -p /tmp/litellm_ui && \
npm install -g npm@latest && \
npm cache clean --force && \
cd ui/litellm-dashboard && \
if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r ./out/* /tmp/litellm_ui/ && \
cd /tmp/litellm_ui && \
RUN mkdir -p /tmp/litellm_ui
RUN npm install -g npm@latest && npm cache clean --force
RUN cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi
RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps
RUN cd /app/ui/litellm-dashboard && npm run build
RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
RUN cd /tmp/litellm_ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
mkdir -p "$folder_name" && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done && \
cd /app/ui/litellm-dashboard && \
rm -rf ./out
done
RUN cd /app/ui/litellm-dashboard && rm -rf ./out
# Build package and wheel dependencies
RUN rm -rf dist/* && python -m build && \

1281
docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md Normal file
View file

File diff suppressed because it is too large Load diff

4
docs/my-website/docs/completion/image_generation_chat.md
View file

@ -224,8 +224,8 @@ asyncio.run(generate_image())
| Provider | Model |
|----------|--------|
| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview` |
| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview`, `gemini/gemini-3-pro-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview`, `vertex_ai/gemini-3-pro-image-preview` |
## Spec

1
docs/my-website/docs/completion/knowledgebase.md
View file

@ -20,6 +20,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.)
- [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
- [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search)
## Quick Start

35
docs/my-website/docs/mcp.md
View file

@ -248,6 +248,41 @@ mcp_servers:
X-Custom-Header: "some-value"
```
### MCP Walkthroughs
- **Strands (STDIO)** – [watch tutorial](https://screen.studio/share/ruv4D73F)
> Add it from the UI
```json title="strands-mcp" showLineNumbers
{
"mcpServers": {
"strands-agents": {
"command": "uvx",
"args": ["strands-agents-mcp-server"],
"env": {
"FASTMCP_LOG_LEVEL": "INFO"
},
"disabled": false,
"autoApprove": ["search_docs", "fetch_doc"]
}
}
}
```
> config.yml
```yaml title="config.yml – strands MCP" showLineNumbers
mcp_servers:
strands_mcp:
transport: "stdio"
command: "uvx"
args: ["strands-agents-mcp-server"]
env:
FASTMCP_LOG_LEVEL: "INFO"
```
### MCP Aliases
You can define aliases for your MCP servers in the `litellm_settings` section. This allows you to:

6
docs/my-website/docs/observability/custom_callback.md
View file

@ -203,7 +203,11 @@ asyncio.run(test_chat_openai())
## What's Available in kwargs?
The kwargs dictionary contains all the details about your API call:
The kwargs dictionary contains all the details about your API call.
:::info
For the complete logging payload specification, see the [Standard Logging Payload Spec](https://docs.litellm.ai/docs/proxy/logging_spec).
:::
```python
def custom_callback(kwargs, completion_response, start_time, end_time):

26
docs/my-website/docs/providers/anthropic.md
View file

@ -18,11 +18,11 @@ LiteLLM supports all anthropic models.
| Property | Details |
|-------|-------|
| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. |
| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) |
| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) |
| API Endpoint for Provider | https://api.anthropic.com |
| Supported Endpoints | `/chat/completions` |
| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. Also available via Azure Foundry. |
| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`). For Azure Foundry deployments, use `azure/claude-*` (see [Azure Anthropic documentation](../providers/azure/azure_anthropic)) |
| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) |
| API Endpoint for Provider | https://api.anthropic.com (or Azure Foundry endpoint: `https://<resource-name>.services.ai.azure.com/anthropic`) |
| Supported Endpoints | `/chat/completions`, `/v1/messages` (passthrough) |
## Supported OpenAI Parameters
@ -163,6 +163,22 @@ os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
# os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
```
:::tip Azure Foundry Support
Claude models are also available via Microsoft Azure Foundry. Use the `azure/` prefix instead of `anthropic/` and configure Azure authentication. See the [Azure Anthropic documentation](../providers/azure/azure_anthropic) for details.
Example:
```python
response = completion(
model="azure/claude-sonnet-4-5",
api_base="https://<resource-name>.services.ai.azure.com/anthropic",
api_key="your-azure-api-key",
messages=[{"role": "user", "content": "Hello!"}]
)
```
:::
### Custom API Base
When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.

279
docs/my-website/docs/providers/anthropic_effort.md Normal file
View file

@ -0,0 +1,279 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Anthropic Effort Parameter
Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency.
## Overview
The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected).
## How Effort Works
By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability.
**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely.
The effort parameter affects **all tokens** in the response, including:
- Text responses and explanations
- Tool calls and function arguments
- Extended thinking (when enabled)
This approach has two major advantages:
1. It doesn't require thinking to be enabled in order to use it.
2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls.
This gives a much greater degree of control over efficiency.
## Effort Levels
| Level | Description | Typical use case |
|-------|-------------|------------------|
| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
## Quick Start
### Using LiteLLM SDK
<Tabs>
<TabItem value="python" label="Python">
```python
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
messages=[{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
output_config={
"effort": "medium"
}
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="typescript" label="TypeScript">
```typescript
import Anthropic from "@anthropic-ai/sdk";
const client = new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY,
});
const response = await client.messages.create({
model: "claude-opus-4-5-20251101",
max_tokens: 4096,
messages: [{
role: "user",
content: "Analyze the trade-offs between microservices and monolithic architectures"
}],
output_config: {
effort: "medium"
}
});
console.log(response.content[0].text);
```
</TabItem>
</Tabs>
### Using LiteLLM Proxy
```bash
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "anthropic/claude-opus-4-5-20251101",
"messages": [{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
"output_config": {
"effort": "medium"
}
}'
```
### Direct Anthropic API Call
```bash
curl https://api.anthropic.com/v1/messages \
--header "x-api-key: $ANTHROPIC_API_KEY" \
--header "anthropic-version: 2023-06-01" \
--header "anthropic-beta: effort-2025-11-24" \
--header "content-type: application/json" \
--data '{
"model": "claude-opus-4-5-20251101",
"max_tokens": 4096,
"messages": [{
"role": "user",
"content": "Analyze the trade-offs between microservices and monolithic architectures"
}],
"output_config": {
"effort": "medium"
}
}'
```
## Model Compatibility
The effort parameter is currently only supported by:
- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
## When Should I Adjust the Effort Parameter?
- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority.
- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort.
- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend.
## Effort with Tool Use
When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to:
- Combine multiple operations into fewer tool calls
- Make fewer tool calls
- Proceed directly to action
Example with tools:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
messages=[{
"role": "user",
"content": "Check the weather in multiple cities"
}],
tools=[{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}],
output_config={
"effort": "low" # Will make fewer tool calls
}
)
```
## Effort with Extended Thinking
The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
messages=[{
"role": "user",
"content": "Solve this complex problem"
}],
thinking={
"type": "enabled",
"budget_tokens": 5000
},
output_config={
"effort": "medium" # Affects both thinking and response tokens
}
)
```
## Best Practices
1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs.
2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency.
3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses.
4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases.
5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity.
## Provider Support
The effort parameter is supported across all Anthropic-compatible providers:
- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5)
- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5)
- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5)
LiteLLM automatically handles the beta header injection for all providers.
## Usage and Pricing
Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs:
```python
response = litellm.completion(
model="anthropic/claude-opus-4-5-20251101",
messages=[{"role": "user", "content": "Analyze this"}],
output_config={"effort": "low"}
)
print(f"Output tokens: {response.usage.completion_tokens}")
print(f"Total tokens: {response.usage.total_tokens}")
```
## Troubleshooting
### Beta header not being added
LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header:
1. Ensure you're using `output_config` with an `effort` field
2. Verify the model is Claude Opus 4.5
3. Check that LiteLLM version supports this feature
### Invalid effort value error
Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
```python
# ❌ This will raise an error
output_config={"effort": "very_low"}
# ✅ Use one of the valid values
output_config={"effort": "low"}
```
### Model not supported
Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
## Related Features
- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process
- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions
- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools
- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs
## Additional Resources
- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort)
- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic)
- [Cost Optimization Best Practices](/docs/guides/cost_optimization)

430
docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md Normal file
View file

@ -0,0 +1,430 @@
# Anthropic Programmatic Tool Calling
Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window.
:::info
Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field.
This feature requires the code execution tool to be enabled.
:::
## Model Compatibility
Programmatic tool calling is available on the following models:
| Model | Tool Version |
|-------|--------------|
| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` |
| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` |
## Quick Start
Here's a simple example where Claude programmatically queries a database multiple times and aggregates results:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{
"role": "user",
"content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue"
}
],
tools=[
{
"type": "code_execution_20250825",
"name": "code_execution"
},
{
"type": "function",
"function": {
"name": "query_database",
"description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
"parameters": {
"type": "object",
"properties": {
"sql": {
"type": "string",
"description": "SQL query to execute"
}
},
"required": ["sql"]
}
},
"allowed_callers": ["code_execution_20250825"]
}
]
)
print(response)
```
## How It Works
When you configure a tool to be callable from code execution and Claude decides to use that tool:
1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic
2. Claude runs this code in a sandboxed container via code execution
3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field
4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window)
5. Once all code execution completes, Claude receives the final output and continues working on the task
This approach is particularly useful for:
- **Large data processing**: Filter or aggregate tool results before they reach Claude's context
- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls
- **Conditional logic**: Make decisions based on intermediate tool results
## The `allowed_callers` Field
The `allowed_callers` field specifies which contexts can invoke a tool:
```python
{
"type": "function",
"function": {
"name": "query_database",
"description": "Execute a SQL query against the database",
"parameters": {...}
},
"allowed_callers": ["code_execution_20250825"]
}
```
**Possible values:**
- `["direct"]` - Only Claude can call this tool directly (default if omitted)
- `["code_execution_20250825"]` - Only callable from within code execution
- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution
:::tip
We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool.
:::
## The `caller` Field in Responses
Every tool use block includes a `caller` field indicating how it was invoked:
**Direct invocation (traditional tool use):**
```python
{
"type": "tool_use",
"id": "toolu_abc123",
"name": "query_database",
"input": {"sql": "<sql>"},
"caller": {"type": "direct"}
}
```
**Programmatic invocation:**
```python
{
"type": "tool_use",
"id": "toolu_xyz789",
"name": "query_database",
"input": {"sql": "<sql>"},
"caller": {
"type": "code_execution_20250825",
"tool_id": "srvtoolu_abc123"
}
}
```
The `tool_id` references the code execution tool that made the programmatic call.
## Container Lifecycle
Programmatic tool calling uses code execution containers:
- **Container creation**: A new container is created for each session unless you reuse an existing one
- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change)
- **Container ID**: Pass the `container` parameter to reuse an existing container
- **Reuse**: Pass the container ID to maintain state across requests
```python
# First request - creates a new container
response1 = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Query the database"}],
tools=[...]
)
# Get container ID from response (if available in response metadata)
container_id = response1.get("container", {}).get("id")
# Second request - reuse the same container
response2 = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[...],
tools=[...],
container=container_id # Reuse container
)
```
:::warning
When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it.
:::
## Example Workflow
### Step 1: Initial Request
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{
"role": "user",
"content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue"
}],
tools=[
{
"type": "code_execution_20250825",
"name": "code_execution"
},
{
"type": "function",
"function": {
"name": "query_database",
"description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
"parameters": {
"type": "object",
"properties": {
"sql": {"type": "string", "description": "SQL query to execute"}
},
"required": ["sql"]
}
},
"allowed_callers": ["code_execution_20250825"]
}
]
)
```
### Step 2: API Response with Tool Call
Claude writes code that calls your tool. The response includes:
```python
{
"role": "assistant",
"content": [
{
"type": "text",
"text": "I'll query the purchase history and analyze the results."
},
{
"type": "server_tool_use",
"id": "srvtoolu_abc123",
"name": "code_execution",
"input": {
"code": "results = await query_database('<sql>')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]"
}
},
{
"type": "tool_use",
"id": "toolu_def456",
"name": "query_database",
"input": {"sql": "<sql>"},
"caller": {
"type": "code_execution_20250825",
"tool_id": "srvtoolu_abc123"
}
}
],
"stop_reason": "tool_use"
}
```
### Step 3: Provide Tool Result
```python
# Add assistant's response and tool result to conversation
messages = [
{"role": "user", "content": "Query customer purchase history..."},
{
"role": "assistant",
"content": response.choices[0].message.content,
"tool_calls": response.choices[0].message.tool_calls
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_def456",
"content": '[{"customer_id": "C1", "revenue": 45000}, ...]'
}
]
}
]
# Continue the conversation
response2 = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=messages,
tools=[...]
)
```
### Step 4: Final Response
Once code execution completes, Claude provides the final response:
```python
{
"content": [
{
"type": "code_execution_tool_result",
"tool_use_id": "srvtoolu_abc123",
"content": {
"type": "code_execution_result",
"stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...",
"stderr": "",
"return_code": 0
}
},
{
"type": "text",
"text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..."
}
],
"stop_reason": "end_turn"
}
```
## Advanced Patterns
### Batch Processing with Loops
Claude can write code that processes multiple items efficiently:
```python
# Claude writes code like this:
regions = ["West", "East", "Central", "North", "South"]
results = {}
for region in regions:
data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'")
results[region] = data[0]["total"]
top_region = max(results.items(), key=lambda x: x[1])
print(f"Top region: {top_region[0]} with ${top_region[1]:,}")
```
This pattern:
- Reduces model round-trips from N (one per region) to 1
- Processes large result sets programmatically before returning to Claude
- Saves tokens by only returning aggregated conclusions
### Early Termination
Claude can stop processing as soon as success criteria are met:
```python
endpoints = ["us-east", "eu-west", "apac"]
for endpoint in endpoints:
status = await check_health(endpoint)
if status == "healthy":
print(f"Found healthy endpoint: {endpoint}")
break # Stop early
```
### Data Filtering
```python
logs = await fetch_logs(server_id)
errors = [log for log in logs if "ERROR" in log]
print(f"Found {len(errors)} errors")
for error in errors[-10:]: # Only return last 10 errors
print(error)
```
## Best Practices
### Tool Design
- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.)
- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing
- **Keep responses concise**: Return only necessary data to minimize processing overhead
### When to Use Programmatic Calling
**Good use cases:**
- Processing large datasets where you only need aggregates or summaries
- Multi-step workflows with 3+ dependent tool calls
- Operations requiring filtering, sorting, or transformation of tool results
- Tasks where intermediate data shouldn't influence Claude's reasoning
- Parallel operations across many items (e.g., checking 50 endpoints)
**Less ideal use cases:**
- Single tool calls with simple responses
- Tools that need immediate user feedback
- Very fast operations where code execution overhead would outweigh the benefit
## Token Efficiency
Programmatic tool calling can significantly reduce token consumption:
- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is
- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens
- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns
For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary.
## Provider Support
LiteLLM supports programmatic tool calling across all Anthropic-compatible providers:
- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field.
## Limitations
### Feature Incompatibilities
- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling
- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice`
- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling
### Tool Restrictions
The following tools cannot currently be called programmatically:
- Web search
- Web fetch
- Tools provided by an MCP connector
## Troubleshooting
### Common Issues
**"Tool not allowed" error**
- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]`
- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5)
**Container expiration**
- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes)
- Consider implementing faster tool execution
**Beta header not added**
- LiteLLM automatically adds the beta header when it detects `allowed_callers`
- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20`
## Related Features
- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation

438
docs/my-website/docs/providers/anthropic_tool_input_examples.md Normal file
View file

@ -0,0 +1,438 @@
# Anthropic Tool Input Examples
Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs.
:::info
Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field.
:::
## When to Use Input Examples
Input examples are most helpful for:
- **Complex nested objects**: Tools with deeply nested parameter structures
- **Optional parameters**: Showing when optional parameters should be included
- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.)
- **Enum values**: Illustrating valid enum choices in context
- **Edge cases**: Showing how to handle special cases
:::tip
**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient.
:::
## Quick Start
Add an `input_examples` field to your tool definition with an array of example input objects:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{"role": "user", "content": "What's the weather like in San Francisco?"}
],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "The unit of temperature"
}
},
"required": ["location"]
}
},
"input_examples": [
{
"location": "San Francisco, CA",
"unit": "fahrenheit"
},
{
"location": "Tokyo, Japan",
"unit": "celsius"
},
{
"location": "New York, NY" # 'unit' is optional
}
]
}
]
)
print(response)
```
## How It Works
When you provide `input_examples`:
1. **LiteLLM detects** the `input_examples` field in your tool definition
2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected
3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema
4. **Claude learns patterns**: The model uses examples to understand proper tool usage
5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats
## Example Formats
### Simple Tool with Examples
```python
{
"type": "function",
"function": {
"name": "send_email",
"description": "Send an email to a recipient",
"parameters": {
"type": "object",
"properties": {
"to": {"type": "string", "description": "Email address"},
"subject": {"type": "string"},
"body": {"type": "string"}
},
"required": ["to", "subject", "body"]
}
},
"input_examples": [
{
"to": "user@example.com",
"subject": "Meeting Reminder",
"body": "Don't forget our meeting tomorrow at 2 PM."
},
{
"to": "team@company.com",
"subject": "Weekly Update",
"body": "Here's this week's progress report..."
}
]
}
```
### Complex Nested Objects
```python
{
"type": "function",
"function": {
"name": "create_calendar_event",
"description": "Create a new calendar event",
"parameters": {
"type": "object",
"properties": {
"title": {"type": "string"},
"start": {
"type": "object",
"properties": {
"date": {"type": "string"},
"time": {"type": "string"}
}
},
"attendees": {
"type": "array",
"items": {
"type": "object",
"properties": {
"email": {"type": "string"},
"optional": {"type": "boolean"}
}
}
}
},
"required": ["title", "start"]
}
},
"input_examples": [
{
"title": "Team Standup",
"start": {
"date": "2025-01-15",
"time": "09:00"
},
"attendees": [
{"email": "alice@example.com", "optional": False},
{"email": "bob@example.com", "optional": True}
]
},
{
"title": "Lunch Break",
"start": {
"date": "2025-01-15",
"time": "12:00"
}
# No attendees - showing optional field
}
]
}
```
### Format-Sensitive Parameters
```python
{
"type": "function",
"function": {
"name": "search_flights",
"description": "Search for available flights",
"parameters": {
"type": "object",
"properties": {
"origin": {"type": "string", "description": "Airport code"},
"destination": {"type": "string", "description": "Airport code"},
"date": {"type": "string", "description": "Date in YYYY-MM-DD format"},
"passengers": {"type": "integer"}
},
"required": ["origin", "destination", "date"]
}
},
"input_examples": [
{
"origin": "SFO",
"destination": "JFK",
"date": "2025-03-15",
"passengers": 2
},
{
"origin": "LAX",
"destination": "ORD",
"date": "2025-04-20",
"passengers": 1
}
]
}
```
## Requirements and Limitations
### Schema Validation
- Each example **must be valid** according to the tool's `input_schema`
- Invalid examples will return a **400 error** from Anthropic
- Validation happens server-side (LiteLLM passes examples through)
### Server-Side Tools Not Supported
Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`:
- `web_search` (web search tool)
- `code_execution` (code execution tool)
- `computer_use` (computer use tool)
- `bash_tool` (bash execution tool)
- `text_editor` (text editor tool)
### Token Costs
Examples add to your prompt tokens:
- **Simple examples**: ~20-50 tokens per example
- **Complex nested objects**: ~100-200 tokens per example
- **Trade-off**: Higher token cost for better tool call accuracy
### Model Compatibility
Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header:
- Claude Opus 4.5 (`claude-opus-4-5-20251101`)
- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`)
- Claude Opus 4.1 (`claude-opus-4-1-20250805`)
:::note
On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples.
:::
## Best Practices
### 1. Show Diverse Examples
Include examples that demonstrate different use cases:
```python
"input_examples": [
{"location": "San Francisco, CA", "unit": "fahrenheit"}, # US city
{"location": "Tokyo, Japan", "unit": "celsius"}, # International
{"location": "New York, NY"} # Optional param omitted
]
```
### 2. Demonstrate Optional Parameters
Show when optional parameters should and shouldn't be included:
```python
"input_examples": [
{
"query": "machine learning",
"filters": {"year": 2024, "category": "research"} # With optional filters
},
{
"query": "artificial intelligence" # Without optional filters
}
]
```
### 3. Illustrate Format Requirements
Make format expectations clear through examples:
```python
"input_examples": [
{
"phone": "+1-555-123-4567", # Shows expected phone format
"date": "2025-01-15", # Shows date format (YYYY-MM-DD)
"time": "14:30" # Shows time format (HH:MM)
}
]
```
### 4. Keep Examples Realistic
Use realistic, production-like examples rather than placeholder data:
```python
# ✅ Good - realistic examples
"input_examples": [
{"email": "alice@company.com", "role": "admin"},
{"email": "bob@company.com", "role": "user"}
]
# ❌ Bad - placeholder examples
"input_examples": [
{"email": "test@test.com", "role": "role1"},
{"email": "example@example.com", "role": "role2"}
]
```
### 5. Limit Example Count
Provide 2-5 examples per tool:
- **Too few** (1): May not show enough variation
- **Just right** (2-5): Demonstrates patterns without bloating tokens
- **Too many** (10+): Wastes tokens, diminishing returns
## Integration with Other Features
Input examples work seamlessly with other Anthropic tool features:
### With Tool Search
```python
{
"type": "function",
"function": {
"name": "query_database",
"description": "Execute a SQL query",
"parameters": {...}
},
"defer_loading": True, # Tool search
"input_examples": [ # Input examples
{"sql": "SELECT * FROM users WHERE id = 1"}
]
}
```
### With Programmatic Tool Calling
```python
{
"type": "function",
"function": {
"name": "fetch_data",
"description": "Fetch data from API",
"parameters": {...}
},
"allowed_callers": ["code_execution_20250825"], # Programmatic calling
"input_examples": [ # Input examples
{"endpoint": "/api/users", "method": "GET"}
]
}
```
### All Features Combined
```python
{
"type": "function",
"function": {
"name": "advanced_tool",
"description": "A complex tool",
"parameters": {...}
},
"defer_loading": True, # Tool search
"allowed_callers": ["code_execution_20250825"], # Programmatic calling
"input_examples": [ # Input examples
{"param1": "value1", "param2": "value2"}
]
}
```
## Provider Support
LiteLLM supports input examples across all Anthropic-compatible providers:
- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
The beta header is automatically added when LiteLLM detects tools with `input_examples` field.
## Troubleshooting
### "Invalid request" error with examples
**Problem**: Receiving 400 error when using input examples
**Solution**: Ensure each example is valid according to your `input_schema`:
```python
# Check that:
# 1. All required fields are present in examples
# 2. Field types match the schema
# 3. Enum values are valid
# 4. Nested objects follow the schema structure
```
### Examples not improving tool calls
**Problem**: Adding examples doesn't seem to help
**Solution**:
1. **Check descriptions first**: Ensure tool descriptions are detailed and clear
2. **Review example quality**: Make sure examples are realistic and diverse
3. **Verify schema**: Confirm examples actually match your schema
4. **Add more variation**: Include examples showing different use cases
### Token usage too high
**Problem**: Input examples consuming too many tokens
**Solution**:
1. **Reduce example count**: Use 2-3 examples instead of 5+
2. **Simplify examples**: Remove unnecessary fields from examples
3. **Consider descriptions**: If descriptions are clear, examples may not be needed
## When NOT to Use Input Examples
Skip input examples if:
- **Tool is simple**: Single parameter tools with clear descriptions
- **Schema is self-explanatory**: Well-structured schema with good descriptions
- **Token budget is tight**: Examples add 20-200 tokens each
- **Server-side tools**: web_search, code_execution, etc. don't support examples
## Related Features
- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution
- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation

397
docs/my-website/docs/providers/anthropic_tool_search.md Normal file
View file

@ -0,0 +1,397 @@
# Anthropic Tool Search
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
## Benefits
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
- **On-demand loading**: Tools are only loaded when Claude needs them
## Supported Models
Tool search is available on:
- Claude Opus 4.5
- Claude Sonnet 4.5
## Supported Platforms
- Anthropic API (direct)
- Azure Anthropic (Microsoft Foundry)
- Google Cloud Vertex AI
- Amazon Bedrock (invoke API only, not converse API)
## Tool Search Variants
LiteLLM supports both tool search variants:
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
Claude constructs regex patterns to search for tools.
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
Claude uses natural language queries to search for tools using the BM25 algorithm.
## Quick Start
### Basic Example with Regex Tool Search
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{"role": "user", "content": "What is the weather in San Francisco?"}
],
tools=[
# Tool search tool (regex variant)
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
# Deferred tool - will be loaded on-demand
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather at a specific location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"]
}
},
"required": ["location"]
}
},
"defer_loading": True # Mark for deferred loading
},
# Another deferred tool
{
"type": "function",
"function": {
"name": "search_files",
"description": "Search through files in the workspace",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"},
"file_types": {
"type": "array",
"items": {"type": "string"}
}
},
"required": ["query"]
}
},
"defer_loading": True
}
]
)
print(response.choices[0].message.content)
```
### BM25 Tool Search Example
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{"role": "user", "content": "Search for Python files containing 'authentication'"}
],
tools=[
# Tool search tool (BM25 variant)
{
"type": "tool_search_tool_bm25_20251119",
"name": "tool_search_tool_bm25"
},
# Deferred tools...
{
"type": "function",
"function": {
"name": "search_codebase",
"description": "Search through codebase files by content and filename",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"},
"file_pattern": {"type": "string"}
},
"required": ["query"]
}
},
"defer_loading": True
}
]
)
```
## Using with Azure Anthropic
```python
import litellm
response = litellm.completion(
model="azure_anthropic/claude-sonnet-4-5",
api_base="https://<your-resource>.services.ai.azure.com/anthropic",
api_key="your-azure-api-key",
messages=[
{"role": "user", "content": "What's the weather like?"}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
},
"defer_loading": True
}
]
)
```
## Using with Vertex AI
```python
import litellm
response = litellm.completion(
model="vertex_ai/claude-sonnet-4-5",
vertex_project="your-project-id",
vertex_location="us-central1",
messages=[
{"role": "user", "content": "Search my documents"}
],
tools=[
{
"type": "tool_search_tool_bm25_20251119",
"name": "tool_search_tool_bm25"
},
# Your deferred tools...
]
)
```
## Streaming Support
Tool search works with streaming:
```python
import litellm
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{"role": "user", "content": "Get the weather"}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
},
"defer_loading": True
}
],
stream=True
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## LiteLLM Proxy
Tool search works automatically through the LiteLLM proxy:
### Proxy Config
```yaml
model_list:
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-5-20250929
api_key: os.environ/ANTHROPIC_API_KEY
```
### Client Request
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-proxy-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-sonnet",
messages=[
{"role": "user", "content": "What's the weather?"}
],
tools=[
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
},
"defer_loading": True
}
]
)
```
## Important Notes
### Beta Header
LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it.
### Deferred Loading
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
- At least one tool must be non-deferred (the tool search tool itself)
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
### Tool Descriptions
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
- Tool names
- Tool descriptions
- Argument names
- Argument descriptions
### Usage Tracking
Tool search requests are tracked in the usage object:
```python
response = litellm.completion(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[{"role": "user", "content": "Search for tools"}],
tools=[...]
)
# Check tool search usage
if response.usage.server_tool_use:
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
```
## Error Handling
### All Tools Deferred
```python
# ❌ This will fail - at least one tool must be non-deferred
tools = [
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
# ✅ Correct - tool search tool is non-deferred
tools = [
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex"
},
{
"type": "function",
"function": {...},
"defer_loading": True
}
]
```
### Missing Tool Definition
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
## Best Practices
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
3. **Choose the right variant**:
- Use **regex** for exact pattern matching (faster)
- Use **BM25** for natural language semantic search
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
## When to Use Tool Search
**Good use cases:**
- 10+ tools available in your system
- Tool definitions consuming >10K tokens
- Experiencing tool selection accuracy issues
- Building systems with multiple tool categories
- Tool library growing over time
**When traditional tool calling is better:**
- Less than 10 tools total
- All tools are frequently used
- Very small tool definitions (\<100 tokens total)
## Limitations
- Not compatible with tool use examples
- Requires Claude Opus 4.5 or Sonnet 4.5
- On Bedrock, only available via invoke API (not converse API)
- Maximum 10,000 tools in catalog
- Returns 3-5 most relevant tools per search
## Additional Resources
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)

14
docs/my-website/docs/providers/azure/azure.md
View file

@ -9,10 +9,10 @@ import TabItem from '@theme/TabItem';
| Property | Details |
|-------|-------|
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude)
## API Keys, Params
api_key, api_base, api_version etc can be passed directly to `litellm.completion` - see here or set as `litellm.api_key` params see here
@ -27,6 +27,12 @@ os.environ["AZURE_AD_TOKEN"] = ""
os.environ["AZURE_API_TYPE"] = ""
```
:::info Azure Foundry Claude Models
Azure also supports Claude models via Azure Foundry. Use `azure/claude-*` model names (e.g., `azure/claude-sonnet-4-5`) with Azure authentication. See the [Azure Anthropic documentation](./azure_anthropic) for details.
:::
## **Usage - LiteLLM Python SDK**
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_Azure_OpenAI.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>

378
docs/my-website/docs/providers/azure/azure_anthropic.md Normal file
View file

@ -0,0 +1,378 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure Anthropic (Claude via Azure Foundry)
LiteLLM supports Claude models deployed via Microsoft Azure Foundry, including Claude Sonnet 4.5, Claude Haiku 4.5, and Claude Opus 4.1.
## Available Models
Azure Foundry supports the following Claude models:
- `claude-sonnet-4-5` - Anthropic's most capable model for building real-world agents and handling complex, long-horizon tasks
- `claude-haiku-4-5` - Near-frontier performance with the right speed and cost for high-volume use cases
- `claude-opus-4-1` - Industry leader for coding, delivering sustained performance on long-running tasks
| Property | Details |
|-------|-------|
| Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. |
| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) |
| Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) |
| API Endpoint | `https://<resource-name>.services.ai.azure.com/anthropic/v1/messages` |
| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`|
## Key Features
- **Extended thinking**: Enhanced reasoning capabilities for complex tasks
- **Image and text input**: Strong vision capabilities for analyzing charts, graphs, technical diagrams, and reports
- **Code generation**: Advanced thinking with code generation, analysis, and debugging (Claude Sonnet 4.5 and Claude Opus 4.1)
- **Same API as Anthropic**: All request/response transformations are identical to the main Anthropic provider
## Authentication
Azure Anthropic supports two authentication methods:
1. **API Key**: Use the `api-key` header
2. **Azure AD Token**: Use `Authorization: Bearer <token>` header (Microsoft Entra ID)
## API Keys and Configuration
```python
import os
# Option 1: API Key authentication
os.environ["AZURE_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
# Option 2: Azure AD Token authentication
os.environ["AZURE_AD_TOKEN"] = "your-azure-ad-token"
os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
# Optional: Azure AD Token Provider (for automatic token refresh)
os.environ["AZURE_TENANT_ID"] = "your-tenant-id"
os.environ["AZURE_CLIENT_ID"] = "your-client-id"
os.environ["AZURE_CLIENT_SECRET"] = "your-client-secret"
os.environ["AZURE_SCOPE"] = "https://cognitiveservices.azure.com/.default"
```
## Usage - LiteLLM Python SDK
### Basic Completion
```python
from litellm import completion
# Set environment variables
os.environ["AZURE_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
# Make a completion request
response = completion(
model="azure/claude-sonnet-4-5",
messages=[
{"role": "user", "content": "What are 3 things to visit in Seattle?"}
],
max_tokens=1000,
temperature=0.7,
)
print(response)
```
### Completion with API Key Parameter
```python
import litellm
response = litellm.completion(
model="azure/claude-sonnet-4-5",
api_base="https://<resource-name>.services.ai.azure.com/anthropic",
api_key="your-azure-api-key",
messages=[
{"role": "user", "content": "Hello!"}
],
max_tokens=1000,
)
```
### Completion with Azure AD Token
```python
import litellm
response = litellm.completion(
model="azure/claude-sonnet-4-5",
api_base="https://<resource-name>.services.ai.azure.com/anthropic",
azure_ad_token="your-azure-ad-token",
messages=[
{"role": "user", "content": "Hello!"}
],
max_tokens=1000,
)
```
### Streaming
```python
from litellm import completion
response = completion(
model="azure/claude-sonnet-4-5",
messages=[
{"role": "user", "content": "Write a short story"}
],
stream=True,
max_tokens=1000,
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
```
### Tool Calling
```python
from litellm import completion
response = completion(
model="azure/claude-sonnet-4-5",
messages=[
{"role": "user", "content": "What's the weather in Seattle?"}
],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}
],
tool_choice="auto",
max_tokens=1000,
)
print(response)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export AZURE_API_KEY="your-azure-api-key"
export AZURE_API_BASE="https://<resource-name>.services.ai.azure.com/anthropic"
```
### 2. Configure the proxy
```yaml
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: azure/claude-sonnet-4-5
api_base: https://<resource-name>.services.ai.azure.com/anthropic
api_key: os.environ/AZURE_API_KEY
```
### 3. Test it
<Tabs>
<TabItem value="curl" label="curl">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "claude-sonnet-4-5",
"messages": [
{
"role": "user",
"content": "Hello!"
}
],
"max_tokens": 1000
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI Python SDK">
```python
from openai import OpenAI
client = OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-sonnet-4-5",
messages=[
{"role": "user", "content": "Hello!"}
],
max_tokens=1000
)
print(response)
```
</TabItem>
</Tabs>
## Messages API
Azure Anthropic also supports the native Anthropic Messages API. The endpoint structure is the same as Anthropic's `/v1/messages` API.
### Using Anthropic SDK
```python
from anthropic import Anthropic
client = Anthropic(
api_key="your-azure-api-key",
base_url="https://<resource-name>.services.ai.azure.com/anthropic"
)
response = client.messages.create(
model="claude-sonnet-4-5",
max_tokens=1000,
messages=[
{"role": "user", "content": "Hello, world"}
]
)
print(response)
```
### Using LiteLLM Proxy
```bash
curl --request POST \
--url http://0.0.0.0:4000/anthropic/v1/messages \
--header 'accept: application/json' \
--header 'content-type: application/json' \
--header "Authorization: bearer sk-anything" \
--data '{
"model": "claude-sonnet-4-5",
"max_tokens": 1024,
"messages": [
{"role": "user", "content": "Hello, world"}
]
}'
```
## Supported OpenAI Parameters
Azure Anthropic supports the same parameters as the main Anthropic provider:
```
"stream",
"stop",
"temperature",
"top_p",
"max_tokens",
"max_completion_tokens",
"tools",
"tool_choice",
"extra_headers",
"parallel_tool_calls",
"response_format",
"user",
"thinking",
"reasoning_effort"
```
:::info
Azure Anthropic API requires `max_tokens` to be passed. LiteLLM automatically passes `max_tokens=4096` when no `max_tokens` are provided.
:::
## Differences from Standard Anthropic Provider
The only difference between Azure Anthropic and the standard Anthropic provider is authentication:
- **Standard Anthropic**: Uses `x-api-key` header
- **Azure Anthropic**: Uses `api-key` header or `Authorization: Bearer <token>` for Azure AD authentication
All other request/response transformations, tool calling, streaming, and feature support are identical.
## API Base URL Format
The API base URL should follow this format:
```
https://<resource-name>.services.ai.azure.com/anthropic
```
LiteLLM will automatically append `/v1/messages` if not already present in the URL.
## Example: Full Configuration
```python
import os
from litellm import completion
# Configure Azure Anthropic
os.environ["AZURE_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
# Make a request
response = completion(
model="azure/claude-sonnet-4-5",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Explain quantum computing in simple terms."}
],
max_tokens=1000,
temperature=0.7,
stream=False,
)
print(response.choices[0].message.content)
```
## Troubleshooting
### Missing API Base Error
If you see an error about missing API base, ensure you've set:
```python
os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
```
Or pass it directly:
```python
response = completion(
model="azure/claude-sonnet-4-5",
api_base="https://<resource-name>.services.ai.azure.com/anthropic",
# ...
)
```
### Authentication Errors
- **API Key**: Ensure `AZURE_API_KEY` is set or passed as `api_key` parameter
- **Azure AD Token**: Ensure `AZURE_AD_TOKEN` is set or passed as `azure_ad_token` parameter
- **Token Provider**: For automatic token refresh, configure `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`
## Related Documentation
- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage
- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models
- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup

202
docs/my-website/docs/providers/bedrock.md
View file

@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
| Property | Details |
|-------|-------|
| Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) |
| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) |
| Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
| Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
| Rerank Endpoint | `/rerank` |
@ -1598,206 +1598,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## Bedrock Imported Models (Deepseek, Deepseek R1)
### Deepseek R1
This is a separate route, as the chat template is different.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/deepseek_r1/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: DeepSeek-R1-Distill-Llama-70B
litellm_params:
model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### Deepseek (not R1)
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/llama/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: DeepSeek-R1-Distill-Llama-70B
litellm_params:
model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### Qwen3 Imported Models
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/qwen3/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
max_tokens=100,
temperature=0.7
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: Qwen3-32B
litellm_params:
model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "Qwen3-32B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### OpenAI GPT OSS
| Property | Details |

369
docs/my-website/docs/providers/bedrock_imported.md Normal file
View file

@ -0,0 +1,369 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Bedrock Imported Models
Bedrock Imported Models (Deepseek, Deepseek R1, Qwen, OpenAI-compatible models)
### Deepseek R1
This is a separate route, as the chat template is different.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/deepseek_r1/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: DeepSeek-R1-Distill-Llama-70B
litellm_params:
model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### Deepseek (not R1)
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/llama/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: DeepSeek-R1-Distill-Llama-70B
litellm_params:
model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### Qwen3 Imported Models
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/qwen3/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
response = completion(
model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
max_tokens=100,
temperature=0.7
)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: Qwen3-32B
litellm_params:
model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "Qwen3-32B", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.)
Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features.
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/openai/{model_arn}` |
| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) |
| Supported Features | Vision (images), tool calling, streaming, system messages |
#### LiteLLMSDK Usage
**Basic Usage**
```python
from litellm import completion
response = completion(
model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", # bedrock/openai/{your-model-arn}
messages=[{"role": "user", "content": "Tell me a joke"}],
max_tokens=300,
temperature=0.5
)
```
**With Vision (Images)**
```python
import base64
from litellm import completion
# Load and encode image
with open("image.jpg", "rb") as f:
image_base64 = base64.b64encode(f.read()).decode("utf-8")
response = completion(
model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z",
messages=[
{
"role": "system",
"content": "You are a helpful assistant that can analyze images."
},
{
"role": "user",
"content": [
{"type": "text", "text": "What's in this image?"},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}
}
]
}
],
max_tokens=300,
temperature=0.5
)
```
**Comparing Multiple Images**
```python
import base64
from litellm import completion
# Load images
with open("image1.jpg", "rb") as f:
image1_base64 = base64.b64encode(f.read()).decode("utf-8")
with open("image2.jpg", "rb") as f:
image2_base64 = base64.b64encode(f.read()).decode("utf-8")
response = completion(
model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z",
messages=[
{
"role": "system",
"content": "You are a helpful assistant that can analyze images."
},
{
"role": "user",
"content": [
{"type": "text", "text": "Spot the difference between these two images?"},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{image1_base64}"}
},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{image2_base64}"}
}
]
}
],
max_tokens=300,
temperature=0.5
)
```
#### LiteLLM Proxy Usage (AI Gateway)
**1. Add to config**
```yaml
model_list:
- model_name: qwen-25vl-72b
litellm_params:
model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
Basic text request:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "qwen-25vl-72b",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"max_tokens": 300
}'
```
With vision (image):
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "qwen-25vl-72b",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant that can analyze images."
},
{
"role": "user",
"content": [
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZ..."}
}
]
}
],
"max_tokens": 300,
"temperature": 0.5
}'
```

241
docs/my-website/docs/providers/elevenlabs.md
View file

@ -7,10 +7,10 @@ ElevenLabs provides high-quality AI voice technology, including speech-to-text c
| Property | Details |
|----------|---------|
| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription capabilities that support multiple languages and speaker diarization. |
| Description | ElevenLabs offers advanced AI voice technology with speech-to-text transcription and text-to-speech capabilities that support multiple languages and speaker diarization. |
| Provider Route on LiteLLM | `elevenlabs/` |
| Provider Doc | [ElevenLabs API ↗](https://elevenlabs.io/docs/api-reference) |
| Supported Endpoints | `/audio/transcriptions` |
| Supported Endpoints | `/audio/transcriptions`, `/audio/speech` |
## Quick Start
@ -228,4 +228,241 @@ ElevenLabs returns transcription responses in OpenAI-compatible format:
1. **Invalid API Key**: Ensure `ELEVENLABS_API_KEY` is set correctly
---
## Text-to-Speech (TTS)
ElevenLabs provides high-quality text-to-speech capabilities through their TTS API, supporting multiple voices, languages, and audio formats.
### Overview
| Property | Details |
|----------|---------|
| Description | Convert text to natural-sounding speech using ElevenLabs' advanced TTS models |
| Provider Route on LiteLLM | `elevenlabs/` |
| Supported Operations | `/audio/speech` |
| Link to Provider Doc | [ElevenLabs TTS API ↗](https://elevenlabs.io/docs/api-reference/text-to-speech) |
### Quick Start
#### LiteLLM Python SDK
```python showLineNumbers title="ElevenLabs Text-to-Speech with SDK"
import litellm
import os
os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key"
# Basic usage with voice mapping
audio = litellm.speech(
model="elevenlabs/eleven_multilingual_v2",
input="Testing ElevenLabs speech from LiteLLM.",
voice="alloy", # Maps to ElevenLabs voice ID automatically
)
# Save audio to file
with open("test_output.mp3", "wb") as f:
f.write(audio.read())
```
#### Advanced Usage: Overriding Parameters and ElevenLabs-Specific Features
```python showLineNumbers title="Advanced TTS with custom parameters"
import litellm
import os
os.environ["ELEVENLABS_API_KEY"] = "your-elevenlabs-api-key"
# Example showing parameter overriding and ElevenLabs-specific parameters
audio = litellm.speech(
model="elevenlabs/eleven_multilingual_v2",
input="Testing ElevenLabs speech from LiteLLM.",
voice="alloy", # Can use mapped voice name or raw ElevenLabs voice_id
response_format="pcm", # Maps to ElevenLabs output_format
speed=1.1, # Maps to voice_settings.speed
# ElevenLabs-specific parameters - passed directly to API
pronunciation_dictionary_locators=[
{"pronunciation_dictionary_id": "dict_123", "version_id": "v1"}
],
model_id="eleven_multilingual_v2", # Override model if needed
)
# Save audio to file
with open("test_output.mp3", "wb") as f:
f.write(audio.read())
```
### Voice Mapping
LiteLLM automatically maps common OpenAI voice names to ElevenLabs voice IDs:
| OpenAI Voice | ElevenLabs Voice ID | Description |
|--------------|---------------------|-------------|
| `alloy` | `21m00Tcm4TlvDq8ikWAM` | Rachel - Neutral and balanced |
| `amber` | `5Q0t7uMcjvnagumLfvZi` | Paul - Warm and friendly |
| `ash` | `AZnzlk1XvdvUeBnXmlld` | Domi - Energetic |
| `august` | `D38z5RcWu1voky8WS1ja` | Fin - Professional |
| `blue` | `2EiwWnXFnvU5JabPnv8n` | Clyde - Deep and authoritative |
| `coral` | `9BWtsMINqrJLrRacOk9x` | Aria - Expressive |
| `lily` | `EXAVITQu4vr4xnSDxMaL` | Sarah - Friendly |
| `onyx` | `29vD33N1CtxCmqQRPOHJ` | Drew - Strong |
| `sage` | `CwhRBWXzGAHq8TQ4Fs17` | Roger - Calm |
| `verse` | `CYw3kZ02Hs0563khs1Fj` | Dave - Conversational |
**Using Custom Voice IDs**: You can also pass any ElevenLabs voice ID directly. If the voice name is not in the mapping, LiteLLM will use it as-is:
```python showLineNumbers title="Using custom ElevenLabs voice ID"
audio = litellm.speech(
model="elevenlabs/eleven_multilingual_v2",
input="Testing with a custom voice.",
voice="21m00Tcm4TlvDq8ikWAM", # Direct ElevenLabs voice ID
)
```
### Response Format Mapping
LiteLLM maps OpenAI response formats to ElevenLabs output formats:
| OpenAI Format | ElevenLabs Format |
|---------------|-------------------|
| `mp3` | `mp3_44100_128` |
| `pcm` | `pcm_44100` |
| `opus` | `opus_48000_128` |
You can also pass ElevenLabs-specific output formats directly using the `output_format` parameter.
### Supported Parameters
```python showLineNumbers title="All Supported Parameters"
audio = litellm.speech(
model="elevenlabs/eleven_multilingual_v2", # Required
input="Text to convert to speech", # Required
voice="alloy", # Required: Voice selection (mapped or raw ID)
response_format="mp3", # Optional: Audio format (mp3, pcm, opus)
speed=1.0, # Optional: Speech speed (maps to voice_settings.speed)
# ElevenLabs-specific parameters (passed directly):
model_id="eleven_multilingual_v2", # Optional: Override model
voice_settings={ # Optional: Voice customization
"stability": 0.5,
"similarity_boost": 0.75,
"speed": 1.0
},
pronunciation_dictionary_locators=[ # Optional: Custom pronunciation
{"pronunciation_dictionary_id": "dict_123", "version_id": "v1"}
],
)
```
### LiteLLM Proxy
#### 1. Configure your proxy
```yaml showLineNumbers title="ElevenLabs TTS configuration in config.yaml"
model_list:
- model_name: elevenlabs-tts
litellm_params:
model: elevenlabs/eleven_multilingual_v2
api_key: os.environ/ELEVENLABS_API_KEY
general_settings:
master_key: your-master-key
```
#### 2. Make TTS requests
##### Simple Usage (OpenAI Parameters)
You can use standard OpenAI-compatible parameters without any provider-specific configuration:
```bash showLineNumbers title="Simple TTS request with curl"
curl http://localhost:4000/v1/audio/speech \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "elevenlabs-tts",
"input": "Testing ElevenLabs speech via the LiteLLM proxy.",
"voice": "alloy",
"response_format": "mp3"
}' \
--output speech.mp3
```
```python showLineNumbers title="Simple TTS with OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
response = client.audio.speech.create(
model="elevenlabs-tts",
input="Testing ElevenLabs speech via the LiteLLM proxy.",
voice="alloy",
response_format="mp3"
)
# Save audio
with open("speech.mp3", "wb") as f:
f.write(response.content)
```
##### Advanced Usage (ElevenLabs-Specific Parameters)
**Note**: When using the proxy, provider-specific parameters (like `pronunciation_dictionary_locators`, `voice_settings`, etc.) must be passed in the `extra_body` field.
```bash showLineNumbers title="Advanced TTS request with curl"
curl http://localhost:4000/v1/audio/speech \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "elevenlabs-tts",
"input": "Testing ElevenLabs speech via the LiteLLM proxy.",
"voice": "alloy",
"response_format": "pcm",
"extra_body": {
"pronunciation_dictionary_locators": [
{"pronunciation_dictionary_id": "dict_123", "version_id": "v1"}
],
"voice_settings": {
"speed": 1.1,
"stability": 0.5,
"similarity_boost": 0.75
}
}
}' \
--output speech.mp3
```
```python showLineNumbers title="Advanced TTS with OpenAI SDK"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000",
api_key="your-litellm-api-key"
)
response = client.audio.speech.create(
model="elevenlabs-tts",
input="Testing ElevenLabs speech via the LiteLLM proxy.",
voice="alloy",
response_format="pcm",
extra_body={
"pronunciation_dictionary_locators": [
{"pronunciation_dictionary_id": "dict_123", "version_id": "v1"}
],
"voice_settings": {
"speed": 1.1,
"stability": 0.5,
"similarity_boost": 0.75
}
}
)
# Save audio
with open("speech.mp3", "wb") as f:
f.write(response.content)
```

4
docs/my-website/docs/providers/gemini.md
View file

@ -74,6 +74,10 @@ Note: Reasoning cannot be turned off on Gemini 2.5 Pro models.
For **Gemini 3+ models** (e.g., `gemini-3-pro-preview`), LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter instead of `thinking_budget`. The `thinking_level` parameter uses `"low"` or `"high"` values for better control over reasoning depth.
:::
:::warning Image Models
**Gemini image models** (e.g., `gemini-3-pro-image-preview`, `gemini-2.0-flash-exp-image-generation`) do **not** support the `thinking_level` parameter. LiteLLM automatically excludes image models from receiving thinking configuration to prevent API errors.
:::
**Mapping for Gemini 2.5 and earlier models**
| reasoning_effort | thinking | Notes |

414
docs/my-website/docs/providers/gemini_file_search.md Normal file
View file

@ -0,0 +1,414 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Gemini File Search
Use Google Gemini's File Search for Retrieval Augmented Generation (RAG) with LiteLLM.
Gemini File Search imports, chunks, and indexes your data to enable fast retrieval of relevant information based on user prompts. This information is then provided as context to the model for more accurate and relevant answers.
[Official Gemini File Search Documentation](https://ai.google.dev/gemini-api/docs/file-search)
## Features
| Feature | Supported | Notes |
|---------|-----------|-------|
| Cost Tracking | ❌ | Cost calculation not yet implemented |
| Logging | ✅ | Full request/response logging |
| RAG Ingest API | ✅ | Upload → Chunk → Embed → Store |
| Vector Store Search | ✅ | Search with metadata filters |
| Custom Chunking | ✅ | Configure chunk size and overlap |
| Metadata Filtering | ✅ | Filter by custom metadata |
| Citations | ✅ | Extract from grounding metadata |
## Quick Start
### Setup
Set your Gemini API key:
```bash
export GEMINI_API_KEY="your-api-key"
# or
export GOOGLE_API_KEY="your-api-key"
```
### Basic RAG Ingest
<Tabs>
<TabItem value="python" label="Python SDK">
```python
import litellm
# Ingest a document
response = await litellm.aingest(
ingest_options={
"name": "my-document-store",
"vector_store": {
"custom_llm_provider": "gemini"
}
},
file_data=("document.txt", b"Your document content", "text/plain")
)
print(f"Vector Store ID: {response['vector_store_id']}")
print(f"File ID: {response['file_id']}")
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```bash
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"file": {
"filename": "document.txt",
"content": "'$(base64 -i document.txt)'",
"content_type": "text/plain"
},
"ingest_options": {
"name": "my-document-store",
"vector_store": {
"custom_llm_provider": "gemini"
}
}
}'
```
</TabItem>
</Tabs>
### Search Vector Store
<Tabs>
<TabItem value="python" label="Python SDK">
```python
import litellm
# Search the vector store
response = await litellm.vector_stores.asearch(
vector_store_id="fileSearchStores/your-store-id",
query="What is the main topic?",
custom_llm_provider="gemini",
max_num_results=5
)
for result in response["data"]:
print(f"Score: {result.get('score')}")
print(f"Content: {result['content'][0]['text']}")
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```bash
curl -X POST "http://localhost:4000/v1/vector_stores/fileSearchStores/your-store-id/search" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "What is the main topic?",
"custom_llm_provider": "gemini",
"max_num_results": 5
}'
```
</TabItem>
</Tabs>
## Advanced Features
### Custom Chunking Configuration
Control how documents are split into chunks:
```python
import litellm
response = await litellm.aingest(
ingest_options={
"name": "custom-chunking-store",
"vector_store": {
"custom_llm_provider": "gemini"
},
"chunking_strategy": {
"white_space_config": {
"max_tokens_per_chunk": 200,
"max_overlap_tokens": 20
}
}
},
file_data=("document.txt", document_content, "text/plain")
)
```
**Chunking Parameters:**
- `max_tokens_per_chunk`: Maximum tokens per chunk (default: 800, min: 100, max: 4096)
- `max_overlap_tokens`: Overlap between chunks (default: 400)
### Metadata Filtering
Attach custom metadata to files and filter searches:
#### Attach Metadata During Ingest
```python
import litellm
response = await litellm.aingest(
ingest_options={
"name": "metadata-store",
"vector_store": {
"custom_llm_provider": "gemini",
"custom_metadata": [
{"key": "author", "string_value": "John Doe"},
{"key": "year", "numeric_value": 2024},
{"key": "category", "string_value": "documentation"}
]
}
},
file_data=("document.txt", document_content, "text/plain")
)
```
#### Search with Metadata Filter
```python
import litellm
response = await litellm.vector_stores.asearch(
vector_store_id="fileSearchStores/your-store-id",
query="What is LiteLLM?",
custom_llm_provider="gemini",
filters={"author": "John Doe", "category": "documentation"}
)
```
**Filter Syntax:**
- Simple equality: `{"key": "value"}`
- Gemini converts to: `key="value"`
- Multiple filters combined with AND
### Using Existing Vector Store
Ingest into an existing File Search store:
```python
import litellm
# First, create a store
create_response = await litellm.vector_stores.acreate(
name="My Persistent Store",
custom_llm_provider="gemini"
)
store_id = create_response["id"]
# Then ingest multiple documents into it
for doc in documents:
await litellm.aingest(
ingest_options={
"vector_store": {
"custom_llm_provider": "gemini",
"vector_store_id": store_id # Reuse existing store
}
},
file_data=(doc["name"], doc["content"], doc["type"])
)
```
### Citation Extraction
Gemini provides grounding metadata with citations:
```python
import litellm
response = await litellm.vector_stores.asearch(
vector_store_id="fileSearchStores/your-store-id",
query="Explain the concept",
custom_llm_provider="gemini"
)
for result in response["data"]:
# Access citation information
if "attributes" in result:
print(f"URI: {result['attributes'].get('uri')}")
print(f"Title: {result['attributes'].get('title')}")
# Content with relevance score
print(f"Score: {result.get('score')}")
print(f"Text: {result['content'][0]['text']}")
```
## Complete Example
End-to-end workflow:
```python
import litellm
# 1. Create a File Search store
store_response = await litellm.vector_stores.acreate(
name="Knowledge Base",
custom_llm_provider="gemini"
)
store_id = store_response["id"]
print(f"Created store: {store_id}")
# 2. Ingest documents with custom chunking and metadata
documents = [
{
"name": "intro.txt",
"content": b"Introduction to LiteLLM...",
"metadata": [
{"key": "section", "string_value": "intro"},
{"key": "priority", "numeric_value": 1}
]
},
{
"name": "advanced.txt",
"content": b"Advanced features...",
"metadata": [
{"key": "section", "string_value": "advanced"},
{"key": "priority", "numeric_value": 2}
]
}
]
for doc in documents:
ingest_response = await litellm.aingest(
ingest_options={
"name": f"ingest-{doc['name']}",
"vector_store": {
"custom_llm_provider": "gemini",
"vector_store_id": store_id,
"custom_metadata": doc["metadata"]
},
"chunking_strategy": {
"white_space_config": {
"max_tokens_per_chunk": 300,
"max_overlap_tokens": 50
}
}
},
file_data=(doc["name"], doc["content"], "text/plain")
)
print(f"Ingested: {doc['name']}")
# 3. Search with filters
search_response = await litellm.vector_stores.asearch(
vector_store_id=store_id,
query="How do I get started?",
custom_llm_provider="gemini",
filters={"section": "intro"},
max_num_results=3
)
# 4. Process results
for i, result in enumerate(search_response["data"]):
print(f"\nResult {i+1}:")
print(f" Score: {result.get('score')}")
print(f" File: {result.get('filename')}")
print(f" Content: {result['content'][0]['text'][:100]}...")
```
## Supported File Types
Gemini File Search supports a wide range of file formats:
### Documents
- PDF (`application/pdf`)
- Microsoft Word (`.docx`, `.doc`)
- Microsoft Excel (`.xlsx`, `.xls`)
- Microsoft PowerPoint (`.pptx`)
- OpenDocument formats (`.odt`, `.ods`, `.odp`)
### Text Files
- Plain text (`text/plain`)
- Markdown (`text/markdown`)
- HTML (`text/html`)
- CSV (`text/csv`)
- JSON (`application/json`)
- XML (`application/xml`)
### Code Files
- Python, JavaScript, TypeScript, Java, C/C++, Go, Rust, etc.
- Most common programming languages supported
See [Gemini's full list of supported file types](https://ai.google.dev/gemini-api/docs/file-search#supported-file-types).
## Pricing
- **Indexing**: $0.15 per 1M tokens (embedding pricing)
- **Storage**: Free
- **Query embeddings**: Free
- **Retrieved tokens**: Charged as regular context tokens
## Supported Models
File Search works with:
- `gemini-3-pro-preview`
- `gemini-2.5-pro`
- `gemini-2.5-flash` (and preview versions)
- `gemini-2.5-flash-lite` (and preview versions)
## Troubleshooting
### Authentication Errors
```python
# Ensure API key is set
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
# Or pass explicitly
response = await litellm.aingest(
ingest_options={
"vector_store": {
"custom_llm_provider": "gemini",
"api_key": "your-api-key"
}
},
file_data=(...)
)
```
### Store Not Found
Ensure you're using the full store name format:
- ✅ `fileSearchStores/abc123`
- ❌ `abc123`
### Large Files
For files >100MB, split them into smaller chunks before ingestion.
### Slow Indexing
After ingestion, Gemini may need time to index documents. Wait a few seconds before searching:
```python
import time
# After ingest
await litellm.aingest(...)
# Wait for indexing
time.sleep(5)
# Then search
await litellm.vector_stores.asearch(...)
```
## Related Resources
- [Gemini File Search Official Docs](https://ai.google.dev/gemini-api/docs/file-search)
- [LiteLLM RAG Ingest API](/docs/rag_ingest)
- [LiteLLM Vector Store Search](/docs/vector_stores/search)
- [Using Vector Stores with Chat](/docs/completion/knowledgebase)

73
docs/my-website/docs/providers/vertex_image.md
View file

@ -1,18 +1,65 @@
# Vertex AI Image Generation
Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions.
Vertex AI supports two types of image generation:
1. **Gemini Image Generation Models** (Nano Banana 🍌) - Conversational image generation using `generateContent` API
2. **Imagen Models** - Traditional image generation using `predict` API
| Property | Details |
|----------|---------|
| Description | Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. |
| Description | Vertex AI Image Generation supports both Gemini image generation models |
| Provider Route on LiteLLM | `vertex_ai/` |
| Provider Doc | [Google Cloud Vertex AI Image Generation ↗](https://cloud.google.com/vertex-ai/docs/generative-ai/image/generate-images) |
| Gemini Image Generation Docs | [Gemini Image Generation ↗](https://ai.google.dev/gemini-api/docs/image-generation) |
## Quick Start
### LiteLLM Python SDK
### Gemini Image Generation Models
```python showLineNumbers title="Basic Image Generation"
Gemini image generation models support conversational image creation with features like:
- Text-to-Image generation
- Image editing (text + image → image)
- Multi-turn image refinement
- High-fidelity text rendering
- Up to 4K resolution (Gemini 3 Pro)
```python showLineNumbers title="Gemini 2.5 Flash Image"
import litellm
# Generate a single image
response = await litellm.aimage_generation(
prompt="A nano banana dish in a fancy restaurant with a Gemini theme",
model="vertex_ai/gemini-2.5-flash-image",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
n=1,
size="1024x1024",
)
print(response.data[0].b64_json) # Gemini returns base64 images
```
```python showLineNumbers title="Gemini 3 Pro Image Preview (4K output)"
import litellm
# Generate high-resolution image
response = await litellm.aimage_generation(
prompt="Da Vinci style anatomical sketch of a dissected Monarch butterfly",
model="vertex_ai/gemini-3-pro-image-preview",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
n=1,
size="1024x1024",
# Optional: specify image size for Gemini 3 Pro
# imageSize="4K", # Options: "1K", "2K", "4K"
)
print(response.data[0].b64_json)
```
### Imagen Models
```python showLineNumbers title="Imagen Image Generation"
import litellm
# Generate a single image
@ -21,9 +68,11 @@ response = await litellm.aimage_generation(
model="vertex_ai/imagen-4.0-generate-001",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
n=1,
size="1024x1024",
)
print(response.data[0].url)
print(response.data[0].b64_json) # Imagen also returns base64 images
```
### LiteLLM Proxy
@ -70,6 +119,18 @@ print(response.data[0].url)
## Supported Models
### Gemini Image Generation Models
- `vertex_ai/gemini-2.5-flash-image` - Fast, efficient image generation (1024px resolution)
- `vertex_ai/gemini-3-pro-image-preview` - Advanced model with 4K output, Google Search grounding, and thinking mode
- `vertex_ai/gemini-2.0-flash-preview-image` - Preview model
- `vertex_ai/gemini-2.5-flash-image-preview` - Preview model
### Imagen Models
- `vertex_ai/imagegeneration@006` - Legacy Imagen model
- `vertex_ai/imagen-4.0-generate-001` - Latest Imagen model
- `vertex_ai/imagen-3.0-generate-001` - Imagen 3.0 model
:::tip
@ -77,7 +138,5 @@ print(response.data[0].url)
:::
LiteLLM supports all Vertex AI Imagen models available through Google Cloud.
For the complete and up-to-date list of supported models, visit: [https://models.litellm.ai/](https://models.litellm.ai/)

2
docs/my-website/docs/proxy/config_settings.md
View file

@ -104,6 +104,7 @@ general_settings:
disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses
enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param
reject_clientside_metadata_tags: boolean # if true, rejects requests with client-side 'metadata.tags' to prevent users from influencing budgets
allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
key_management_system: google_kms # either google_kms or azure_kms
master_key: string
@ -201,6 +202,7 @@ router_settings:
| disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints |
| enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
| enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
| reject_clientside_metadata_tags | boolean | If true, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. |
| allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
| key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) |
| master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) |

4
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

@ -60,6 +60,8 @@ litellm_settings:
set_verbose: true # Enable detailed logging
```
**Note:** Virtual key context is **automatically passed** as headers - no additional configuration needed!
### 3. Start the Proxy
```bash
@ -210,7 +212,7 @@ export PILLAR_API_KEY="your_api_key_here"
export PILLAR_API_BASE="https://api.pillar.security"
export PILLAR_ON_FLAGGED_ACTION="monitor"
export PILLAR_FALLBACK_ON_ERROR="allow"
export PILLAR_TIMEOUT="30.0"
export PILLAR_TIMEOUT="5.0"
```
### Session Tracking

4
docs/my-website/docs/proxy/guardrails/tool_permission.md
View file

@ -2,9 +2,9 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Tool Permission Guardrail
# LiteLLM Tool Permission Guardrail
LiteLLM provides a Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml

120
docs/my-website/docs/proxy/reject_clientside_metadata_tags.md Normal file
View file

@ -0,0 +1,120 @@
# Reject Client-Side Metadata Tags
## Overview
The `reject_clientside_metadata_tags` setting allows you to prevent users from passing client-side `metadata.tags` in their API requests. This ensures that tags are only inherited from the API key metadata and cannot be overridden by users to potentially influence budget tracking or routing decisions.
## Use Case
This feature is particularly useful in multi-tenant scenarios where:
- You want to enforce strict budget tracking based on API key tags
- You want to prevent users from manipulating routing decisions by sending custom client-side tags
- You need to ensure consistent tag-based filtering and reporting
## Configuration
Add the following to your `config.yaml`:
```yaml
general_settings:
reject_clientside_metadata_tags: true # Default is false/null
```
## Behavior
### When `reject_clientside_metadata_tags: true`
**Rejected Request Example:**
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"tags": ["custom-tag"] # This will be rejected
}
}'
```
**Error Response:**
```json
{
"error": {
"message": "Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'=True. Tags can only be set via API key metadata.",
"type": "bad_request_error",
"param": "metadata.tags",
"code": 400
}
}
```
**Allowed Request Example:**
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"metadata": {
"custom_field": "value" # Other metadata fields are allowed
}
}'
```
### When `reject_clientside_metadata_tags: false` or not set
All requests are allowed, including those with client-side `metadata.tags`.
## Setting Tags via API Key
When `reject_clientside_metadata_tags` is enabled, tags should be set on the API key metadata:
```bash
curl -X POST http://localhost:4000/key/generate \
-H "Authorization: Bearer sk-master-key" \
-H "Content-Type: application/json" \
-d '{
"metadata": {
"tags": ["team-a", "production"]
}
}'
```
These tags will be automatically inherited by all requests made with that API key.
## Complete Example Configuration
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234
database_url: "postgresql://user:password@localhost:5432/litellm"
# Reject client-side tags
reject_clientside_metadata_tags: true
# Optional: Also enforce user parameter
enforce_user_param: true
```
## Similar Features
- `enforce_user_param` - Requires all requests to include a 'user' parameter
- Tag-based routing - Use tags for intelligent request routing
- Budget tracking - Track spending per tag
## Notes
- This check only applies to LLM API routes (e.g., `/chat/completions`, `/embeddings`)
- Management endpoints (e.g., `/key/generate`) are not affected
- The check validates that client-side `metadata.tags` is not present in the request body
- Other metadata fields can still be passed in requests
- Tags set on API keys will still be applied to all requests

273
docs/my-website/docs/rag_ingest.md Normal file
View file

@ -0,0 +1,273 @@
# /rag/ingest
All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector Store**
| Feature | Supported |
|---------|-----------|
| Cost Tracking | ❌ |
| Logging | ✅ |
| Supported Providers | `openai`, `bedrock`, `gemini` |
## Quick Start
### OpenAI
```bash showLineNumbers title="Ingest to OpenAI vector store"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"document.txt\",
\"content\": \"$(base64 -i document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"vector_store\": {
\"custom_llm_provider\": \"openai\"
}
}
}"
```
### Bedrock
```bash showLineNumbers title="Ingest to Bedrock Knowledge Base"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"document.txt\",
\"content\": \"$(base64 -i document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"vector_store\": {
\"custom_llm_provider\": \"bedrock\"
}
}
}"
```
### Gemini
```bash showLineNumbers title="Ingest to Gemini File Search"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"document.txt\",
\"content\": \"$(base64 -i document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"vector_store\": {
\"custom_llm_provider\": \"gemini\"
}
}
}"
```
**With Custom Chunking:**
```bash showLineNumbers title="Ingest with custom chunking"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"file": {
"filename": "document.txt",
"content": "'$(base64 -i document.txt)'",
"content_type": "text/plain"
},
"ingest_options": {
"vector_store": {
"custom_llm_provider": "gemini"
},
"chunking_strategy": {
"white_space_config": {
"max_tokens_per_chunk": 200,
"max_overlap_tokens": 20
}
}
}
}'
```
## Response
```json
{
"id": "ingest_abc123",
"status": "completed",
"vector_store_id": "vs_xyz789",
"file_id": "file_123"
}
```
## Query the Vector Store
After ingestion, query with `/vector_stores/{vector_store_id}/search`:
```bash showLineNumbers title="Search the vector store"
curl -X POST "http://localhost:4000/v1/vector_stores/vs_xyz789/search" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "What is the main topic?",
"max_num_results": 5
}'
```
## End-to-End Example
### OpenAI
#### 1. Ingest Document
```bash showLineNumbers title="Step 1: Ingest"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"test_document.txt\",
\"content\": \"$(base64 -i test_document.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"name\": \"test-basic-ingest\",
\"vector_store\": {
\"custom_llm_provider\": \"openai\"
}
}
}"
```
Response:
```json
{
"id": "ingest_d834f544-fc5e-4751-902d-fb0bcc183b85",
"status": "completed",
"vector_store_id": "vs_692658d337c4819183f2ad8488d12fc9",
"file_id": "file-M2pJJiWH56cfUP4Fe7rJay"
}
```
#### 2. Query
```bash showLineNumbers title="Step 2: Query"
curl -X POST "http://localhost:4000/v1/vector_stores/vs_692658d337c4819183f2ad8488d12fc9/search" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"query": "What is LiteLLM?",
"custom_llm_provider": "openai"
}'
```
Response:
```json
{
"object": "vector_store.search_results.page",
"search_query": ["What is LiteLLM?"],
"data": [
{
"file_id": "file-M2pJJiWH56cfUP4Fe7rJay",
"filename": "test_document.txt",
"score": 0.4004629778869299,
"attributes": {},
"content": [
{
"type": "text",
"text": "Test document abc123 for RAG ingestion.\nThis is a sample document to test the RAG ingest API.\nLiteLLM provides a unified interface for vector stores."
}
]
}
],
"has_more": false,
"next_page": null
}
```
## Request Parameters
### Top-Level
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `file` | object | One of file/file_url/file_id required | Base64-encoded file |
| `file.filename` | string | Yes | Filename with extension |
| `file.content` | string | Yes | Base64-encoded content |
| `file.content_type` | string | Yes | MIME type (e.g., `text/plain`) |
| `file_url` | string | One of file/file_url/file_id required | URL to fetch file from |
| `file_id` | string | One of file/file_url/file_id required | Existing file ID |
| `ingest_options` | object | Yes | Pipeline configuration |
### ingest_options
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `vector_store` | object | Yes | Vector store configuration |
| `name` | string | No | Pipeline name for logging |
### vector_store (OpenAI)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `custom_llm_provider` | string | - | `"openai"` |
| `vector_store_id` | string | auto-create | Existing vector store ID |
### vector_store (Bedrock)
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `custom_llm_provider` | string | - | `"bedrock"` |
| `vector_store_id` | string | auto-create | Existing Knowledge Base ID |
| `wait_for_ingestion` | boolean | `false` | Wait for indexing to complete |
| `ingestion_timeout` | integer | `300` | Timeout in seconds (if waiting) |
| `s3_bucket` | string | auto-create | S3 bucket for documents |
| `s3_prefix` | string | `"data/"` | S3 key prefix |
| `embedding_model` | string | `amazon.titan-embed-text-v2:0` | Bedrock embedding model |
| `aws_region_name` | string | `us-west-2` | AWS region |
:::info Bedrock Auto-Creation
When `vector_store_id` is omitted, LiteLLM automatically creates:
- S3 bucket for document storage
- OpenSearch Serverless collection
- IAM role with required permissions
- Bedrock Knowledge Base
- Data Source
:::
## Input Examples
### File (Base64)
```json title="Request body"
{
"file": {
"filename": "document.txt",
"content": "<base64-encoded-content>",
"content_type": "text/plain"
},
"ingest_options": {
"vector_store": {"custom_llm_provider": "openai"}
}
}
```
### File URL
```bash showLineNumbers title="Ingest from URL"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"file_url": "https://example.com/document.pdf",
"ingest_options": {"vector_store": {"custom_llm_provider": "openai"}}
}'
```

1
docs/my-website/docs/text_to_speech.md
View file

@ -103,6 +103,7 @@ litellm --config /path/to/config.yaml
| Azure AI Speech Service (AVA)| [Usage](../docs/providers/azure_ai_speech) |
| Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) |
| Gemini | [Usage](#gemini-text-to-speech) |
| ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) |
## `/audio/speech` to `/chat/completions` Bridge

37
docs/my-website/docs/vector_stores/search.md
View file

@ -12,7 +12,7 @@ Search a vector store for relevant chunks based on a query and file attributes f
| Cost Tracking | ✅ | Tracked per search operation |
| Logging | ✅ | Works across all integrations |
| End-user Tracking | ✅ | |
| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus** | Full vector stores API support across providers |
| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus, Gemini** | Full vector stores API support across providers |
## Usage
@ -164,6 +164,41 @@ print(response)
[See full Milvus vector store documentation](../providers/milvus_vector_stores.md)
</TabItem>
<TabItem value="gemini-provider" label="Gemini Provider">
#### Using Gemini File Search
```python showLineNumbers title="Search Vector Store - Gemini Provider"
import litellm
import os
# Set credentials
os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
response = await litellm.vector_stores.asearch(
vector_store_id="fileSearchStores/your-store-id",
query="What is the capital of France?",
custom_llm_provider="gemini",
max_num_results=5
)
print(response)
```
**With Metadata Filter:**
```python showLineNumbers title="Search with Metadata Filter"
response = await litellm.vector_stores.asearch(
vector_store_id="fileSearchStores/your-store-id",
query="What is LiteLLM?",
custom_llm_provider="gemini",
filters={"author": "John Doe", "category": "documentation"},
max_num_results=5
)
print(response)
```
[See full Gemini File Search documentation](../providers/gemini_file_search.md)
</TabItem>
</Tabs>

2
docs/my-website/sidebars.js
View file

@ -412,6 +412,7 @@ const sidebars = {
"proxy/pass_through"
]
},
"rag_ingest",
"realtime",
"rerank",
"response_api",
@ -530,6 +531,7 @@ const sidebars = {
items: [
"providers/bedrock",
"providers/bedrock_embedding",
"providers/bedrock_imported",
"providers/bedrock_image_gen",
"providers/bedrock_rerank",
"providers/bedrock_agentcore",

142
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -130,6 +130,60 @@ class ProxyExtrasDBManager:
capture_output=True,
)
@staticmethod
def _is_permission_error(error_message: str) -> bool:
"""
Check if the error message indicates a database permission error.
Permission errors should NOT be marked as applied, as the migration
did not actually execute successfully.
Args:
error_message: The error message from Prisma migrate
Returns:
bool: True if this is a permission error, False otherwise
"""
permission_patterns = [
r"Database error code: 42501", # PostgreSQL insufficient privilege
r"must be owner of table",
r"permission denied for schema",
r"permission denied for table",
r"must be owner of schema",
]
for pattern in permission_patterns:
if re.search(pattern, error_message, re.IGNORECASE):
return True
return False
@staticmethod
def _is_idempotent_error(error_message: str) -> bool:
"""
Check if the error message indicates an idempotent operation error.
Idempotent errors (like "column already exists") mean the migration
has effectively already been applied, so it's safe to mark as applied.
Args:
error_message: The error message from Prisma migrate
Returns:
bool: True if this is an idempotent error, False otherwise
"""
idempotent_patterns = [
r"already exists",
r"column .* already exists",
r"duplicate key value violates",
r"relation .* already exists",
r"constraint .* already exists",
]
for pattern in idempotent_patterns:
if re.search(pattern, error_message, re.IGNORECASE):
return True
return False
@staticmethod
def _resolve_all_migrations(
migrations_dir: str, schema_path: str, mark_all_applied: bool = True
@ -320,29 +374,79 @@ class ProxyExtrasDBManager:
)
logger.info("✅ All migrations resolved.")
return True
elif (
"P3018" in e.stderr
): # PostgreSQL error code for duplicate column
logger.info(
"Migration already exists, resolving specific migration"
)
# Extract the migration name from the error message
migration_match = re.search(
r"Migration name: (\d+_.*)", e.stderr
)
if migration_match:
migration_name = migration_match.group(1)
logger.info(f"Rolling back migration {migration_name}")
ProxyExtrasDBManager._roll_back_migration(
migration_name
elif "P3018" in e.stderr:
# Check if this is a permission error or idempotent error
if ProxyExtrasDBManager._is_permission_error(e.stderr):
# Permission errors should NOT be marked as applied
# Extract migration name for logging
migration_match = re.search(
r"Migration name: (\d+_.*)", e.stderr
)
migration_name = (
migration_match.group(1)
if migration_match
else "unknown"
)
logger.error(
f"❌ Migration {migration_name} failed due to insufficient permissions. "
f"Please check database user privileges. Error: {e.stderr}"
)
# Mark as rolled back and exit with error
if migration_match:
try:
ProxyExtrasDBManager._roll_back_migration(
migration_name
)
logger.info(
f"Migration {migration_name} marked as rolled back"
)
except Exception as rollback_error:
logger.warning(
f"Failed to mark migration as rolled back: {rollback_error}"
)
# Re-raise the error to prevent silent failures
raise RuntimeError(
f"Migration failed due to permission error. Migration {migration_name} "
f"was NOT applied. Please grant necessary database permissions and retry."
) from e
elif ProxyExtrasDBManager._is_idempotent_error(e.stderr):
# Idempotent errors mean the migration has effectively been applied
logger.info(
f"Resolving migration {migration_name} that failed due to existing columns"
"Migration failed due to idempotent error (e.g., column already exists), "
"resolving as applied"
)
ProxyExtrasDBManager._resolve_specific_migration(
migration_name
# Extract the migration name from the error message
migration_match = re.search(
r"Migration name: (\d+_.*)", e.stderr
)
logger.info("✅ Migration resolved.")
if migration_match:
migration_name = migration_match.group(1)
logger.info(
f"Rolling back migration {migration_name}"
)
ProxyExtrasDBManager._roll_back_migration(
migration_name
)
logger.info(
f"Resolving migration {migration_name} that failed "
f"due to existing schema objects"
)
ProxyExtrasDBManager._resolve_specific_migration(
migration_name
)
logger.info("✅ Migration resolved.")
else:
# Unknown P3018 error - log and re-raise for safety
logger.warning(
f"P3018 error encountered but could not classify "
f"as permission or idempotent error. "
f"Error: {e.stderr}"
)
raise
else:
# Use prisma db push with increased timeout
subprocess.run(

8
litellm/__init__.py
View file

@ -1114,6 +1114,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig
from .llms.datarobot.chat.transformation import DataRobotConfig
from .llms.anthropic.chat.transformation import AnthropicConfig
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.azure.anthropic.transformation import AzureAnthropicConfig
from .llms.groq.stt.transformation import GroqSTTConfig
from .llms.anthropic.completion.transformation import AnthropicTextConfig
from .llms.triton.completion.transformation import TritonConfig
@ -1225,6 +1226,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation impor
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
AmazonBedrockOpenAIConfig,
)
from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig
from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config
@ -1431,6 +1435,7 @@ from .skills.main import (
)
from .containers.main import *
from .ocr.main import *
from .rag.main import *
from .search.main import *
from .realtime_api.main import _arealtime
from .fine_tuning.main import *
@ -1466,6 +1471,9 @@ from .vector_stores.vector_store_registry import (
vector_store_registry: Optional[VectorStoreRegistry] = None
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
### RAG ###
from . import rag
### CUSTOM LLMs ###
from .types.llms.custom_llm import CustomLLMItem
from .types.utils import GenericStreamingChunk

62
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -234,6 +234,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
cast(List[Dict[str, Any]], value)
)
)
elif key == "response_format":
# Convert response_format to text.format
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key == "metadata":
@ -666,6 +671,63 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return Reasoning(effort="minimal")
return None
def _transform_response_format_to_text_format(
self, response_format: Union[Dict[str, Any], Any]
) -> Optional[Dict[str, Any]]:
"""
Transform Chat Completion response_format parameter to Responses API text.format parameter.
Chat Completion response_format structure:
{
"type": "json_schema",
"json_schema": {
"name": "schema_name",
"schema": {...},
"strict": True
}
}
Responses API text parameter structure:
{
"format": {
"type": "json_schema",
"name": "schema_name",
"schema": {...},
"strict": True
}
}
"""
if not response_format:
return None
if isinstance(response_format, dict):
format_type = response_format.get("type")
if format_type == "json_schema":
json_schema = response_format.get("json_schema", {})
return {
"format": {
"type": "json_schema",
"name": json_schema.get("name", "response_schema"),
"schema": json_schema.get("schema", {}),
"strict": json_schema.get("strict", False),
}
}
elif format_type == "json_object":
return {
"format": {
"type": "json_object"
}
}
elif format_type == "text":
return {
"format": {
"type": "text"
}
}
return None
def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str:
"""Map responses API status to chat completion finish_reason"""
if not status:

4
litellm/constants.py
View file

@ -1211,3 +1211,7 @@ SENTRY_PII_DENYLIST = [
COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)
)
########################### RAG Text Splitter Constants ###########################
DEFAULT_CHUNK_SIZE = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000))
DEFAULT_CHUNK_OVERLAP = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200))

51
litellm/cost_calculator.py
View file

@ -1031,6 +1031,57 @@ def completion_cost( # noqa: PLR0915
billed_units.get("search_units") or 1
) # cohere charges per request by default.
completion_tokens = search_units
elif (
call_type == CallTypes.search.value
or call_type == CallTypes.asearch.value
):
from litellm.search import search_provider_cost_per_query
# Extract number_of_queries from optional_params or default to 1
number_of_queries = 1
if optional_params is not None:
# Check if query is a list (multiple queries)
query = optional_params.get("query")
if isinstance(query, list):
number_of_queries = len(query)
elif query is not None:
number_of_queries = 1
search_model = model or ""
if custom_llm_provider and "/" not in search_model:
# If model is like "tavily-search", construct "tavily/search" for cost lookup
search_model = f"{custom_llm_provider}/search"
prompt_cost, completion_cost_result = search_provider_cost_per_query(
model=search_model,
custom_llm_provider=custom_llm_provider,
number_of_queries=number_of_queries,
optional_params=optional_params,
)
# Return the total cost (prompt_cost + completion_cost, but for search it's just prompt_cost)
_final_cost = prompt_cost + completion_cost_result
# Apply discount
original_cost = _final_cost
_final_cost, discount_percent, discount_amount = _apply_cost_discount(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
# Store cost breakdown in logging object if available
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_cost,
completion_tokens_cost_usd_dollar=completion_cost_result,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=_final_cost,
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
)
return _final_cost
elif call_type == CallTypes.arealtime.value and isinstance(
completion_response, LiteLLMRealtimeStreamLoggingObject
):

45
litellm/images/main.py
View file

@ -19,6 +19,8 @@ from litellm.llms.custom_llm import CustomLLM
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
from openai.types.audio.transcription_create_params import FileTypes # type: ignore
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -26,7 +28,6 @@ from litellm.main import (
bedrock_image_generation,
openai_chat_completions,
openai_image_variations,
vertex_image_generation,
)
###########################################
@ -36,7 +37,6 @@ from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
FileTypes,
LlmProviders,
all_litellm_params,
)
@ -344,6 +344,7 @@ def image_generation( # noqa: PLR0915
litellm.LlmProviders.GEMINI,
litellm.LlmProviders.FAL_AI,
litellm.LlmProviders.RUNWAYML,
litellm.LlmProviders.VERTEX_AI,
):
if image_generation_config is None:
raise ValueError(
@ -430,46 +431,6 @@ def image_generation( # noqa: PLR0915
api_base=api_base,
api_key=api_key,
)
elif custom_llm_provider == "vertex_ai":
vertex_ai_project = (
optional_params.pop("vertex_project", None)
or optional_params.pop("vertex_ai_project", None)
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.pop("vertex_location", None)
or optional_params.pop("vertex_ai_location", None)
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = (
optional_params.pop("vertex_credentials", None)
or optional_params.pop("vertex_ai_credentials", None)
or get_secret_str("VERTEXAI_CREDENTIALS")
)
api_base = (
api_base
or litellm.api_base
or get_secret_str("VERTEXAI_API_BASE")
or get_secret_str("VERTEX_API_BASE")
)
model_response = vertex_image_generation.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
vertex_credentials=vertex_credentials,
aimg_generation=aimg_generation,
api_base=api_base,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider

11
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -121,5 +121,16 @@ def get_litellm_params(
"use_litellm_proxy": use_litellm_proxy,
"litellm_request_debug": litellm_request_debug,
"aws_region_name": kwargs.get("aws_region_name"),
# AWS credentials for Bedrock/Sagemaker
"aws_access_key_id": kwargs.get("aws_access_key_id"),
"aws_secret_access_key": kwargs.get("aws_secret_access_key"),
"aws_session_token": kwargs.get("aws_session_token"),
"aws_session_name": kwargs.get("aws_session_name"),
"aws_profile_name": kwargs.get("aws_profile_name"),
"aws_role_name": kwargs.get("aws_role_name"),
"aws_web_identity_token": kwargs.get("aws_web_identity_token"),
"aws_sts_endpoint": kwargs.get("aws_sts_endpoint"),
"aws_external_id": kwargs.get("aws_external_id"),
"aws_bedrock_runtime_endpoint": kwargs.get("aws_bedrock_runtime_endpoint"),
}
return litellm_params

18
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -22,6 +22,19 @@ def _is_non_openai_azure_model(model: str) -> bool:
return False
def _is_azure_anthropic_model(model: str) -> Optional[str]:
try:
model_parts = model.split("/", 1)
if len(model_parts) > 1:
model_name = model_parts[1].lower()
# Check if model name contains claude
if "claude" in model_name or model_name.startswith("claude"):
return model_parts[1] # Return model name without "azure/" prefix
except Exception:
pass
return None
def handle_cohere_chat_model_custom_llm_provider(
model: str, custom_llm_provider: Optional[str] = None
) -> Tuple[str, Optional[str]]:
@ -123,6 +136,11 @@ def get_llm_provider( # noqa: PLR0915
# AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere
# If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus
if model.split("/", 1)[0] == "azure":
# Check if it's an Azure Anthropic model (claude models)
azure_anthropic_model = _is_azure_anthropic_model(model)
if azure_anthropic_model:
custom_llm_provider = "azure_anthropic"
return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base
if _is_non_openai_azure_model(model):
custom_llm_provider = "openai"
return model, custom_llm_provider, dynamic_api_key, api_base

5
litellm/litellm_core_utils/litellm_logging.py
View file

@ -69,6 +69,7 @@ from litellm.litellm_core_utils.redact_messages import (
redact_message_input_output_from_logging,
)
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.containers.main import ContainerObject
from litellm.types.llms.openai import (
@ -1314,6 +1315,7 @@ class Logging(LiteLLMLoggingBaseClass):
OpenAIFileObject,
LiteLLMRealtimeStreamLoggingObject,
OpenAIModerationResponse,
"SearchResponse",
],
cache_hit: Optional[bool] = None,
litellm_model_name: Optional[str] = None,
@ -1726,8 +1728,11 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject)
or isinstance(logging_result, OpenAIModerationResponse)
or isinstance(logging_result, OCRResponse) # OCR
or isinstance(logging_result, SearchResponse) # Search API
or isinstance(logging_result, dict)
and logging_result.get("object") == "vector_store.search_results.page"
or isinstance(logging_result, dict)
and logging_result.get("object") == "search" # Search API (dict format)
or isinstance(logging_result, VideoObject)
or isinstance(logging_result, ContainerObject)
or (self.call_type == CallTypes.call_mcp_tool.value)

12
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -18,6 +18,7 @@ from litellm.types.utils import (
ModelResponseStream,
PromptTokensDetailsWrapper,
Usage,
ServerToolUse
)
from litellm.utils import print_verbose, token_counter
@ -418,7 +419,8 @@ class ChunkProcessor:
## anthropic prompt caching information ##
cache_creation_input_tokens: Optional[int] = None
cache_read_input_tokens: Optional[int] = None
server_tool_use: Optional[ServerToolUse] = None
web_search_requests: Optional[int] = None
completion_tokens_details: Optional[CompletionTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
@ -462,6 +464,8 @@ class ChunkProcessor:
completion_tokens_details = usage_chunk_dict[
"completion_tokens_details"
]
if hasattr(usage_chunk, 'server_tool_use') and usage_chunk.server_tool_use is not None:
server_tool_use = usage_chunk.server_tool_use
if (
usage_chunk_dict["prompt_tokens_details"] is not None
and getattr(
@ -483,6 +487,7 @@ class ChunkProcessor:
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
server_tool_use=server_tool_use,
web_search_requests=web_search_requests,
completion_tokens_details=completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
@ -513,6 +518,9 @@ class ChunkProcessor:
"cache_read_input_tokens"
]
server_tool_use: Optional[ServerToolUse] = calculated_usage_per_chunk[
"server_tool_use"
]
web_search_requests: Optional[int] = calculated_usage_per_chunk[
"web_search_requests"
]
@ -576,6 +584,8 @@ class ChunkProcessor:
if prompt_tokens_details is not None:
returned_usage.prompt_tokens_details = prompt_tokens_details
if server_tool_use is not None:
returned_usage.server_tool_use = server_tool_use
if web_search_requests is not None:
if returned_usage.prompt_tokens_details is None:
returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper(

81
litellm/llms/anthropic/chat/handler.py
View file

@ -42,6 +42,7 @@ from litellm.types.llms.openai import (
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
)
from litellm.types.utils import (
Delta,
@ -550,15 +551,18 @@ class ModelResponseIterator:
if "text" in content_block["delta"]:
text = content_block["delta"]["text"]
elif "partial_json" in content_block["delta"]:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": content_block["delta"]["partial_json"],
tool_use = cast(
ChatCompletionToolCallChunk,
{
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": content_block["delta"]["partial_json"],
},
"index": self.tool_index,
},
"index": self.tool_index,
}
)
elif "citation" in content_block["delta"]:
provider_specific_fields["citation"] = content_block["delta"]["citation"]
elif (
@ -569,7 +573,7 @@ class ModelResponseIterator:
ChatCompletionThinkingBlock(
type="thinking",
thinking=content_block["delta"].get("thinking") or "",
signature=content_block["delta"].get("signature"),
signature=str(content_block["delta"].get("signature") or ""),
)
]
provider_specific_fields["thinking_blocks"] = thinking_blocks
@ -625,7 +629,7 @@ class ModelResponseIterator:
return content_block_start
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915
try:
type_chunk = chunk.get("type", "") or ""
@ -672,15 +676,32 @@ class ModelResponseIterator:
text = content_block_start["content_block"]["text"]
elif content_block_start["content_block"]["type"] == "tool_use":
self.tool_index += 1
tool_use = {
"id": content_block_start["content_block"]["id"],
"type": "function",
"function": {
"name": content_block_start["content_block"]["name"],
"arguments": "",
},
"index": self.tool_index,
}
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content_block_start["content_block"]["name"],
arguments="",
),
index=self.tool_index,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in content_block_start["content_block"]:
caller_data = content_block_start["content_block"]["caller"]
if caller_data:
tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item]
elif content_block_start["content_block"]["type"] == "server_tool_use":
# Handle server tool use (for tool search)
self.tool_index += 1
tool_use = ChatCompletionToolCallChunk(
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content_block_start["content_block"]["name"],
arguments="",
),
index=self.tool_index,
)
elif (
content_block_start["content_block"]["type"] == "redacted_thinking"
):
@ -696,17 +717,21 @@ class ModelResponseIterator:
# check if tool call content block
is_empty = self.check_empty_tool_call_args()
if is_empty:
tool_use = {
"id": None,
"type": "function",
"function": {
"name": None,
"arguments": "{}",
},
"index": self.tool_index,
}
tool_use = ChatCompletionToolCallChunk(
id=None, # type: ignore[typeddict-item]
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=None, # type: ignore[typeddict-item]
arguments="{}",
),
index=self.tool_index,
)
# Reset response_format tool tracking when block stops
self.is_response_format_tool = False
elif type_chunk == "tool_result":
# Handle tool_result blocks (for tool search results with tool_reference)
# These are automatically handled by Anthropic API, we just pass them through
pass
elif type_chunk == "message_delta":
finish_reason, usage = self._handle_message_delta(chunk)
elif type_chunk == "message_start":

266
litellm/llms/anthropic/chat/transformation.py
View file

@ -54,7 +54,10 @@ from litellm.types.utils import (
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.types.utils import (
PromptTokensDetailsWrapper,
ServerToolUse,
)
from litellm.utils import (
ModelResponse,
Usage,
@ -187,7 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return _tool_choice
def _map_tool_helper(
def _map_tool_helper( # noqa: PLR0915
self, tool: ChatCompletionToolParam
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
returned_tool: Optional[AllAnthropicToolsValues] = None
@ -250,9 +253,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
returned_tool = _computer_tool
elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS):
function_name = tool.get("name", tool.get("function", {}).get("name"))
if function_name is None or not isinstance(function_name, str):
function_name_obj = tool.get("name", tool.get("function", {}).get("name"))
if function_name_obj is None or not isinstance(function_name_obj, str):
raise ValueError("Missing required parameter: name")
function_name = function_name_obj
additional_tool_params = {}
for k, v in tool.items():
@ -268,6 +272,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_server = self._map_openai_mcp_server_tool(
cast(OpenAIMcpServerTool, tool)
)
elif tool["type"] == "tool_search_tool_regex_20251119":
# Tool search tool using regex
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
tool_name_obj = tool.get("name", "tool_search_tool_regex")
if not isinstance(tool_name_obj, str):
raise ValueError("Tool search tool must have a valid name")
tool_name = tool_name_obj
returned_tool = AnthropicToolSearchToolRegex(
type="tool_search_tool_regex_20251119",
name=tool_name,
)
elif tool["type"] == "tool_search_tool_bm25_20251119":
# Tool search tool using BM25
from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25
tool_name_obj = tool.get("name", "tool_search_tool_bm25")
if not isinstance(tool_name_obj, str):
raise ValueError("Tool search tool must have a valid name")
tool_name = tool_name_obj
returned_tool = AnthropicToolSearchToolBM25(
type="tool_search_tool_bm25_20251119",
name=tool_name,
)
if returned_tool is None and mcp_server is None:
raise ValueError(f"Unsupported tool type: {tool['type']}")
@ -275,14 +303,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_cache_control = tool.get("cache_control", None)
_cache_control_function = tool.get("function", {}).get("cache_control", None)
if returned_tool is not None:
if _cache_control is not None:
returned_tool["cache_control"] = _cache_control
elif _cache_control_function is not None and isinstance(
_cache_control_function, dict
):
returned_tool["cache_control"] = ChatCompletionCachedContent(
**_cache_control_function # type: ignore
)
# Only set cache_control on tools that support it (not tool search tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
if _cache_control is not None:
returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item]
elif _cache_control_function is not None and isinstance(
_cache_control_function, dict
):
returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item]
**_cache_control_function # type: ignore
)
## check if defer_loading is set in the tool
_defer_loading = tool.get("defer_loading", None)
_defer_loading_function = tool.get("function", {}).get("defer_loading", None)
if returned_tool is not None:
# Only set defer_loading on tools that support it (not tool search tools or computer tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
if _defer_loading is not None:
if not isinstance(_defer_loading, bool):
raise ValueError("defer_loading must be a boolean")
returned_tool["defer_loading"] = _defer_loading # type: ignore[typeddict-item]
elif _defer_loading_function is not None:
if not isinstance(_defer_loading_function, bool):
raise ValueError("defer_loading must be a boolean")
returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item]
## check if allowed_callers is set in the tool
_allowed_callers = tool.get("allowed_callers", None)
_allowed_callers_function = tool.get("function", {}).get("allowed_callers", None)
if returned_tool is not None:
# Only set allowed_callers on tools that support it (not tool search tools or computer tools)
tool_type = returned_tool.get("type", "")
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
if _allowed_callers is not None:
if not isinstance(_allowed_callers, list) or not all(
isinstance(item, str) for item in _allowed_callers
):
raise ValueError("allowed_callers must be a list of strings")
returned_tool["allowed_callers"] = _allowed_callers # type: ignore[typeddict-item]
elif _allowed_callers_function is not None:
if not isinstance(_allowed_callers_function, list) or not all(
isinstance(item, str) for item in _allowed_callers_function
):
raise ValueError("allowed_callers must be a list of strings")
returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item]
## check if input_examples is set in the tool
_input_examples = tool.get("input_examples", None)
_input_examples_function = tool.get("function", {}).get("input_examples", None)
if returned_tool is not None:
# Only set input_examples on user-defined tools (type "custom" or no type)
tool_type = returned_tool.get("type", "")
if tool_type == "custom" or (tool_type == "" and "name" in returned_tool):
if _input_examples is not None and isinstance(_input_examples, list):
returned_tool["input_examples"] = _input_examples # type: ignore[typeddict-item]
elif _input_examples_function is not None and isinstance(
_input_examples_function, list
):
returned_tool["input_examples"] = _input_examples_function # type: ignore[typeddict-item]
return returned_tool, mcp_server
@ -334,6 +415,82 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_servers.append(mcp_server_tool)
return anthropic_tools, mcp_servers
def _detect_tool_search_tools(self, tools: Optional[List]) -> bool:
"""Check if tool search tools are present in the tools list."""
if not tools:
return False
for tool in tools:
tool_type = tool.get("type", "")
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
return True
return False
def _separate_deferred_tools(
self, tools: List
) -> Tuple[List, List]:
"""
Separate tools into deferred and non-deferred lists.
Returns:
Tuple of (non_deferred_tools, deferred_tools)
"""
non_deferred = []
deferred = []
for tool in tools:
if tool.get("defer_loading", False):
deferred.append(tool)
else:
non_deferred.append(tool)
return non_deferred, deferred
def _expand_tool_references(
self,
content: List,
deferred_tools: List,
) -> List:
"""
Expand tool_reference blocks to full tool definitions.
When Anthropic's tool search returns results, it includes tool_reference blocks
that reference tools by name. This method expands those references to full
tool definitions from the deferred_tools catalog.
Args:
content: Response content that may contain tool_reference blocks
deferred_tools: List of deferred tools that can be referenced
Returns:
Content with tool_reference blocks expanded to full tool definitions
"""
if not deferred_tools:
return content
# Create a mapping of tool names to tool definitions
tool_map = {}
for tool in deferred_tools:
tool_name = tool.get("name") or tool.get("function", {}).get("name")
if tool_name:
tool_map[tool_name] = tool
# Expand tool references in content
expanded_content = []
for item in content:
if isinstance(item, dict) and item.get("type") == "tool_reference":
tool_name = item.get("tool_name")
if tool_name and tool_name in tool_map:
# Replace reference with full tool definition
expanded_content.append(tool_map[tool_name])
else:
# Keep the reference if we can't find the tool
expanded_content.append(item)
else:
expanded_content.append(item)
return expanded_content
def _map_stop_sequences(
self, stop: Optional[Union[str, List[str]]]
) -> Optional[List[str]]:
@ -822,6 +979,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"messages": anthropic_messages,
**optional_params,
}
## Handle output_config (Anthropic-specific parameter)
if "output_config" in optional_params:
output_config = optional_params.get("output_config")
if output_config and isinstance(output_config, dict):
effort = output_config.get("effort")
if effort and effort not in ["high", "medium", "low"]:
raise ValueError(
f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'"
)
data["output_config"] = output_config
return data
@ -870,18 +1038,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_content += content["text"]
## TOOL CALLING
elif content["type"] == "tool_use":
tool_calls.append(
ChatCompletionToolCallChunk(
id=content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content["name"],
arguments=json.dumps(content["input"]),
),
index=idx,
)
tool_call = ChatCompletionToolCallChunk(
id=content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content["name"],
arguments=json.dumps(content["input"]),
),
index=idx,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in content:
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
tool_calls.append(tool_call)
## SERVER TOOL USE (for tool search)
elif content["type"] == "server_tool_use":
# Server tool use blocks are for tool search - treat as tool calls
tool_call = ChatCompletionToolCallChunk(
id=content["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content["name"],
arguments=json.dumps(content.get("input", {})),
),
index=idx,
)
# Include caller information if present (for programmatic tool calling)
if "caller" in content:
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
tool_calls.append(tool_call)
## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery)
elif content["type"] == "tool_search_tool_result":
# This block contains tool_references that were discovered
# We don't need to include this in the response as it's internal metadata
pass
elif content.get("thinking", None) is not None:
if thinking_blocks is None:
thinking_blocks = []
@ -916,7 +1106,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return text_content, citations, thinking_blocks, reasoning_content, tool_calls
def calculate_usage(
self, usage_object: dict, reasoning_content: Optional[str]
self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
@ -926,6 +1116,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_read_input_tokens: int = 0
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
web_search_requests: Optional[int] = None
tool_search_requests: Optional[int] = None
if (
"cache_creation_input_tokens" in _usage
and _usage["cache_creation_input_tokens"] is not None
@ -946,6 +1137,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
web_search_requests = cast(
int, _usage["server_tool_use"]["web_search_requests"]
)
if (
"tool_search_requests" in _usage["server_tool_use"]
and _usage["server_tool_use"]["tool_search_requests"] is not None
):
tool_search_requests = cast(
int, _usage["server_tool_use"]["tool_search_requests"]
)
# Count tool_search_requests from content blocks if not in usage
# Anthropic doesn't always include tool_search_requests in the usage object
if tool_search_requests is None and completion_response is not None:
tool_search_count = 0
for content in completion_response.get("content", []):
if content.get("type") == "server_tool_use":
tool_name = content.get("name", "")
if "tool_search" in tool_name:
tool_search_count += 1
if tool_search_count > 0:
tool_search_requests = tool_search_count
if "cache_creation" in _usage and _usage["cache_creation"] is not None:
cache_creation_token_details = CacheCreationTokenDetails(
@ -982,8 +1192,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
cache_read_input_tokens=cache_read_input_tokens,
completion_tokens_details=completion_token_details,
server_tool_use=(
ServerToolUse(web_search_requests=web_search_requests)
if web_search_requests is not None
ServerToolUse(
web_search_requests=web_search_requests,
tool_search_requests=tool_search_requests,
)
if (web_search_requests is not None or tool_search_requests is not None)
else None
),
)
@ -1077,6 +1290,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
usage = self.calculate_usage(
usage_object=completion_response["usage"],
reasoning_content=reasoning_content,
completion_response=completion_response,
)
setattr(model_response, "usage", usage) # type: ignore

101
litellm/llms/anthropic/common_utils.py
View file

@ -88,6 +88,86 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return True
return False
def is_tool_search_used(self, tools: Optional[List]) -> bool:
"""
Check if tool search tools are present in the tools list.
"""
if not tools:
return False
for tool in tools:
tool_type = tool.get("type", "")
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
return True
return False
def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool:
"""
Check if programmatic tool calling is being used (tools with allowed_callers field).
Returns True if any tool has allowed_callers containing 'code_execution_20250825'.
"""
if not tools:
return False
for tool in tools:
# Check top-level allowed_callers
allowed_callers = tool.get("allowed_callers", None)
if allowed_callers and isinstance(allowed_callers, list):
if "code_execution_20250825" in allowed_callers:
return True
# Check function.allowed_callers for OpenAI format tools
function = tool.get("function", {})
if isinstance(function, dict):
function_allowed_callers = function.get("allowed_callers", None)
if function_allowed_callers and isinstance(function_allowed_callers, list):
if "code_execution_20250825" in function_allowed_callers:
return True
return False
def is_input_examples_used(self, tools: Optional[List]) -> bool:
"""
Check if input_examples is being used in any tools.
Returns True if any tool has input_examples field.
"""
if not tools:
return False
for tool in tools:
# Check top-level input_examples
input_examples = tool.get("input_examples", None)
if input_examples and isinstance(input_examples, list) and len(input_examples) > 0:
return True
# Check function.input_examples for OpenAI format tools
function = tool.get("function", {})
if isinstance(function, dict):
function_input_examples = function.get("input_examples", None)
if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0:
return True
return False
def is_effort_used(self, optional_params: Optional[dict]) -> bool:
"""
Check if effort parameter is being used via output_config.
Returns True if output_config with effort field is present.
"""
if not optional_params:
return False
output_config = optional_params.get("output_config")
if output_config and isinstance(output_config, dict):
effort = output_config.get("effort")
if effort and isinstance(effort, str):
return True
return False
def _get_user_anthropic_beta_headers(
self, anthropic_beta_header: Optional[str]
) -> Optional[List[str]]:
@ -122,6 +202,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
pdf_used: bool = False,
file_id_used: bool = False,
mcp_server_used: bool = False,
tool_search_used: bool = False,
programmatic_tool_calling_used: bool = False,
input_examples_used: bool = False,
effort_used: bool = False,
is_vertex_request: bool = False,
user_anthropic_beta_headers: Optional[List[str]] = None,
) -> dict:
@ -138,6 +222,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
betas.add("code-execution-2025-05-22")
if mcp_server_used:
betas.add("mcp-client-2025-04-04")
# Tool search, programmatic tool calling, and input_examples all use the same beta header
if tool_search_used or programmatic_tool_calling_used or input_examples_used:
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
# Effort parameter uses a separate beta header
if effort_used:
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
betas.add(ANTHROPIC_EFFORT_BETA_HEADER)
headers = {
"anthropic-version": anthropic_version or "2023-06-01",
@ -182,6 +275,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
)
pdf_used = self.is_pdf_used(messages=messages)
file_id_used = self.is_file_id_used(messages=messages)
tool_search_used = self.is_tool_search_used(tools=tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
input_examples_used = self.is_input_examples_used(tools=tools)
effort_used = self.is_effort_used(optional_params=optional_params)
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
)
@ -194,6 +291,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,
mcp_server_used=mcp_server_used,
tool_search_used=tool_search_used,
programmatic_tool_calling_used=programmatic_tool_calling_used,
input_examples_used=input_examples_used,
effort_used=effort_used,
)
headers = {**headers, **anthropic_headers}

2
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -645,7 +645,7 @@ class LiteLLMAnthropicMessagesAdapter:
type="tool_use",
id=choice.delta.tool_calls[0].id or str(uuid.uuid4()),
name=choice.delta.tool_calls[0].function.name or "",
input={},
input={}, # type: ignore[typeddict-item]
)
elif isinstance(choice, StreamingChoices) and hasattr(
choice.delta, "thinking_blocks"

12
litellm/llms/azure/anthropic/__init__.py Normal file
View file

@ -0,0 +1,12 @@
"""
Azure Anthropic provider - supports Claude models via Azure Foundry
"""
from .handler import AzureAnthropicChatCompletion
from .transformation import AzureAnthropicConfig
try:
from .messages_transformation import AzureAnthropicMessagesConfig
__all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig", "AzureAnthropicMessagesConfig"]
except ImportError:
__all__ = ["AzureAnthropicChatCompletion", "AzureAnthropicConfig"]

236
litellm/llms/azure/anthropic/handler.py Normal file
View file

@ -0,0 +1,236 @@
"""
Azure Anthropic handler - reuses AnthropicChatCompletion logic with Azure authentication
"""
import copy
import json
from typing import TYPE_CHECKING, Callable, Union
import httpx
import litellm
from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
from .transformation import AzureAnthropicConfig
if TYPE_CHECKING:
pass
class AzureAnthropicChatCompletion(AnthropicChatCompletion):
"""
Azure Anthropic chat completion handler.
Reuses all Anthropic logic but with Azure authentication.
"""
def __init__(self) -> None:
super().__init__()
def completion(
self,
model: str,
messages: list,
api_base: str,
custom_llm_provider: str,
custom_prompt_dict: dict,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
api_key,
logging_obj,
optional_params: dict,
timeout: Union[float, httpx.Timeout],
litellm_params: dict,
acompletion=None,
logger_fn=None,
headers={},
client=None,
):
"""
Completion method that uses Azure authentication instead of Anthropic's x-api-key.
All other logic is the same as AnthropicChatCompletion.
"""
from litellm.utils import ProviderConfigManager
optional_params = copy.deepcopy(optional_params)
stream = optional_params.pop("stream", None)
json_mode: bool = optional_params.pop("json_mode", False)
is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
_is_function_call = False
messages = copy.deepcopy(messages)
# Use AzureAnthropicConfig instead of AnthropicConfig
headers = AzureAnthropicConfig().validate_environment(
api_key=api_key,
headers=headers,
model=model,
messages=messages,
optional_params={**optional_params, "is_vertex_request": is_vertex_request},
litellm_params=litellm_params,
)
config = ProviderConfigManager.get_provider_chat_config(
model=model,
provider=litellm.types.utils.LlmProviders(custom_llm_provider),
)
if config is None:
raise ValueError(
f"Provider config not found for model: {model} and provider: {custom_llm_provider}"
)
data = config.transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
## LOGGING
logging_obj.pre_call(
input=messages,
api_key=api_key,
additional_args={
"complete_input_dict": data,
"api_base": api_base,
"headers": headers,
},
)
print_verbose(f"_is_function_call: {_is_function_call}")
if acompletion is True:
if (
stream is True
): # if function call - fake the streaming (need complete blocks for output parsing in openai format)
print_verbose("makes async azure anthropic streaming POST request")
data["stream"] = stream
return self.acompletion_stream_function(
model=model,
messages=messages,
data=data,
api_base=api_base,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
api_key=api_key,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream,
_is_function_call=_is_function_call,
json_mode=json_mode,
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=headers,
timeout=timeout,
client=(
client
if client is not None and isinstance(client, AsyncHTTPHandler)
else None
),
)
else:
return self.acompletion_function(
model=model,
messages=messages,
data=data,
api_base=api_base,
custom_prompt_dict=custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
api_key=api_key,
provider_config=config,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream,
_is_function_call=_is_function_call,
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=headers,
client=client,
json_mode=json_mode,
timeout=timeout,
)
else:
## COMPLETION CALL
if (
stream is True
): # if function call - fake the streaming (need complete blocks for output parsing in openai format)
data["stream"] = stream
# Import the make_sync_call from parent
from litellm.llms.anthropic.chat.handler import make_sync_call
completion_stream, response_headers = make_sync_call(
client=client,
api_base=api_base,
headers=headers, # type: ignore
data=json.dumps(data),
model=model,
messages=messages,
logging_obj=logging_obj,
timeout=timeout,
json_mode=json_mode,
)
from litellm.llms.anthropic.common_utils import (
process_anthropic_headers,
)
return CustomStreamWrapper(
completion_stream=completion_stream,
model=model,
custom_llm_provider="azure_anthropic",
logging_obj=logging_obj,
_response_headers=process_anthropic_headers(response_headers),
)
else:
if client is None or not isinstance(client, HTTPHandler):
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
client = _get_httpx_client(params={"timeout": timeout})
else:
client = client
try:
response = client.post(
api_base,
headers=headers,
data=json.dumps(data),
timeout=timeout,
)
except Exception as e:
from litellm.llms.anthropic.common_utils import AnthropicError
status_code = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
error_text = getattr(e, "text", str(e))
error_response = getattr(e, "response", None)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
if error_response and hasattr(error_response, "text"):
error_text = getattr(error_response, "text", error_text)
raise AnthropicError(
message=error_text,
status_code=status_code,
headers=error_headers,
)
return config.transform_response(
model=model,
raw_response=response,
model_response=model_response,
logging_obj=logging_obj,
api_key=api_key,
request_data=data,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
json_mode=json_mode,
)

117
litellm/llms/azure/anthropic/messages_transformation.py Normal file
View file

@ -0,0 +1,117 @@
"""
Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication
"""
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.types.router import GenericLiteLLMParams
if TYPE_CHECKING:
pass
class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
Azure Anthropic messages configuration that extends AnthropicMessagesConfig.
The only difference is authentication - Azure uses x-api-key header (not api-key)
and Azure endpoint format.
"""
def validate_anthropic_messages_environment(
self,
headers: dict,
model: str,
messages: List[Any],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Tuple[dict, Optional[str]]:
"""
Validate environment and set up Azure authentication headers for /v1/messages endpoint.
Azure Anthropic uses x-api-key header (not api-key).
"""
# Convert dict to GenericLiteLLMParams if needed
if isinstance(litellm_params, dict):
if api_key and "api_key" not in litellm_params:
litellm_params = {**litellm_params, "api_key": api_key}
litellm_params_obj = GenericLiteLLMParams(**litellm_params)
else:
litellm_params_obj = litellm_params or GenericLiteLLMParams()
if api_key and not litellm_params_obj.api_key:
litellm_params_obj.api_key = api_key
# Use Azure authentication logic
headers = BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=litellm_params_obj
)
# Azure Anthropic uses x-api-key header (not api-key)
# Convert api-key to x-api-key if present
if "api-key" in headers and "x-api-key" not in headers:
headers["x-api-key"] = headers.pop("api-key")
# Set anthropic-version header
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
# Set content-type header
if "content-type" not in headers:
headers["content-type"] = "application/json"
# Update headers with optional anthropic beta features
headers = self._update_headers_with_optional_anthropic_beta(
headers=headers,
context_management=optional_params.get("context_management"),
)
return headers, api_base
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for Azure Anthropic /v1/messages endpoint.
Azure Foundry endpoint format: https://<resource-name>.services.ai.azure.com/anthropic/v1/messages
"""
from litellm.secret_managers.main import get_secret_str
api_base = api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
"Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. "
"Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
)
# Ensure the URL ends with /v1/messages
api_base = api_base.rstrip("/")
if api_base.endswith("/v1/messages"):
# Already correct
pass
elif api_base.endswith("/anthropic/v1/messages"):
# Already correct
pass
else:
# Check if /anthropic is already in the path
if "/anthropic" in api_base:
# /anthropic exists, ensure we end with /anthropic/v1/messages
# Extract the base URL up to and including /anthropic
parts = api_base.split("/anthropic", 1)
api_base = parts[0] + "/anthropic"
else:
# /anthropic not in path, add it
api_base = api_base + "/anthropic"
# Add /v1/messages
api_base = api_base + "/v1/messages"
return api_base

96
litellm/llms/azure/anthropic/transformation.py Normal file
View file

@ -0,0 +1,96 @@
"""
Azure Anthropic transformation config - extends AnthropicConfig with Azure authentication
"""
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
if TYPE_CHECKING:
pass
class AzureAnthropicConfig(AnthropicConfig):
"""
Azure Anthropic configuration that extends AnthropicConfig.
The only difference is authentication - Azure uses api-key header or Azure AD token
instead of x-api-key header.
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "azure_anthropic"
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: Union[dict, GenericLiteLLMParams],
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> Dict:
"""
Validate environment and set up Azure authentication headers.
Azure supports:
1. API key via 'api-key' header
2. Azure AD token via 'Authorization: Bearer <token>' header
"""
# Convert dict to GenericLiteLLMParams if needed
if isinstance(litellm_params, dict):
# Ensure api_key is included if provided
if api_key and "api_key" not in litellm_params:
litellm_params = {**litellm_params, "api_key": api_key}
litellm_params_obj = GenericLiteLLMParams(**litellm_params)
else:
litellm_params_obj = litellm_params or GenericLiteLLMParams()
# Set api_key if provided and not already set
if api_key and not litellm_params_obj.api_key:
litellm_params_obj.api_key = api_key
# Use Azure authentication logic
headers = BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=litellm_params_obj
)
# Azure Anthropic uses x-api-key header (not api-key)
# Convert api-key to x-api-key if present
if "api-key" in headers and "x-api-key" not in headers:
headers["x-api-key"] = headers.pop("api-key")
# Get tools and other anthropic-specific setup
tools = optional_params.get("tools")
prompt_caching_set = self.is_cache_control_set(messages=messages)
computer_tool_used = self.is_computer_tool_used(tools=tools)
mcp_server_used = self.is_mcp_server_used(
mcp_servers=optional_params.get("mcp_servers")
)
pdf_used = self.is_pdf_used(messages=messages)
file_id_used = self.is_file_id_used(messages=messages)
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
anthropic_beta_header=headers.get("anthropic-beta")
)
# Get anthropic headers (but we'll replace x-api-key with Azure auth)
anthropic_headers = self.get_anthropic_headers(
computer_tool_used=computer_tool_used,
prompt_caching_set=prompt_caching_set,
pdf_used=pdf_used,
api_key=api_key or "", # Azure auth is already in headers
file_id_used=file_id_used,
is_vertex_request=optional_params.get("is_vertex_request", False),
user_anthropic_beta_headers=user_anthropic_beta_headers,
mcp_server_used=mcp_server_used,
)
# Merge headers - Azure auth (api-key or Authorization) takes precedence
headers = {**anthropic_headers, **headers}
# Ensure anthropic-version header is set
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"
return headers

36
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -2,6 +2,8 @@
from typing import List
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.types.llms.openai import AllMessageValues
@ -33,7 +35,34 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
drop_params: bool,
api_version: str = "",
) -> dict:
return OpenAIGPT5Config.map_openai_params(
reasoning_effort_value = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
if reasoning_effort_value == "none":
if litellm.drop_params is True or (
drop_params is not None and drop_params is True
):
non_default_params = non_default_params.copy()
optional_params = optional_params.copy()
if non_default_params.get("reasoning_effort") == "none":
non_default_params.pop("reasoning_effort")
if optional_params.get("reasoning_effort") == "none":
optional_params.pop("reasoning_effort")
else:
raise UnsupportedParamsError(
status_code=400,
message=(
"Azure OpenAI does not support reasoning_effort='none'. "
"Supported values are: 'low', 'medium', and 'high'. "
"To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
"Issue: https://github.com/BerriAI/litellm/issues/16704"
),
)
result = OpenAIGPT5Config.map_openai_params(
self,
non_default_params=non_default_params,
optional_params=optional_params,
@ -41,6 +70,11 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
drop_params=drop_params,
)
if result.get("reasoning_effort") == "none":
result.pop("reasoning_effort")
return result
def transform_request(
self,
model: str,

34
litellm/llms/azure/videos/transformation.py
View file

@ -1,9 +1,8 @@
from typing import TYPE_CHECKING, Any, Dict, Optional
from litellm.types.videos.main import VideoCreateOptionalRequestParams
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.llms.azure.common_utils import BaseAzureLLM
import litellm
from litellm.llms.openai.videos.transformation import OpenAIVideoConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -56,22 +55,27 @@ class AzureVideoConfig(OpenAIVideoConfig):
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
"""
Validate Azure environment and set up authentication headers.
Uses _base_validate_azure_environment to properly handle credentials from litellm_credential_name.
"""
# If litellm_params is provided, use it; otherwise create a new one
if litellm_params is None:
litellm_params = GenericLiteLLMParams()
if api_key and not litellm_params.api_key:
litellm_params.api_key = api_key
# Use the base Azure validation method which properly handles:
# 1. Credentials from litellm_credential_name via litellm_params
# 2. Sets the correct "api-key" header (not "Authorization: Bearer")
return BaseAzureLLM._base_validate_azure_environment(
headers=headers,
litellm_params=litellm_params
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
)
return headers
def get_complete_url(
self,
model: str,

1
litellm/llms/base_llm/videos/transformation.py
View file

@ -66,6 +66,7 @@ class BaseVideoConfig(ABC):
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
return {}

14
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -580,6 +580,20 @@ class AmazonConverseConfig(BaseConfig):
non_default_params=non_default_params, optional_params=optional_params
)
final_is_thinking_enabled = self.is_thinking_enabled(optional_params)
if (
final_is_thinking_enabled
and "tool_choice" in optional_params
):
tool_choice_block = optional_params["tool_choice"]
if isinstance(tool_choice_block, dict):
if "any" in tool_choice_block or "tool" in tool_choice_block:
verbose_logger.info(
f"{model} does not support forced tool use (tool_choice='required' or specific tool) "
f"when reasoning is enabled. Changing tool_choice to 'auto'."
)
optional_params["tool_choice"] = ToolChoiceValuesBlock(auto={})
return optional_params
def _translate_response_format_param(

186
litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py Normal file
View file

@ -0,0 +1,186 @@
"""
Transformation for Bedrock imported models that use OpenAI Chat Completions format.
Use this for models imported into Bedrock that accept the OpenAI API format.
Model format: bedrock/openai/<model-id>
Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123
"""
from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
import httpx
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.openai import AllMessageValues
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
"""
Configuration for Bedrock imported models that use OpenAI Chat Completions format.
This class handles the transformation of requests and responses for Bedrock
imported models that accept the OpenAI API format directly.
Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling
and response transformation, while adding Bedrock-specific URL generation
and AWS request signing.
Usage:
model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123"
"""
def __init__(self, **kwargs):
OpenAIGPTConfig.__init__(self, **kwargs)
BaseAWSLLM.__init__(self, **kwargs)
@property
def custom_llm_provider(self) -> Optional[str]:
return "bedrock"
def _get_openai_model_id(self, model: str) -> str:
"""
Extract the actual model ID from the LiteLLM model name.
Input format: bedrock/openai/<model-id>
Returns: <model-id>
"""
# Remove bedrock/ prefix if present
if model.startswith("bedrock/"):
model = model[8:]
# Remove openai/ prefix
if model.startswith("openai/"):
model = model[7:]
return model
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for the Bedrock invoke endpoint.
Uses the standard Bedrock invoke endpoint format.
"""
model_id = self._get_openai_model_id(model)
# Get AWS region
aws_region_name = self._get_aws_region_name(
optional_params=optional_params, model=model
)
# Get runtime endpoint
aws_bedrock_runtime_endpoint = optional_params.get(
"aws_bedrock_runtime_endpoint", None
)
endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_region_name=aws_region_name,
)
# Build the invoke URL
if stream:
endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream"
else:
endpoint_url = f"{endpoint_url}/model/{model_id}/invoke"
return endpoint_url
def sign_request(
self,
headers: dict,
optional_params: dict,
request_data: dict,
api_base: str,
api_key: Optional[str] = None,
model: Optional[str] = None,
stream: Optional[bool] = None,
fake_stream: Optional[bool] = None,
) -> Tuple[dict, Optional[bytes]]:
"""
Sign the request using AWS Signature Version 4.
"""
return self._sign_request(
service_name="bedrock",
headers=headers,
optional_params=optional_params,
request_data=request_data,
api_base=api_base,
api_key=api_key,
model=model,
stream=stream,
fake_stream=fake_stream,
)
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the request to OpenAI Chat Completions format for Bedrock imported models.
Removes AWS-specific params and stream param (handled separately in URL),
then delegates to parent class for standard OpenAI request transformation.
"""
# Remove stream from optional_params as it's handled separately in URL
optional_params.pop("stream", None)
# Remove AWS-specific params that shouldn't be in the request body
inference_params = {
k: v
for k, v in optional_params.items()
if k not in self.aws_authentication_params
}
# Use parent class transform_request for OpenAI format
return super().transform_request(
model=self._get_openai_model_id(model),
messages=messages,
optional_params=inference_params,
litellm_params=litellm_params,
headers=headers,
)
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate the environment and return headers.
For Bedrock, we don't need Bearer token auth since we use AWS SigV4.
"""
return headers
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BedrockError:
"""Return the appropriate error class for Bedrock."""
return BedrockError(status_code=status_code, message=error_message)

18
litellm/llms/bedrock/common_utils.py
View file

@ -403,6 +403,9 @@ class BedrockModelInfo(BaseLLMModelInfo):
if model.startswith("invoke/"):
model = model.split("/", 1)[1]
if model.startswith("openai/"):
model = model.split("/", 1)[1]
return model
@staticmethod
@ -446,12 +449,12 @@ class BedrockModelInfo(BaseLLMModelInfo):
@staticmethod
def get_bedrock_route(
model: str,
) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke"]:
) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke", "openai"]:
"""
Get the bedrock route for the given model.
"""
route_mappings: Dict[
str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke"]
str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke", "openai"]
] = {
"invoke/": "invoke",
"converse_like/": "converse_like",
@ -459,6 +462,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
"agent/": "agent",
"agentcore/": "agentcore",
"async_invoke/": "async_invoke",
"openai/": "openai",
}
# Check explicit routes first
@ -517,6 +521,14 @@ class BedrockModelInfo(BaseLLMModelInfo):
"""
return "async_invoke/" in model
@staticmethod
def _explicit_openai_route(model: str) -> bool:
"""
Check if the model is an explicit openai route.
Used for Bedrock imported models that use OpenAI Chat Completions format.
"""
return "openai/" in model
@staticmethod
def get_bedrock_provider_config_for_messages_api(
model: str,
@ -566,6 +578,8 @@ def get_bedrock_chat_config(model: str):
# Handle explicit routes first
if bedrock_route == "converse" or bedrock_route == "converse_like":
return litellm.AmazonConverseConfig()
elif bedrock_route == "openai":
return litellm.AmazonBedrockOpenAIConfig()
elif bedrock_route == "agent":
from litellm.llms.bedrock.chat.invoke_agent.transformation import (
AmazonInvokeAgentConfig,

20
litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py
View file

@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
from openai.types.image import Image
from litellm import get_model_info
from litellm.types.llms.bedrock import (
AmazonNovaCanvasColorGuidedGenerationParams,
AmazonNovaCanvasColorGuidedRequest,
@ -197,3 +198,22 @@ class AmazonNovaCanvasConfig:
model_response.data = openai_images
return model_response
@classmethod
def cost_calculator(
cls,
model: str,
image_response: ImageResponse,
size: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> float:
model_info = get_model_info(
model=model,
custom_llm_provider="bedrock",
)
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

59
litellm/llms/bedrock/image/amazon_stability1_transformation.py
View file

@ -1,8 +1,11 @@
import copy
import os
import types
from typing import List, Optional
from openai.types.image import Image
from litellm import get_model_info
from litellm.types.utils import ImageResponse
@ -90,6 +93,31 @@ class AmazonStabilityConfig:
return optional_params
@classmethod
def transform_request_body(
cls,
text: str,
optional_params: dict,
) -> dict:
inference_params = copy.deepcopy(optional_params)
inference_params.pop(
"user", None
) # make sure user is not passed in for bedrock call
prompt = text.replace(os.linesep, " ")
## LOAD CONFIG
config = cls.get_config()
for k, v in config.items():
if (
k not in inference_params
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
inference_params[k] = v
return {
"text_prompts": [{"text": prompt, "weight": 1}],
**inference_params,
}
@classmethod
def transform_response_dict_to_openai_response(
cls, model_response: ImageResponse, response_dict: dict
@ -102,3 +130,34 @@ class AmazonStabilityConfig:
model_response.data = image_list
return model_response
@classmethod
def cost_calculator(
cls,
model: str,
image_response: ImageResponse,
size: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> float:
optional_params = optional_params or {}
# see model_prices_and_context_window.json for details on how steps is used
# Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/
_steps = optional_params.get("steps", 50)
steps = "max-steps" if _steps > 50 else "50-steps"
# size is stored in model_prices_and_context_window.json as 1024-x-1024
# current size has 1024x1024
size = size or "1024-x-1024"
model = f"{size}/{steps}/{model}"
model_info = get_model_info(
model=model,
custom_llm_provider="bedrock",
)
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

31
litellm/llms/bedrock/image/amazon_stability3_transformation.py
View file

@ -3,6 +3,8 @@ from typing import List, Optional
from openai.types.image import Image
from litellm import get_model_info
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.llms.bedrock import (
AmazonStability3TextToImageRequest,
AmazonStability3TextToImageResponse,
@ -66,12 +68,12 @@ class AmazonStability3Config:
@classmethod
def transform_request_body(
cls, prompt: str, optional_params: dict
cls, text: str, optional_params: dict
) -> AmazonStability3TextToImageRequest:
"""
Transform the request body for the Stability 3 models
"""
data = AmazonStability3TextToImageRequest(prompt=prompt, **optional_params)
data = AmazonStability3TextToImageRequest(prompt=text, **optional_params)
return data
@classmethod
@ -92,9 +94,34 @@ class AmazonStability3Config:
"""
stability_3_response = AmazonStability3TextToImageResponse(**response_dict)
finish_reasons = stability_3_response.get("finish_reasons", [])
finish_reasons = [reason for reason in finish_reasons if reason]
if len(finish_reasons) > 0:
raise BedrockError(status_code=400, message="; ".join(finish_reasons))
openai_images: List[Image] = []
for _img in stability_3_response.get("images", []):
openai_images.append(Image(b64_json=_img))
model_response.data = openai_images
return model_response
@classmethod
def cost_calculator(
cls,
model: str,
image_response: ImageResponse,
size: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> float:
model_info = get_model_info(
model=model,
custom_llm_provider="bedrock",
)
output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

6
litellm/llms/bedrock/image/amazon_titan_transformation.py
View file

@ -103,16 +103,16 @@ class AmazonTitanImageGenerationConfig:
return optional_params
@classmethod
def _transform_request(
def transform_request_body(
cls,
input: str,
text: str,
optional_params: dict,
) -> AmazonTitanImageGenerationRequestBody:
from typing import Any, Dict
image_generation_config = optional_params.pop("imageGenerationConfig", {})
negative_text = optional_params.pop("negativeText", None)
text_to_image_params: Dict[str, Any] = {"text": input}
text_to_image_params: Dict[str, Any] = {"text": text}
if negative_text:
text_to_image_params["negativeText"] = negative_text
task_type = optional_params.pop("taskType", "TEXT_IMAGE")

41
litellm/llms/bedrock/image/cost_calculator.py
View file

@ -1,9 +1,6 @@
from typing import Optional
import litellm
from litellm.llms.bedrock.image.amazon_titan_transformation import (
AmazonTitanImageGenerationConfig,
)
from litellm.llms.bedrock.image.image_handler import BedrockImageGeneration
from litellm.types.utils import ImageResponse
@ -18,36 +15,10 @@ def cost_calculator(
Handles both Stability 1 and Stability 3 models
"""
if litellm.AmazonStability3Config()._is_stability_3_model(model=model):
pass
elif AmazonTitanImageGenerationConfig._is_titan_model(model=model):
return AmazonTitanImageGenerationConfig.cost_calculator(
model=model,
image_response=image_response,
size=size,
optional_params=optional_params,
)
else:
# Stability 1 models
optional_params = optional_params or {}
# see model_prices_and_context_window.json for details on how steps is used
# Reference pricing by steps for stability 1: https://aws.amazon.com/bedrock/pricing/
_steps = optional_params.get("steps", 50)
steps = "max-steps" if _steps > 50 else "50-steps"
# size is stored in model_prices_and_context_window.json as 1024-x-1024
# current size has 1024x1024
size = size or "1024-x-1024"
model = f"{size}/{steps}/{model}"
_model_info = litellm.get_model_info(
config_class = BedrockImageGeneration.get_config_class(model=model)
return config_class.cost_calculator(
model=model,
custom_llm_provider="bedrock",
image_response=image_response,
size=size,
optional_params=optional_params,
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

87
litellm/llms/bedrock/image/image_handler.py
View file

@ -1,13 +1,12 @@
import copy
from __future__ import annotations
import json
import os
from typing import TYPE_CHECKING, Any, Optional, Union
import httpx
from pydantic import BaseModel
import litellm
from litellm import BEDROCK_INVOKE_PROVIDERS_LITERAL
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image.amazon_nova_canvas_transformation import (
@ -47,11 +46,30 @@ class BedrockImagePreparedRequest(BaseModel):
data: dict
BedrockImageConfigClass = Union[
type[AmazonTitanImageGenerationConfig],
type[AmazonNovaCanvasConfig],
type[AmazonStability3Config],
type[litellm.AmazonStabilityConfig],
]
class BedrockImageGeneration(BaseAWSLLM):
"""
Bedrock Image Generation handler
"""
@classmethod
def get_config_class(cls, model: str | None) -> BedrockImageConfigClass:
if AmazonTitanImageGenerationConfig._is_titan_model(model):
return AmazonTitanImageGenerationConfig
elif AmazonNovaCanvasConfig._is_nova_model(model):
return AmazonNovaCanvasConfig
elif AmazonStability3Config._is_stability_3_model(model):
return AmazonStability3Config
else:
return litellm.AmazonStabilityConfig
def image_generation(
self,
model: str,
@ -202,7 +220,6 @@ class BedrockImageGeneration(BaseAWSLLM):
model=model,
prompt=prompt,
optional_params=optional_params,
bedrock_provider=bedrock_provider,
)
# Make POST Request
@ -241,7 +258,6 @@ class BedrockImageGeneration(BaseAWSLLM):
def _get_request_body(
self,
model: str,
bedrock_provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL],
prompt: str,
optional_params: dict,
) -> dict:
@ -253,49 +269,9 @@ class BedrockImageGeneration(BaseAWSLLM):
Returns:
dict: The request body to use for the Bedrock Image Generation API
"""
if bedrock_provider == "amazon" or bedrock_provider == "nova":
# Handle Amazon Nova Canvas models
provider = "amazon"
elif bedrock_provider == "stability":
provider = "stability"
else:
# Fallback to original logic for backward compatibility
provider = model.split(".")[0]
inference_params = copy.deepcopy(optional_params)
inference_params.pop(
"user", None
) # make sure user is not passed in for bedrock call
data = {}
if provider == "stability":
if litellm.AmazonStability3Config._is_stability_3_model(model):
request_body = litellm.AmazonStability3Config.transform_request_body(
prompt=prompt, optional_params=optional_params
)
return dict(request_body)
else:
prompt = prompt.replace(os.linesep, " ")
## LOAD CONFIG
config = litellm.AmazonStabilityConfig.get_config()
for k, v in config.items():
if (
k not in inference_params
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
inference_params[k] = v
data = {
"text_prompts": [{"text": prompt, "weight": 1}],
**inference_params,
}
elif provider == "amazon":
return dict(
litellm.AmazonNovaCanvasConfig.transform_request_body(
text=prompt, optional_params=optional_params
)
)
else:
raise BedrockError(
status_code=422, message=f"Unsupported model={model}, passed in"
)
return data
config_class = self.get_config_class(model=model)
request_body = config_class.transform_request_body(text=prompt, optional_params=optional_params)
return dict(request_body)
def _transform_response_dict_to_openai_response(
self,
@ -323,20 +299,7 @@ class BedrockImageGeneration(BaseAWSLLM):
if response_dict is None:
raise ValueError("Error in response object format, got None")
config_class: Union[
type[AmazonTitanImageGenerationConfig],
type[AmazonNovaCanvasConfig],
type[AmazonStability3Config],
type[litellm.AmazonStabilityConfig],
]
if AmazonTitanImageGenerationConfig._is_titan_model(model=model):
config_class = AmazonTitanImageGenerationConfig
elif AmazonNovaCanvasConfig._is_nova_model(model=model):
config_class = AmazonNovaCanvasConfig
elif AmazonStability3Config._is_stability_3_model(model=model):
config_class = AmazonStability3Config
else:
config_class = litellm.AmazonStabilityConfig
config_class = self.get_config_class(model=model)
config_class.transform_response_dict_to_openai_response(
model_response=model_response,

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -4126,6 +4126,7 @@ class BaseLLMHTTPHandler:
headers=video_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,
litellm_params=litellm_params,
)
if extra_headers:
@ -4226,6 +4227,7 @@ class BaseLLMHTTPHandler:
headers=video_generation_optional_request_params.get("extra_headers", {})
or {},
model=model,
litellm_params=litellm_params,
)
if extra_headers:

332
litellm/llms/elevenlabs/text_to_speech/transformation.py Normal file
View file

@ -0,0 +1,332 @@
"""
Elevenlabs Text-to-Speech transformation
Maps OpenAI TTS spec to Elevenlabs TTS API
"""
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
from urllib.parse import urlencode
import httpx
from httpx import Headers
import litellm
from litellm.types.utils import all_litellm_params
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.secret_managers.main import get_secret_str
from ..common_utils import ElevenLabsException
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.openai import HttpxBinaryResponseContent
else:
LiteLLMLoggingObj = Any
HttpxBinaryResponseContent = Any
class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
"""
Configuration for ElevenLabs Text-to-Speech
Reference: https://elevenlabs.io/docs/api-reference/text-to-speech/convert
"""
TTS_BASE_URL = "https://api.elevenlabs.io"
TTS_ENDPOINT_PATH = "/v1/text-to-speech"
DEFAULT_OUTPUT_FORMAT = "pcm_44100"
VOICE_MAPPINGS = {
"alloy": "21m00Tcm4TlvDq8ikWAM", # Rachel
"amber": "5Q0t7uMcjvnagumLfvZi", # Paul
"ash": "AZnzlk1XvdvUeBnXmlld", # Domi
"august": "D38z5RcWu1voky8WS1ja", # Fin
"blue": "2EiwWnXFnvU5JabPnv8n", # Clyde
"coral": "9BWtsMINqrJLrRacOk9x", # Aria
"lily": "EXAVITQu4vr4xnSDxMaL", # Sarah
"onyx": "29vD33N1CtxCmqQRPOHJ", # Drew
"sage": "CwhRBWXzGAHq8TQ4Fs17", # Roger
"verse": "CYw3kZ02Hs0563khs1Fj", # Dave
}
# Response format mappings from OpenAI to ElevenLabs
FORMAT_MAPPINGS = {
"mp3": "mp3_44100_128",
"pcm": "pcm_44100",
"opus": "opus_48000_128",
# ElevenLabs does not support WAV, AAC, or FLAC formats.
}
ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__"
ELEVENLABS_VOICE_ID_KEY = "__elevenlabs_voice_id__"
def get_supported_openai_params(self, model: str) -> list:
"""
ElevenLabs TTS supports these OpenAI parameters
"""
return ["voice", "response_format", "speed"]
def _extract_voice_id(self, voice: str) -> str:
"""
Normalize the provided voice information into an ElevenLabs voice_id.
"""
normalized_voice = voice.strip()
mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower())
return mapped_voice or normalized_voice
def _resolve_voice_id(
self,
voice: Optional[Union[str, Dict[str, Any]]],
params: Dict[str, Any],
) -> str:
"""
Determine the ElevenLabs voice_id based on provided voice input or parameters.
"""
mapped_voice: Optional[str] = None
if isinstance(voice, str) and voice.strip():
mapped_voice = self._extract_voice_id(voice)
elif isinstance(voice, dict):
for key in ("voice_id", "id", "name"):
candidate = voice.get(key)
if isinstance(candidate, str) and candidate.strip():
mapped_voice = self._extract_voice_id(candidate)
break
elif voice is not None:
mapped_voice = self._extract_voice_id(str(voice))
if mapped_voice is None:
voice_override = params.pop("voice_id", None)
if isinstance(voice_override, str) and voice_override.strip():
mapped_voice = self._extract_voice_id(voice_override)
if mapped_voice is None:
raise ValueError(
"ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`."
)
return mapped_voice
def map_openai_params(
self,
model: str,
optional_params: Dict,
voice: Optional[Union[str, Dict]] = None,
drop_params: bool = False,
kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[str], Dict]:
"""
Map OpenAI parameters to ElevenLabs TTS parameters
"""
mapped_params: Dict[str, Any] = {}
query_params: Dict[str, Any] = {}
# Work on a copy so we don't mutate the caller's dictionary
params = dict(optional_params) if optional_params else {}
passthrough_kwargs: Dict[str, Any] = kwargs if kwargs is not None else {}
# Extract voice identifier
mapped_voice = self._resolve_voice_id(voice, params)
# Response/output format → query parameter
response_format = params.pop("response_format", None)
if isinstance(response_format, str):
mapped_format = self.FORMAT_MAPPINGS.get(response_format, response_format)
query_params["output_format"] = mapped_format
# ElevenLabs does not support OpenAI speed directly.
# Drop it to avoid sending unsupported keys unless caller already provided voice_settings.
speed = params.pop("speed", None)
if speed is not None:
speed_value: Optional[float]
try:
speed_value = float(speed)
except (TypeError, ValueError):
speed_value = None
if speed_value is not None:
if isinstance(params.get("voice_settings"), dict):
params["voice_settings"]["speed"] = speed_value # type: ignore[index]
else:
params["voice_settings"] = {"speed": speed_value}
# Instructions parameter is OpenAI-specific; omit to prevent API errors.
params.pop("instructions", None)
self._add_elevenlabs_specific_params(
mapped_voice=mapped_voice,
query_params=query_params,
mapped_params=mapped_params,
kwargs=passthrough_kwargs,
remaining_params=params,
)
return mapped_voice, mapped_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate Azure environment and set up authentication headers
"""
api_key = (
api_key
or litellm.api_key
or litellm.openai_key
or get_secret_str("ELEVENLABS_API_KEY")
)
if api_key is None:
raise ValueError(
"ElevenLabs API key is required. Set ELEVENLABS_API_KEY environment variable."
)
headers.update(
{
"xi-api-key": api_key,
"Content-Type": "application/json",
}
)
return headers
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return ElevenLabsException(
message=error_message, status_code=status_code, headers=headers
)
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: Optional[str],
optional_params: Dict,
litellm_params: Dict,
headers: dict,
) -> TextToSpeechRequestData:
"""
Build the ElevenLabs TTS request payload.
"""
params = dict(optional_params) if optional_params else {}
extra_body = params.pop("extra_body", None)
request_body: Dict[str, Any] = {
"text": input,
"model_id": model,
}
for key, value in params.items():
if value is None:
continue
request_body[key] = value
if isinstance(extra_body, dict):
for key, value in extra_body.items():
if value is None:
continue
request_body[key] = value
return TextToSpeechRequestData(
dict_body=request_body,
headers={"Content-Type": "application/json"},
)
def _add_elevenlabs_specific_params(
self,
mapped_voice: str,
query_params: Dict[str, Any],
mapped_params: Dict[str, Any],
kwargs: Optional[Dict[str, Any]],
remaining_params: Dict[str, Any],
) -> None:
if kwargs is None:
kwargs = {}
for key, value in remaining_params.items():
if value is None:
continue
mapped_params[key] = value
reserved_kwarg_keys = set(all_litellm_params) | {
self.ELEVENLABS_QUERY_PARAMS_KEY,
self.ELEVENLABS_VOICE_ID_KEY,
"voice",
"model",
"response_format",
"output_format",
"extra_body",
"user",
}
extra_body_from_kwargs = kwargs.pop("extra_body", None)
if isinstance(extra_body_from_kwargs, dict):
for key, value in extra_body_from_kwargs.items():
if value is None:
continue
mapped_params[key] = value
for key in list(kwargs.keys()):
if key in reserved_kwarg_keys:
continue
value = kwargs[key]
if value is None:
continue
mapped_params[key] = value
kwargs.pop(key, None)
if query_params:
kwargs[self.ELEVENLABS_QUERY_PARAMS_KEY] = query_params
else:
kwargs.pop(self.ELEVENLABS_QUERY_PARAMS_KEY, None)
kwargs[self.ELEVENLABS_VOICE_ID_KEY] = mapped_voice
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> "HttpxBinaryResponseContent":
"""
Wrap ElevenLabs binary audio response.
"""
from litellm.types.llms.openai import HttpxBinaryResponseContent
return HttpxBinaryResponseContent(raw_response)
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Construct the ElevenLabs endpoint URL, including path voice_id and query params.
"""
base_url = (
api_base
or get_secret_str("ELEVENLABS_API_BASE")
or self.TTS_BASE_URL
)
base_url = base_url.rstrip("/")
voice_id = litellm_params.get(self.ELEVENLABS_VOICE_ID_KEY)
if not isinstance(voice_id, str) or not voice_id.strip():
raise ValueError(
"ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`."
)
url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}"
query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {})
if query_params:
url = f"{url}?{urlencode(query_params)}"
return url

4
litellm/llms/gemini/count_tokens/handler.py
View file

@ -30,6 +30,10 @@ class GoogleAIStudioTokenCounter:
from google.genai.types import FunctionResponse
# Handle None or empty contents
if not contents:
return contents
cleaned_contents = copy.deepcopy(contents)
for content in cleaned_contents:

3
litellm/llms/gemini/image_generation/transformation.py
View file

@ -25,6 +25,7 @@ FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS = (
"2.0-flash-preview-image",
"2.0-flash-preview-image-generation",
"2.5-flash-image-preview",
"3-pro-image-preview",
)
class GoogleImageGenConfig(BaseImageGenerationConfig):
DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta"
@ -75,7 +76,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
"1792x1024": "16:9",
"1024x1792": "9:16",
"1280x896": "4:3",
"896x1280": "3:4"
"896x1280": "3:4",
}
return aspect_ratio_map.get(size, "1:1")

6
litellm/llms/gemini/vector_stores/__init__.py Normal file
View file

@ -0,0 +1,6 @@
"""Gemini File Search Vector Store module."""
from .transformation import GeminiVectorStoreConfig
__all__ = ["GeminiVectorStoreConfig"]

357
litellm/llms/gemini/vector_stores/transformation.py Normal file
View file

@ -0,0 +1,357 @@
"""
Gemini File Search Vector Store Transformation Layer.
Implements the transformation between LiteLLM's unified vector store API
and Google Gemini's File Search API.
"""
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.gemini.common_utils import (
GeminiError,
GeminiModelInfo,
get_api_key_from_env,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_stores import (
VECTOR_STORE_OPENAI_PARAMS,
BaseVectorStoreAuthCredentials,
VectorStoreCreateOptionalRequestParams,
VectorStoreCreateResponse,
VectorStoreFileCounts,
VectorStoreIndexEndpoints,
VectorStoreResultContent,
VectorStoreSearchOptionalRequestParams,
VectorStoreSearchResponse,
VectorStoreSearchResult,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class GeminiVectorStoreConfig(BaseVectorStoreConfig):
"""
Vector store configuration for Google Gemini File Search.
"""
def __init__(self) -> None:
super().__init__()
self.model_info = GeminiModelInfo()
self._cached_api_key: Optional[str] = None
def get_auth_credentials(
self, litellm_params: dict
) -> BaseVectorStoreAuthCredentials:
"""Gemini uses API key in query params, not headers."""
return {}
def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
"""
Gemini File Search endpoints.
Note: Search is done via generateContent with file_search tool,
not a dedicated search endpoint.
"""
return {
"read": [("POST", "/models/{model}:generateContent")],
"write": [("POST", "/fileSearchStores")],
}
def get_supported_openai_params(
self, model: str
) -> List[VECTOR_STORE_OPENAI_PARAMS]:
"""Supported parameters for Gemini File Search."""
return ["max_num_results", "filters"]
def validate_environment(
self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
"""Validate and set up headers for Gemini API."""
headers = headers or {}
headers.setdefault("Content-Type", "application/json")
if litellm_params:
api_key = litellm_params.get("api_key") or get_api_key_from_env()
if api_key:
self._cached_api_key = api_key
return headers
def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str:
"""
Get the complete base URL for Gemini API.
Note: This returns the base URL WITHOUT the API key.
The API key will be appended to specific endpoint URLs in the transform methods.
"""
if api_base is None:
api_base = GeminiModelInfo.get_api_base()
if api_base is None:
raise ValueError("GEMINI_API_BASE is not set")
# Ensure we're using the v1beta version for File Search
api_version = "v1beta"
return f"{api_base}/{api_version}"
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> GeminiError:
"""Return Gemini-specific error class."""
return GeminiError(
status_code=status_code,
message=error_message,
headers=headers,
)
def transform_search_vector_store_request(
self,
vector_store_id: str,
query: Union[str, List[str]],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
api_base: str,
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
) -> Tuple[str, Dict]:
"""
Transform search request to Gemini's generateContent format.
Gemini File Search works by calling generateContent with a file_search tool.
"""
# Convert query list to single string if needed
if isinstance(query, list):
query = " ".join(query)
# Get model from litellm_params or use default
# Note: File Search requires gemini-2.5-flash or later
model = litellm_params.get("model") or "gemini-2.5-flash"
if model and model.startswith("gemini/"):
model = model.replace("gemini/", "")
# Get API key - Gemini requires it as a query parameter
api_key = litellm_params.get("api_key") or GeminiModelInfo.get_api_key()
if not api_key:
raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required")
# Build the URL for generateContent with API key
url = f"{api_base}/models/{model}:generateContent?key={api_key}"
# Build file_search tool configuration (using snake_case as per Gemini docs)
file_search_config: Dict[str, Any] = {
"file_search_store_names": [vector_store_id]
}
# Add metadata filter if provided
metadata_filter = vector_store_search_optional_params.get("filters")
if metadata_filter:
# Convert to Gemini filter syntax if it's a dict
if isinstance(metadata_filter, dict):
# Simple conversion - may need more sophisticated mapping
filter_parts = []
for key, value in metadata_filter.items():
if isinstance(value, str):
filter_parts.append(f'{key} = "{value}"')
else:
filter_parts.append(f'{key} = {value}')
file_search_config["metadata_filter"] = " AND ".join(filter_parts)
else:
file_search_config["metadata_filter"] = metadata_filter
# Build request body
request_body: Dict[str, Any] = {
"contents": [
{
"parts": [{"text": query}]
}
],
"tools": [
{
"file_search": file_search_config
}
],
}
# Add max_num_results if specified
max_results = vector_store_search_optional_params.get("max_num_results")
if max_results:
# This might need to be added to generationConfig or tool config
# depending on Gemini's API requirements
request_body.setdefault("generationConfig", {})["candidateCount"] = 1
litellm_logging_obj.model_call_details["query"] = query
litellm_logging_obj.model_call_details["vector_store_id"] = vector_store_id
return url, request_body
def transform_search_vector_store_response(
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
) -> VectorStoreSearchResponse:
"""
Transform Gemini's generateContent response to standard format.
Extracts grounding metadata and citations from the response.
"""
try:
response_data = response.json()
results: List[VectorStoreSearchResult] = []
# Extract candidates and grounding metadata
candidates = response_data.get("candidates", [])
for candidate in candidates:
grounding_metadata = candidate.get("groundingMetadata", {})
grounding_chunks = grounding_metadata.get("groundingChunks", [])
# Process each grounding chunk
for chunk in grounding_chunks:
retrieved_context = chunk.get("retrievedContext")
if retrieved_context:
# This is from file search
text = retrieved_context.get("text", "")
uri = retrieved_context.get("uri", "")
title = retrieved_context.get("title", "")
# Extract file_id from URI if available
file_id = uri if uri else None
results.append(
VectorStoreSearchResult(
score=None, # Gemini doesn't provide explicit scores
content=[VectorStoreResultContent(text=text, type="text")],
file_id=file_id,
filename=title if title else None,
attributes={
"uri": uri,
"title": title,
},
)
)
# Also extract from grounding supports for more detailed citations
grounding_supports = grounding_metadata.get("groundingSupports", [])
for support in grounding_supports:
segment = support.get("segment", {})
text = segment.get("text", "")
grounding_chunk_indices = support.get("groundingChunkIndices", [])
confidence_scores = support.get("confidenceScores", [])
# Use first confidence score as relevance score
score = confidence_scores[0] if confidence_scores else None
# Only add if we have meaningful text and it's not a duplicate
if text:
already_exists = False
for record in results:
contents = record.get("content") or []
if contents and contents[0].get("text") == text:
already_exists = True
break
if already_exists:
continue
results.append(
VectorStoreSearchResult(
score=score,
content=[VectorStoreResultContent(text=text, type="text")],
attributes={
"grounding_chunk_indices": grounding_chunk_indices,
},
)
)
query = litellm_logging_obj.model_call_details.get("query", "")
return VectorStoreSearchResponse(
object="vector_store.search_results.page",
search_query=query,
data=results,
)
except Exception as e:
raise self.get_error_class(
error_message=f"Failed to parse Gemini response: {str(e)}",
status_code=response.status_code,
headers=response.headers,
)
def transform_create_vector_store_request(
self,
vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
api_base: str,
) -> Tuple[str, Dict]:
"""
Transform create request to Gemini's fileSearchStores format.
"""
url = f"{api_base}/fileSearchStores"
# Append API key as query parameter (required by Gemini)
api_key = self._cached_api_key or get_api_key_from_env()
if api_key:
url = f"{url}?key={api_key}"
request_body: Dict[str, Any] = {}
# Add display name if provided
name = vector_store_create_optional_params.get("name")
if name:
request_body["displayName"] = name
return url, request_body
def transform_create_vector_store_response(
self, response: httpx.Response
) -> VectorStoreCreateResponse:
"""
Transform Gemini's fileSearchStore response to standard format.
"""
try:
response_data = response.json()
# Extract store name (format: fileSearchStores/xxxxxxx)
store_name = response_data.get("name", "")
display_name = response_data.get("displayName", "")
create_time = response_data.get("createTime", "")
# Convert ISO timestamp to Unix timestamp
import datetime
created_at = None
if create_time:
try:
dt = datetime.datetime.fromisoformat(create_time.replace("Z", "+00:00"))
created_at = int(dt.timestamp())
except Exception:
created_at = None
return VectorStoreCreateResponse(
id=store_name,
object="vector_store",
created_at=created_at or 0,
name=display_name,
bytes=0, # Gemini doesn't provide size info on creation
file_counts=VectorStoreFileCounts(
in_progress=0,
completed=0,
failed=0,
cancelled=0,
total=0,
),
status="completed",
expires_after=None,
expires_at=None,
last_active_at=None,
metadata=None,
)
except Exception as e:
raise self.get_error_class(
error_message=f"Failed to parse Gemini create response: {str(e)}",
status_code=response.status_code,
headers=response.headers,
)

5
litellm/llms/gemini/videos/transformation.py
View file

@ -160,11 +160,16 @@ class GeminiVideoConfig(BaseVideoConfig):
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""
Validate environment and add Gemini API key to headers.
Gemini uses x-goog-api-key header for authentication.
"""
# Use api_key from litellm_params if available, otherwise fall back to other sources
if litellm_params and litellm_params.api_key:
api_key = api_key or litellm_params.api_key
api_key = (
api_key
or litellm.api_key

25
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -25,6 +25,15 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
def is_model_gpt_5_codex_model(cls, model: str) -> bool:
"""Check if the model is specifically a GPT-5 Codex variant."""
return "gpt-5-codex" in model
@classmethod
def is_model_gpt_5_1_model(cls, model: str) -> bool:
"""Check if the model is a gpt-5.1 variant.
gpt-5.1 supports temperature when reasoning_effort="none",
unlike gpt-5 which only supports temperature=1.
"""
return "gpt-5.1" in model
def get_supported_openai_params(self, model: str) -> list:
from litellm.utils import supports_tool_choice
@ -69,14 +78,26 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
if "temperature" in non_default_params:
temperature_value: Optional[float] = non_default_params.pop("temperature")
if temperature_value is not None:
if temperature_value == 1:
is_gpt_5_1 = self.is_model_gpt_5_1_model(model)
reasoning_effort = (
non_default_params.get("reasoning_effort")
or optional_params.get("reasoning_effort")
)
# gpt-5.1 supports any temperature when reasoning_effort="none" (or not specified, as it defaults to "none")
if is_gpt_5_1 and (reasoning_effort == "none" or reasoning_effort is None):
optional_params["temperature"] = temperature_value
elif temperature_value == 1:
optional_params["temperature"] = temperature_value
elif litellm.drop_params or drop_params:
pass
else:
raise litellm.utils.UnsupportedParamsError(
message=(
"gpt-5 models (including gpt-5-codex) don't support temperature={}. Only temperature=1 is supported. To drop unsupported params set `litellm.drop_params = True`"
"gpt-5 models (including gpt-5-codex) don't support temperature={}. "
"Only temperature=1 is supported. "
"For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). "
"To drop unsupported params set `litellm.drop_params = True`"
).format(temperature_value),
status_code=400,
)

5
litellm/llms/openai/videos/transformation.py
View file

@ -61,7 +61,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
# Use api_key from litellm_params if available, otherwise fall back to other sources
if litellm_params and litellm_params.api_key:
api_key = api_key or litellm_params.api_key
api_key = (
api_key
or litellm.api_key

5
litellm/llms/runwayml/videos/transformation.py
View file

@ -114,11 +114,16 @@ class RunwayMLVideoConfig(BaseVideoConfig):
headers: dict,
model: str,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""
Validate environment and set up authentication headers.
RunwayML uses Bearer token authentication via RUNWAYML_API_SECRET.
"""
# Use api_key from litellm_params if available, otherwise fall back to other sources
if litellm_params and litellm_params.api_key:
api_key = api_key or litellm_params.api_key
api_key = (
api_key
or litellm.api_key

54
litellm/llms/vertex_ai/common_utils.py
View file

@ -274,6 +274,57 @@ def _fix_enum_empty_strings(schema, depth=0):
_fix_enum_empty_strings(items, depth=depth + 1)
def _fix_enum_types(schema, depth=0):
"""Remove `enum` fields when the schema type is not string.
Gemini / Vertex APIs only allow enums for string-typed fields. When an enum
is present on a non-string typed property (or when `anyOf` types do not
include a string type), remove the enum to avoid provider validation errors.
"""
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(
f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema."
)
if not isinstance(schema, dict):
return
# If enum exists but type is not string (and anyOf doesn't include string), drop enum
if "enum" in schema and isinstance(schema["enum"], list):
schema_type = schema.get("type")
keep_enum = False
if isinstance(schema_type, str) and schema_type.lower() == "string":
keep_enum = True
else:
anyof = schema.get("anyOf")
if isinstance(anyof, list):
for item in anyof:
if isinstance(item, dict):
item_type = item.get("type")
if isinstance(item_type, str) and item_type.lower() == "string":
keep_enum = True
break
if not keep_enum:
schema.pop("enum", None)
# Recurse into nested structures
properties = schema.get("properties", None)
if properties is not None:
for _, value in properties.items():
_fix_enum_types(value, depth=depth + 1)
items = schema.get("items", None)
if items is not None:
_fix_enum_types(items, depth=depth + 1)
anyof = schema.get("anyOf", None)
if anyof is not None and isinstance(anyof, list):
for item in anyof:
if isinstance(item, dict):
_fix_enum_types(item, depth=depth + 1)
def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
"""
This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
@ -307,6 +358,9 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
# Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
_fix_enum_empty_strings(parameters)
# Remove enums for non-string typed fields (Gemini requires enum only on strings)
_fix_enum_types(parameters)
# Handle empty items objects
process_items(parameters)
add_object_type(parameters)

16
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -904,13 +904,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
if VertexGeminiConfig._is_gemini_3_or_newer(model):
if "temperature" not in optional_params:
optional_params["temperature"] = 1.0
thinking_config = optional_params.get("thinkingConfig", {})
if (
"thinkingLevel" not in thinking_config
and "thinkingBudget" not in thinking_config
):
thinking_config["thinkingLevel"] = "low"
optional_params["thinkingConfig"] = thinking_config
# Only add thinkingLevel if model supports it (exclude image models)
if "image" not in model.lower():
thinking_config = optional_params.get("thinkingConfig", {})
if (
"thinkingLevel" not in thinking_config
and "thinkingBudget" not in thinking_config
):
thinking_config["thinkingLevel"] = "low"
optional_params["thinkingConfig"] = thinking_config
return optional_params

43
litellm/llms/vertex_ai/image_generation/__init__.py Normal file
View file

@ -0,0 +1,43 @@
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
get_vertex_ai_model_route,
)
from .vertex_gemini_transformation import VertexAIGeminiImageGenerationConfig
from .vertex_imagen_transformation import VertexAIImagenImageGenerationConfig
__all__ = [
"VertexAIGeminiImageGenerationConfig",
"VertexAIImagenImageGenerationConfig",
"get_vertex_ai_image_generation_config",
]
def get_vertex_ai_image_generation_config(model: str) -> BaseImageGenerationConfig:
"""
Get the appropriate image generation config for a Vertex AI model.
Routes to the correct transformation class based on the model type:
- Gemini image generation models use generateContent API (VertexAIGeminiImageGenerationConfig)
- Imagen models use predict API (VertexAIImagenImageGenerationConfig)
Args:
model: The model name (e.g., "gemini-2.5-flash-image", "imagegeneration@006")
Returns:
BaseImageGenerationConfig: The appropriate configuration class
"""
# Determine the model route
model_route = get_vertex_ai_model_route(model)
if model_route == VertexAIModelRoute.GEMINI:
# Gemini models use generateContent API
return VertexAIGeminiImageGenerationConfig()
else:
# Default to Imagen for other models (imagegeneration, etc.)
# This includes NON_GEMINI models like imagegeneration@006
return VertexAIImagenImageGenerationConfig()

264
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py Normal file
View file

@ -0,0 +1,264 @@
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional
import httpx
import litellm
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
"""
Vertex AI Gemini Image Generation Configuration
Uses generateContent API for Gemini image generation models on Vertex AI
Supports models like gemini-2.5-flash-image, gemini-3-pro-image-preview, etc.
"""
def __init__(self) -> None:
BaseImageGenerationConfig.__init__(self)
VertexLLM.__init__(self)
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
"""
Gemini image generation supported parameters
"""
return [
"n",
"size",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
mapped_params = {}
for k, v in non_default_params.items():
if k not in optional_params.keys():
if k in supported_params:
# Map OpenAI parameters to Gemini format
if k == "n":
mapped_params["candidate_count"] = v
elif k == "size":
# Map OpenAI size format to Gemini aspectRatio
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v)
else:
mapped_params[k] = v
return mapped_params
def _map_size_to_aspect_ratio(self, size: str) -> str:
"""
Map OpenAI size format to Gemini aspect ratio format
"""
aspect_ratio_map = {
"1024x1024": "1:1",
"1792x1024": "16:9",
"1024x1792": "9:16",
"1280x896": "4:3",
"896x1280": "3:4"
}
return aspect_ratio_map.get(size, "1:1")
def _resolve_vertex_project(self) -> Optional[str]:
return (
getattr(self, "_vertex_project", None)
or os.environ.get("VERTEXAI_PROJECT")
or getattr(litellm, "vertex_project", None)
or get_secret_str("VERTEXAI_PROJECT")
)
def _resolve_vertex_location(self) -> Optional[str]:
return (
getattr(self, "_vertex_location", None)
or os.environ.get("VERTEXAI_LOCATION")
or os.environ.get("VERTEX_LOCATION")
or getattr(litellm, "vertex_location", None)
or get_secret_str("VERTEXAI_LOCATION")
or get_secret_str("VERTEX_LOCATION")
)
def _resolve_vertex_credentials(self) -> Optional[str]:
return (
getattr(self, "_vertex_credentials", None)
or os.environ.get("VERTEXAI_CREDENTIALS")
or getattr(litellm, "vertex_credentials", None)
or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
or get_secret_str("VERTEXAI_CREDENTIALS")
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for Vertex AI Gemini generateContent API
"""
vertex_project = self._resolve_vertex_project()
vertex_location = self._resolve_vertex_location()
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Use the model name as provided, handling vertex_ai prefix
model_name = model
if model.startswith("vertex_ai/"):
model_name = model.replace("vertex_ai/", "")
if api_base:
base_url = api_base.rstrip("/")
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent"
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
headers = headers or {}
vertex_project = self._resolve_vertex_project()
vertex_credentials = self._resolve_vertex_credentials()
access_token, _ = self._ensure_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
)
return self.set_headers(access_token, headers)
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the image generation request to Gemini format
Uses generateContent API with responseModalities: ["IMAGE"]
"""
# Prepare messages with the prompt
contents = [
{
"role": "user",
"parts": [{"text": prompt}]
}
]
# Prepare generation config
generation_config: Dict[str, Any] = {
"responseModalities": ["IMAGE"]
}
# Handle image-specific config parameters
image_config: Dict[str, Any] = {}
# Map aspectRatio
if "aspectRatio" in optional_params:
image_config["aspectRatio"] = optional_params["aspectRatio"]
elif "aspect_ratio" in optional_params:
image_config["aspectRatio"] = optional_params["aspect_ratio"]
# Map imageSize (for Gemini 3 Pro)
if "imageSize" in optional_params:
image_config["imageSize"] = optional_params["imageSize"]
elif "image_size" in optional_params:
image_config["imageSize"] = optional_params["image_size"]
if image_config:
generation_config["imageConfig"] = image_config
# Handle candidate_count (n parameter)
if "candidate_count" in optional_params:
generation_config["candidateCount"] = optional_params["candidate_count"]
elif "n" in optional_params:
generation_config["candidateCount"] = optional_params["n"]
request_body: Dict[str, Any] = {
"contents": contents,
"generationConfig": generation_config
}
return request_body
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform Gemini image generation response to litellm ImageResponse format
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error transforming image generation response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
# Gemini image generation models return in candidates format
candidates = response_data.get("candidates", [])
for candidate in candidates:
content = candidate.get("content", {})
parts = content.get("parts", [])
for part in parts:
# Look for inlineData with image
if "inlineData" in part:
inline_data = part["inlineData"]
if "data" in inline_data:
model_response.data.append(ImageObject(
b64_json=inline_data["data"],
url=None,
))
return model_response

230
litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py Normal file
View file

@ -0,0 +1,230 @@
import os
from typing import TYPE_CHECKING, Any, List, Optional
import httpx
import litellm
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
"""
Vertex AI Imagen Image Generation Configuration
Uses predict API for Imagen models on Vertex AI
Supports models like imagegeneration@006
"""
def __init__(self) -> None:
BaseImageGenerationConfig.__init__(self)
VertexLLM.__init__(self)
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
"""
Imagen API supported parameters
"""
return [
"n",
"size"
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
mapped_params = {}
for k, v in non_default_params.items():
if k not in optional_params.keys():
if k in supported_params:
# Map OpenAI parameters to Imagen format
if k == "n":
mapped_params["sampleCount"] = v
elif k == "size":
# Map OpenAI size format to Imagen aspectRatio
mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v)
else:
mapped_params[k] = v
return mapped_params
def _map_size_to_aspect_ratio(self, size: str) -> str:
"""
Map OpenAI size format to Imagen aspect ratio format
"""
aspect_ratio_map = {
"1024x1024": "1:1",
"1792x1024": "16:9",
"1024x1792": "9:16",
"1280x896": "4:3",
"896x1280": "3:4"
}
return aspect_ratio_map.get(size, "1:1")
def _resolve_vertex_project(self) -> Optional[str]:
return (
getattr(self, "_vertex_project", None)
or os.environ.get("VERTEXAI_PROJECT")
or getattr(litellm, "vertex_project", None)
or get_secret_str("VERTEXAI_PROJECT")
)
def _resolve_vertex_location(self) -> Optional[str]:
return (
getattr(self, "_vertex_location", None)
or os.environ.get("VERTEXAI_LOCATION")
or os.environ.get("VERTEX_LOCATION")
or getattr(litellm, "vertex_location", None)
or get_secret_str("VERTEXAI_LOCATION")
or get_secret_str("VERTEX_LOCATION")
)
def _resolve_vertex_credentials(self) -> Optional[str]:
return (
getattr(self, "_vertex_credentials", None)
or os.environ.get("VERTEXAI_CREDENTIALS")
or getattr(litellm, "vertex_credentials", None)
or os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
or get_secret_str("VERTEXAI_CREDENTIALS")
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for Vertex AI Imagen predict API
"""
vertex_project = self._resolve_vertex_project()
vertex_location = self._resolve_vertex_location()
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Use the model name as provided, handling vertex_ai prefix
model_name = model
if model.startswith("vertex_ai/"):
model_name = model.replace("vertex_ai/", "")
if api_base:
base_url = api_base.rstrip("/")
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict"
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
headers = headers or {}
vertex_project = self._resolve_vertex_project()
vertex_credentials = self._resolve_vertex_credentials()
access_token, _ = self._ensure_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
)
return self.set_headers(access_token, headers)
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the image generation request to Imagen format
Uses predict API with instances and parameters
"""
# Default parameters
default_params = {
"sampleCount": 1,
}
# Merge with optional params
parameters = {**default_params, **optional_params}
request_body = {
"instances": [{"prompt": prompt}],
"parameters": parameters,
}
return request_body
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform Imagen image generation response to litellm ImageResponse format
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error transforming image generation response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
# Imagen format - predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Imagen returns images as bytesBase64Encoded
if "bytesBase64Encoded" in prediction:
model_response.data.append(ImageObject(
b64_json=prediction["bytesBase64Encoded"],
url=None,
))
return model_response

2
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -65,6 +65,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
# Use custom api_base if provided, otherwise construct default
if api_base:
base_url = api_base
elif vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"

12
litellm/llms/vertex_ai/videos/transformation.py
View file

@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, cast
import httpx
from httpx._types import RequestFiles
from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.vertex_ai.common_utils import (
_convert_vertex_datetime_to_openai_datetime,
@ -23,8 +25,6 @@ from litellm.types.videos.utils import (
encode_video_id_with_provider,
extract_original_video_id,
)
from litellm.images.utils import ImageEditRequestUtils
from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -160,13 +160,11 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
def validate_environment(
self,
headers: Dict,
headers: dict,
model: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
litellm_params: Optional[dict] = None,
**kwargs,
) -> Dict:
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""
Validate environment and return headers for Vertex AI OCR.

136
litellm/main.py
View file

@ -154,6 +154,7 @@ from .litellm_core_utils.prompt_templates.factory import (
)
from .litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor
from .llms.anthropic.chat import AnthropicChatCompletion
from .llms.azure.anthropic.handler import AzureAnthropicChatCompletion
from .llms.azure.audio_transcriptions import AzureAudioTranscription
from .llms.azure.azure import AzureChatCompletion, _check_dynamic_azure_params
from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion
@ -255,6 +256,7 @@ openai_image_variations = OpenAIImageVariationsHandler()
groq_chat_completions = GroqChatCompletion()
azure_ai_embedding = AzureAIEmbedding()
anthropic_chat_completions = AnthropicChatCompletion()
azure_anthropic_chat_completions = AzureAnthropicChatCompletion()
azure_chat_completions = AzureChatCompletion()
azure_o1_chat_completions = AzureOpenAIO1ChatCompletion()
azure_text_completions = AzureTextCompletion()
@ -2357,6 +2359,70 @@ def completion( # type: ignore # noqa: PLR0915
original_response=response,
)
response = response
elif custom_llm_provider == "azure_anthropic":
# Azure Anthropic uses same API as Anthropic but with Azure authentication
api_key = (
api_key
or litellm.azure_key
or litellm.api_key
or get_secret("AZURE_API_KEY")
or get_secret("AZURE_OPENAI_API_KEY")
)
custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict
# Azure Foundry endpoint format: https://<resource-name>.services.ai.azure.com/anthropic/v1/messages
api_base = (
api_base
or litellm.api_base
or get_secret("AZURE_API_BASE")
)
if api_base is None:
raise ValueError(
"Missing Azure API Base - Please set `api_base` or `AZURE_API_BASE` environment variable. "
"Expected format: https://<resource-name>.services.ai.azure.com/anthropic"
)
# Ensure the URL ends with /v1/messages
api_base = api_base.rstrip("/")
if api_base.endswith("/v1/messages"):
pass
elif api_base.endswith("/anthropic/v1/messages"):
pass
else:
if "/anthropic" in api_base:
parts = api_base.split("/anthropic", 1)
api_base = parts[0] + "/anthropic"
else:
api_base = api_base + "/anthropic"
api_base = api_base + "/v1/messages"
response = azure_anthropic_chat_completions.completion(
model=model,
messages=messages,
api_base=api_base,
acompletion=acompletion,
custom_prompt_dict=litellm.custom_prompt_dict,
model_response=model_response,
print_verbose=print_verbose,
optional_params=optional_params,
litellm_params=litellm_params,
logger_fn=logger_fn,
encoding=encoding, # for calculating input/output tokens
api_key=api_key,
logging_obj=logging,
headers=headers,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
)
if optional_params.get("stream", False) or acompletion is True:
## LOGGING
logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
)
response = response
elif custom_llm_provider == "nlp_cloud":
nlp_cloud_key = (
api_key
@ -4019,7 +4085,11 @@ def embedding( # noqa: PLR0915
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
aembedding: Optional[bool] = kwargs.get("aembedding", None)
extra_headers = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None)
headers = kwargs.get("headers", None) or extra_headers
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
### CUSTOM MODEL COST ###
input_cost_per_token = kwargs.get("input_cost_per_token", None)
output_cost_per_token = kwargs.get("output_cost_per_token", None)
@ -4330,7 +4400,7 @@ def embedding( # noqa: PLR0915
litellm_params={},
api_base=api_base,
print_verbose=print_verbose,
extra_headers=extra_headers,
extra_headers=headers,
api_key=api_key,
)
elif custom_llm_provider == "triton":
@ -5764,7 +5834,9 @@ def speech( # noqa: PLR0915
custom_llm_provider: Optional[str] = None,
aspeech: Optional[bool] = None,
**kwargs,
) -> HttpxBinaryResponseContent:
) -> Union[
HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
]:
user = kwargs.get("user", None)
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
proxy_server_request = kwargs.get("proxy_server_request", None)
@ -5824,7 +5896,11 @@ def speech( # noqa: PLR0915
},
custom_llm_provider=custom_llm_provider,
)
response: Optional[HttpxBinaryResponseContent] = None
response: Union[
HttpxBinaryResponseContent,
Coroutine[Any, Any, HttpxBinaryResponseContent],
None,
] = None
if (
custom_llm_provider == "openai"
or custom_llm_provider in litellm.openai_compatible_providers
@ -5962,6 +6038,58 @@ def speech( # noqa: PLR0915
aspeech=aspeech,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "elevenlabs":
from litellm.llms.elevenlabs.text_to_speech.transformation import (
ElevenLabsTextToSpeechConfig,
)
if text_to_speech_provider_config is None:
text_to_speech_provider_config = ElevenLabsTextToSpeechConfig()
elevenlabs_config = cast(
ElevenLabsTextToSpeechConfig, text_to_speech_provider_config
)
voice_id = voice if isinstance(voice, str) else None
if voice_id is None or not voice_id.strip():
raise litellm.BadRequestError(
message="'voice' must resolve to an ElevenLabs voice id for ElevenLabs TTS",
model=model,
llm_provider=custom_llm_provider,
)
voice_id = voice_id.strip()
query_params = kwargs.pop(
ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY, None
)
if isinstance(query_params, dict):
litellm_params_dict[
ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY
] = query_params
litellm_params_dict[
ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY
] = voice_id
if api_base is not None:
litellm_params_dict["api_base"] = api_base
if api_key is not None:
litellm_params_dict["api_key"] = api_key
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice_id,
text_to_speech_provider_config=elevenlabs_config,
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=logging_obj,
timeout=timeout,
extra_headers=extra_headers,
client=client,
_is_async=aspeech or False,
)
elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
generic_optional_params = GenericLiteLLMParams(**kwargs)

115
litellm/model_prices_and_context_window_backup.json
View file

@ -1143,6 +1143,60 @@
"output_cost_per_token": 1.5e-05,
"supports_function_calling": true
},
"azure/claude-haiku-4-5": {
"input_cost_per_token": 1e-06,
"litellm_provider": "azure_anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/claude-opus-4-1": {
"input_cost_per_token": 1.5e-05,
"litellm_provider": "azure_anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/claude-sonnet-4-5": {
"input_cost_per_token": 3e-06,
"litellm_provider": "azure_anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/computer-use-preview": {
"input_cost_per_token": 3e-06,
"litellm_provider": "azure",
@ -9468,6 +9522,15 @@
"output_cost_per_token": 0.0,
"supports_embedding_image_input": true
},
"embed-multilingual-light-v3.0": {
"input_cost_per_token": 1e-04,
"litellm_provider": "cohere",
"max_input_tokens": 1024,
"max_tokens": 1024,
"mode": "embedding",
"output_cost_per_token": 0.0,
"supports_embedding_image_input": true
},
"eu.amazon.nova-lite-v1:0": {
"input_cost_per_token": 7.8e-08,
"litellm_provider": "bedrock_converse",
@ -24551,6 +24614,58 @@
"supports_tool_choice": true,
"supports_vision": true
},
"vertex_ai/claude-opus-4-5": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"vertex_ai/claude-opus-4-5@20251101": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 2.5e-05,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
},
"vertex_ai/claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,

2
litellm/passthrough/main.py
View file

@ -258,7 +258,7 @@ def llm_passthrough_route(
model=model,
messages=[],
optional_params={},
litellm_params={},
litellm_params=litellm_params_dict,
api_key=provider_api_key,
api_base=base_target_url,
)

2
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -1976,7 +1976,7 @@ class MCPServerManager:
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
self.add_update_server(server)
await self.add_update_server(server)
verbose_logger.debug(
f"Registry now contains {len(self.get_registry())} servers"

6
litellm/proxy/_types.py
View file

@ -348,6 +348,8 @@ class LiteLLMRoutes(enum.Enum):
# search
"/search",
"/v1/search",
"/search/{search_tool_name}",
"/v1/search/{search_tool_name}",
# OCR
"/ocr",
"/v1/ocr",
@ -1889,6 +1891,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
allowed_routes: Optional[List] = Field(
None, description="Proxy API Endpoints you want users to be able to access"
)
reject_clientside_metadata_tags: Optional[bool] = Field(
None,
description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.",
)
enable_public_model_hub: bool = Field(
default=False,
description="Public model hub for users to see what models they have access to, supported openai params, etc.",

27
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -154,8 +154,13 @@ async def anthropic_response( # noqa: PLR0915
response = responses[1]
# Extract model_id from request metadata (set by router during routing)
litellm_metadata = data.get("litellm_metadata", {}) or {}
model_info = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id", "") or ""
# Get other metadata from hidden_params
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
@ -216,12 +221,32 @@ async def anthropic_response( # noqa: PLR0915
str(e)
)
)
# Extract model_id from request metadata (same as success path)
litellm_metadata = data.get("litellm_metadata", {}) or {}
model_info = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id", "") or ""
# Get headers
headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
call_id=data.get("litellm_call_id", ""),
model_id=model_id,
version=version,
response_cost=0,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
request_data=data,
timeout=getattr(e, "timeout", None),
litellm_logging_obj=None,
)
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
headers=headers,
)

18
litellm/proxy/auth/auth_checks.py
View file

@ -186,6 +186,24 @@ async def common_checks(
raise Exception(
f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}"
)
# 6.1 [OPTIONAL] If 'reject_clientside_metadata_tags' enabled - reject request if it has client-side 'metadata.tags'
if (
general_settings.get("reject_clientside_metadata_tags", None) is not None
and general_settings["reject_clientside_metadata_tags"] is True
):
if (
RouteChecks.is_llm_api_route(route=route)
and "metadata" in request_body
and isinstance(request_body["metadata"], dict)
and "tags" in request_body["metadata"]
):
raise ProxyException(
message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.",
type=ProxyErrorTypes.bad_request_error,
param="metadata.tags",
code=status.HTTP_400_BAD_REQUEST,
)
# 7. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget
if (
litellm.max_budget > 0

66
litellm/proxy/common_request_processing.py
View file

@ -330,6 +330,7 @@ class ProxyBaseLLMRequestProcessing:
"avideo_remix",
"acreate_container",
"alist_containers",
"aingest",
"aretrieve_container",
"adelete_container",
"acreate_skill",
@ -344,6 +345,7 @@ class ProxyBaseLLMRequestProcessing:
user_max_tokens: Optional[int] = None,
user_api_base: Optional[str] = None,
model: Optional[str] = None,
llm_router: Optional[Router] = None,
) -> Tuple[dict, LiteLLMLoggingObj]:
start_time = datetime.now() # start before calling guardrail hooks
@ -452,6 +454,7 @@ class ProxyBaseLLMRequestProcessing:
"avideo_remix",
"acreate_container",
"alist_containers",
"aingest",
"aretrieve_container",
"adelete_container",
"acreate_skill",
@ -498,6 +501,7 @@ class ProxyBaseLLMRequestProcessing:
user_api_base=user_api_base,
model=model,
route_type=route_type,
llm_router=llm_router,
)
tasks = []
@ -536,6 +540,13 @@ class ProxyBaseLLMRequestProcessing:
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", None) or ""
# Fallback: extract model_id from litellm_metadata if not in hidden_params
if not model_id:
litellm_metadata = self.data.get("litellm_metadata", {}) or {}
model_info = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id", "") or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
@ -756,11 +767,19 @@ class ProxyBaseLLMRequestProcessing:
_litellm_logging_obj: Optional[LiteLLMLoggingObj] = self.data.get(
"litellm_logging_obj", None
)
# Attempt to get model_id from logging object
#
# Note: We check the direct model_info path first (not nested in metadata) because that's where the router sets it.
# The nested metadata path is only a fallback for cases where model_info wasn't set at the top level.
model_id = self.maybe_get_model_id(_litellm_logging_obj)
custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
call_id=(
_litellm_logging_obj.litellm_call_id if _litellm_logging_obj else None
),
model_id=model_id,
version=version,
response_cost=0,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
@ -1073,3 +1092,50 @@ class ProxyBaseLLMRequestProcessing:
obj.setdefault("usage", {})["cost"] = cost_val
return obj
return None
def maybe_get_model_id(self, _logging_obj: Optional[LiteLLMLoggingObj]) -> Optional[str]:
"""
Get model_id from logging object or request metadata.
The router sets model_info.id when selecting a deployment. This tries multiple locations
where the ID might be stored depending on the request lifecycle stage.
"""
model_id = None
if _logging_obj:
# 1. Try getting from litellm_params (updated during call)
if (
hasattr(_logging_obj, "litellm_params")
and _logging_obj.litellm_params
):
# First check direct model_info path (set by router.py with selected deployment)
model_info = _logging_obj.litellm_params.get("model_info") or {}
model_id = model_info.get("id", None)
# Fallback to nested metadata path
if not model_id:
metadata = _logging_obj.litellm_params.get("metadata") or {}
model_info = metadata.get("model_info") or {}
model_id = model_info.get("id", None)
# 2. Fallback to kwargs (initial)
if not model_id:
_kwargs = getattr(_logging_obj, "kwargs", None)
if _kwargs:
litellm_params = _kwargs.get("litellm_params", {})
# First check direct model_info path
model_info = litellm_params.get("model_info") or {}
model_id = model_info.get("id", None)
# Fallback to nested metadata path
if not model_id:
metadata = litellm_params.get("metadata") or {}
model_info = metadata.get("model_info") or {}
model_id = model_info.get("id", None)
# 3. Final fallback to self.data["litellm_metadata"] (for routes like /v1/responses that populate data before error)
if not model_id:
litellm_metadata = self.data.get("litellm_metadata", {}) or {}
model_info = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id", None)
return model_id

2
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -39,6 +39,8 @@ async def _read_request_body(request: Optional[Request]) -> Dict:
if "form" in content_type:
parsed_body = dict(await request.form())
if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):
parsed_body["metadata"] = json.loads(parsed_body["metadata"])
else:
# Read the request body
body = await request.body()

14
litellm/proxy/example_config_yaml/reject_clientside_metadata_tags_config.yaml Normal file
View file

@ -0,0 +1,14 @@
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234
database_url: "postgresql://user:password@localhost:5432/litellm"
# Reject requests that contain client-side metadata.tags
# This prevents users from influencing budgets by sending different tags
# Tags can only be inherited from the API key metadata
reject_clientside_metadata_tags: true

99
litellm/proxy/guardrails/guardrail_endpoints.py
View file

@ -31,6 +31,7 @@ from litellm.types.guardrails import (
PiiEntityType,
PresidioPresidioConfigModelUserInterface,
SupportedGuardrailIntegrations,
ToolPermissionGuardrailConfigModel,
)
#### GUARDRAILS ENDPOINTS ####
@ -635,7 +636,9 @@ async def get_guardrail_info(guardrail_id: str):
raise HTTPException(status_code=500, detail="Prisma client not initialized")
try:
guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = GUARDRAIL_DEFINITION_LOCATION.DB
guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = (
GUARDRAIL_DEFINITION_LOCATION.DB
)
result = await GUARDRAIL_REGISTRY.get_guardrail_by_id_from_db(
guardrail_id=guardrail_id, prisma_client=prisma_client
)
@ -702,10 +705,12 @@ async def get_guardrail_ui_settings():
# Convert the PII_ENTITY_CATEGORIES_MAP to the format expected by the UI
category_maps = []
for category, entities in PII_ENTITY_CATEGORIES_MAP.items():
category_maps.append({
"category": category.value,
"entities": [entity.value for entity in entities]
})
category_maps.append(
{
"category": category.value,
"entities": [entity.value for entity in entities],
}
)
return GuardrailUIAddGuardrailSettings(
supported_entities=[entity.value for entity in PiiEntityType],
@ -728,20 +733,20 @@ async def get_guardrail_ui_settings():
async def validate_blocked_words_file(request: Dict[str, str]):
"""
Validate a blocked_words YAML file content.
Args:
request: Dictionary with 'file_content' key containing the YAML string
Returns:
Dictionary with 'valid' boolean and either 'message'/'errors' depending on result
Example Request:
```json
{
"file_content": "blocked_words:\\n - keyword: \\"test\\"\\n action: \\"BLOCK\\""
}
```
Example Success Response:
```json
{
@ -749,7 +754,7 @@ async def validate_blocked_words_file(request: Dict[str, str]):
"message": "Valid YAML file with 2 blocked words"
}
```
Example Error Response:
```json
{
@ -759,56 +764,54 @@ async def validate_blocked_words_file(request: Dict[str, str]):
```
"""
import yaml
try:
file_content = request.get("file_content", "")
if not file_content:
return {
"valid": False,
"error": "No file content provided"
}
return {"valid": False, "error": "No file content provided"}
data = yaml.safe_load(file_content)
if not isinstance(data, dict) or "blocked_words" not in data:
return {
"valid": False,
"error": "Invalid format: file must contain 'blocked_words' key with a list"
"error": "Invalid format: file must contain 'blocked_words' key with a list",
}
blocked_words_list = data["blocked_words"]
if not isinstance(blocked_words_list, list):
return {
"valid": False,
"error": "'blocked_words' must be a list"
}
return {"valid": False, "error": "'blocked_words' must be a list"}
# Validate each entry
errors = []
for idx, word_data in enumerate(blocked_words_list):
if not isinstance(word_data, dict):
errors.append(f"Entry {idx}: must be an object")
continue
if "keyword" not in word_data:
errors.append(f"Entry {idx}: missing 'keyword' field")
elif not isinstance(word_data["keyword"], str):
errors.append(f"Entry {idx}: 'keyword' must be a string")
if "action" not in word_data:
errors.append(f"Entry {idx}: missing 'action' field")
elif word_data["action"] not in ["BLOCK", "MASK"]:
errors.append(f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'")
if "description" in word_data and not isinstance(word_data["description"], str):
errors.append(
f"Entry {idx}: action must be 'BLOCK' or 'MASK', got '{word_data['action']}'"
)
if "description" in word_data and not isinstance(
word_data["description"], str
):
errors.append(f"Entry {idx}: 'description' must be a string")
if errors:
return {"valid": False, "errors": errors}
return {
"valid": True,
"message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)"
"message": f"Valid YAML file with {len(blocked_words_list)} blocked word(s)",
}
except yaml.YAMLError as e:
return {"valid": False, "error": f"Invalid YAML syntax: {str(e)}"}
@ -931,30 +934,32 @@ def _should_skip_optional_params(field_name: str, field_annotation: Any) -> bool
"""Check if optional_params field should be skipped (not meaningfully overridden)."""
if field_name != "optional_params":
return False
if field_annotation is None:
return True
# Check if the annotation is still a generic TypeVar (not specialized)
if isinstance(field_annotation, TypeVar) or (
hasattr(field_annotation, "__origin__")
and field_annotation.__origin__ is TypeVar
):
return True
# Also skip if it's a generic type that wasn't specialized
if hasattr(field_annotation, "__name__") and field_annotation.__name__ in (
"T",
"TypeVar",
):
return True
# Handle Optional[T] where T is still a TypeVar
if hasattr(field_annotation, "__args__"):
non_none_args = [arg for arg in field_annotation.__args__ if arg is not type(None)]
non_none_args = [
arg for arg in field_annotation.__args__ if arg is not type(None)
]
if non_none_args and isinstance(non_none_args[0], TypeVar):
return True
return False
@ -1041,9 +1046,11 @@ def _extract_fields_recursive(
for field_name, field in model.model_fields.items():
field_annotation = field.annotation
# Skip optional_params if it's not meaningfully overridden
if _should_skip_optional_params(field_name=field_name, field_annotation=field_annotation):
if _should_skip_optional_params(
field_name=field_name, field_annotation=field_annotation
):
continue
# Handle Optional types and get the actual type
@ -1153,12 +1160,18 @@ async def get_provider_specific_params():
bedrock_fields = _get_fields_from_model(BedrockGuardrailConfigModel)
presidio_fields = _get_fields_from_model(PresidioPresidioConfigModelUserInterface)
lakera_v2_fields = _get_fields_from_model(LakeraV2GuardrailConfigModel)
tool_permission_fields = _get_fields_from_model(ToolPermissionGuardrailConfigModel)
tool_permission_fields[
"ui_friendly_name"
] = ToolPermissionGuardrailConfigModel.ui_friendly_name()
# Return the provider-specific parameters
provider_params = {
SupportedGuardrailIntegrations.BEDROCK.value: bedrock_fields,
SupportedGuardrailIntegrations.PRESIDIO.value: presidio_fields,
SupportedGuardrailIntegrations.LAKERA_V2.value: lakera_v2_fields,
SupportedGuardrailIntegrations.TOOL_PERMISSION.value: tool_permission_fields,
}
### get the config model for the guardrail - go through the registry and get the config model for the guardrail
@ -1175,6 +1188,7 @@ async def get_provider_specific_params():
return provider_params
@router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse)
@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse)
async def apply_guardrail(
@ -1183,11 +1197,11 @@ async def apply_guardrail(
):
"""
Apply a guardrail to text input and return the processed result.
This endpoint allows testing guardrails by applying them to custom text inputs.
"""
from litellm.proxy.utils import handle_exception_on_proxy
try:
active_guardrail: Optional[
CustomGuardrail
@ -1207,4 +1221,3 @@ async def apply_guardrail(
return ApplyGuardrailResponse(response_text=response_text)
except Exception as e:
raise handle_exception_on_proxy(e)

51
litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py
View file

@ -90,6 +90,10 @@ class PillarGuardrail(CustomGuardrail):
fallback_on_error: Action when API errors occur ('allow' or 'block')
timeout: Timeout for API calls in seconds
**kwargs: Additional arguments passed to parent class
Note:
LiteLLM virtual key context (user_id, team_id, key_alias, etc.) is always
automatically passed as X-LiteLLM-* headers to enable application/user tracking.
"""
self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
self.api_key = api_key or os.environ.get("PILLAR_API_KEY")
@ -222,7 +226,7 @@ class PillarGuardrail(CustomGuardrail):
return data
verbose_proxy_logger.debug("Pillar Guardrail: Pre-call hook")
result = await self.run_pillar_guardrail(data)
result = await self.run_pillar_guardrail(data, user_api_key_dict)
# Add guardrail name to response headers
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name)
@ -265,7 +269,7 @@ class PillarGuardrail(CustomGuardrail):
return data
verbose_proxy_logger.debug("Pillar Guardrail: During-call moderation hook")
result = await self.run_pillar_guardrail(data)
result = await self.run_pillar_guardrail(data, user_api_key_dict)
# Add guardrail name to response headers
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name)
@ -315,7 +319,7 @@ class PillarGuardrail(CustomGuardrail):
post_call_data["messages"] = data.get("messages", []) + response_messages
# Reuse the existing guardrail logic - zero duplication!
await self.run_pillar_guardrail(post_call_data)
await self.run_pillar_guardrail(post_call_data, user_api_key_dict)
# Add guardrail name to response headers
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name)
@ -326,12 +330,13 @@ class PillarGuardrail(CustomGuardrail):
# CORE LOGIC METHOD
# =========================================================================
async def run_pillar_guardrail(self, data: dict) -> dict:
async def run_pillar_guardrail(self, data: dict, user_api_key_dict: UserAPIKeyAuth) -> dict:
"""
Core method to run the Pillar guardrail scan.
Args:
data: Request data containing messages and metadata
user_api_key_dict: User API key authentication info containing key context
Returns:
Original data if safe or in monitor mode
@ -345,7 +350,7 @@ class PillarGuardrail(CustomGuardrail):
return data
try:
headers = self._prepare_headers()
headers = self._prepare_headers(user_api_key_dict)
payload = self._prepare_payload(data)
response = await self._call_pillar_api(
@ -403,8 +408,16 @@ class PillarGuardrail(CustomGuardrail):
},
)
def _prepare_headers(self) -> Dict[str, str]:
"""Prepare headers for the Pillar API request."""
def _prepare_headers(self, user_api_key_dict: UserAPIKeyAuth) -> Dict[str, str]:
"""
Prepare headers for the Pillar API request.
Args:
user_api_key_dict: User API key authentication info containing key context
Returns:
Dictionary of headers to send to Pillar API
"""
if not self.api_key:
msg = (
"Couldn't get Pillar API key, either set the `PILLAR_API_KEY` in the environment or "
@ -415,7 +428,7 @@ class PillarGuardrail(CustomGuardrail):
headers: Dict[str, str] = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
}
# Add Pillar-specific headers based on configuration
self._set_bool_header(headers, "plr_scanners", self.include_scanners)
@ -423,6 +436,20 @@ class PillarGuardrail(CustomGuardrail):
self._set_bool_header(headers, "plr_async", self.async_mode)
self._set_bool_header(headers, "plr_persist", self.persist_session)
# Always add LiteLLM virtual key context headers (metadata excluded for security)
context_mapping = {
"X-LiteLLM-Key-Name": user_api_key_dict.key_name,
"X-LiteLLM-Key-Alias": user_api_key_dict.key_alias,
"X-LiteLLM-User-Id": user_api_key_dict.user_id,
"X-LiteLLM-User-Email": user_api_key_dict.user_email,
"X-LiteLLM-Team-Id": user_api_key_dict.team_id,
"X-LiteLLM-Team-Name": user_api_key_dict.team_alias,
"X-LiteLLM-Org-Id": user_api_key_dict.org_id,
}
for header_name, value in context_mapping.items():
if value:
headers[header_name] = str(value)
return headers
def _set_bool_header(self, headers: Dict[str, str], header_name: str, value: Optional[bool]) -> None:
@ -517,6 +544,14 @@ class PillarGuardrail(CustomGuardrail):
"""
Prepare the payload for the Pillar API request following the /api/v1/protect contract.
This method supports multi-modal content (images, files, audio, video, etc.) as messages
are passed through without modification. The messages array can contain any OpenAI-compatible
message structure including:
- Text content (string)
- Multi-modal content blocks (image_url, image_file, audio, video, document, file)
- Attachments
- Tool calls
Args:
data: Request data

19
litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py
View file

@ -1,13 +1,18 @@
import os
import re
import asyncio
import base64
import os
import re
from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Type, Union
from fastapi import HTTPException
from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, httpxSpecialProvider
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import (
Choices,
@ -15,7 +20,7 @@ from litellm.types.utils import (
EmbeddingResponse,
ImageResponse,
ModelResponse,
ModelResponseStream
ModelResponseStream,
)
if TYPE_CHECKING:
@ -267,8 +272,10 @@ class PromptSecurityGuardrail(CustomGuardrail):
content = msg.get('content', '')
# Handle both string and list content types
if isinstance(content, str):
if content.startswith('### '): return False
if '"follow_ups": [' in content: return False
if content.startswith('### '):
return False
if '"follow_ups": [' in content:
return False
return True
messages = list(filter(lambda msg: good_msg(msg), messages))

40
litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
View file

@ -23,6 +23,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
ToolResult,
)
from litellm.types.utils import (
CallTypesLiteral,
ChatCompletionMessageToolCall,
Choices,
LLMResponseTypes,
@ -62,8 +63,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
self.rules: List[ToolPermissionRule] = []
self._compiled_rule_patterns: Dict[str, Dict[str, re.Pattern]] = {}
if rules:
for rule_dict in rules:
rule = ToolPermissionRule(**rule_dict)
for rule_item in rules:
if isinstance(rule_item, ToolPermissionRule):
rule = rule_item
else:
rule = ToolPermissionRule(**rule_item)
self.rules.append(rule)
if rule.allowed_param_patterns:
@ -88,6 +92,14 @@ class ToolPermissionGuardrail(CustomGuardrail):
self.default_action,
)
@staticmethod
def get_config_model():
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
ToolPermissionGuardrailConfigModel,
)
return ToolPermissionGuardrailConfigModel
def _matches_pattern(self, tool_name: str, pattern: str) -> bool:
"""
Check if a tool name matches a pattern
@ -191,16 +203,21 @@ class ToolPermissionGuardrail(CustomGuardrail):
return {}
def _collect_argument_paths(
self, value: Any, current_path: str, collected: Dict[str, List[Any]]
self, value: Any, current_path: str, collected: Dict[str, List[Any]], depth: int = 0
) -> None:
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
if depth > DEFAULT_MAX_RECURSE_DEPTH:
return
if isinstance(value, dict):
for key, sub_value in value.items():
next_path = f"{current_path}.{key}" if current_path else key
self._collect_argument_paths(sub_value, next_path, collected)
self._collect_argument_paths(sub_value, next_path, collected, depth + 1)
elif isinstance(value, list):
list_path = f"{current_path}[]" if current_path else "[]"
for item in value:
self._collect_argument_paths(item, list_path, collected)
self._collect_argument_paths(item, list_path, collected, depth + 1)
else:
if not current_path:
return
@ -426,18 +443,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: Literal[
"completion",
"text_completion",
"embeddings",
"image_generation",
"moderation",
"audio_transcription",
"pass_through_endpoint",
"rerank",
"mcp_call",
"anthropic_messages",
],
call_type: CallTypesLiteral,
) -> Union[Exception, str, dict, None]:
""" """
verbose_proxy_logger.debug("Tool Permission Guardrail Pre-Call Hook")

13
litellm/proxy/guardrails/guardrail_initializers.py
View file

@ -1,4 +1,6 @@
# litellm/proxy/guardrails/guardrail_initializers.py
from typing import Any, Dict, List, Optional
import litellm
from litellm.proxy._types import CommonProxyErrors
from litellm.types.guardrails import *
@ -128,10 +130,19 @@ def initialize_tool_permission(litellm_params: LitellmParams, guardrail: Guardra
ToolPermissionGuardrail,
)
rules: Optional[List[Dict[str, Any]]] = None
if litellm_params.rules:
rules = []
for rule in litellm_params.rules:
if hasattr(rule, "model_dump"):
rules.append(rule.model_dump())
else:
rules.append(dict(rule))
_tool_permission_callback = ToolPermissionGuardrail(
guardrail_name=guardrail.get("guardrail_name", ""),
event_hook=litellm_params.mode,
rules=litellm_params.rules,
rules=rules,
default_action=getattr(litellm_params, "default_action", "deny"),
on_disallowed_action=getattr(litellm_params, "on_disallowed_action", "block"),
default_on=litellm_params.default_on,

18
litellm/proxy/image_endpoints/endpoints.py
View file

@ -215,9 +215,11 @@ async def image_generation(
async def image_edit_api(
request: Request,
fastapi_response: Response,
image: List[UploadFile] = File(...),
mask: Optional[List[UploadFile]] = File(None),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
image: Optional[List[UploadFile]] = File(None),
image_array: Optional[List[UploadFile]] = File(None, alias="image[]"),
mask: Optional[List[UploadFile]] = File(None),
mask_array: Optional[List[UploadFile]] = File(None, alias="mask[]"),
model: Optional[str] = None,
):
"""
@ -233,6 +235,18 @@ async def image_edit_api(
-F 'prompt=Create a studio ghibli image of this'
```
"""
if image is not None and image_array is not None:
raise HTTPException(status_code=422, detail="Cannot specify both 'image' and 'image[]'")
if mask is not None and mask_array is not None:
raise HTTPException(status_code=422, detail="Cannot specify both 'mask' and 'mask[]'")
if image is None and image_array is not None:
image = image_array
if mask is None and mask_array is not None:
mask = mask_array
if image is None:
raise HTTPException(status_code=422, detail="Field required: image")
from litellm.proxy.proxy_server import (
_read_request_body,
general_settings,

9
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -1878,6 +1878,15 @@ async def team_member_delete(
where={"team_id": data.team_id, "user_id": _uid}
)
## DELETE KEYS CREATED BY USER FOR THIS TEAM
if user_ids_to_delete:
await prisma_client.db.litellm_verificationtoken.delete_many(
where={
"user_id": {"in": list(user_ids_to_delete)},
"team_id": data.team_id,
}
)
return existing_team_row

138
litellm/proxy/pass_through_endpoints/llm_provider_handlers/cohere_passthrough_logging_handler.py
View file

@ -1,14 +1,30 @@
from datetime import datetime
from typing import List, Optional, Union
import httpx
import litellm
from litellm import stream_chunk_builder
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload,
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.cohere.chat.v2_transformation import CohereV2ChatConfig
from litellm.llms.cohere.common_utils import (
ModelResponseIterator as CohereModelResponseIterator,
)
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
from litellm.llms.cohere.embed.v1_transformation import CohereEmbeddingConfig
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
PassthroughStandardLoggingPayload,
)
from litellm.types.utils import (
LlmProviders,
ModelResponse,
TextCompletionResponse,
)
from .base_passthrough_logging_handler import BasePassthroughLoggingHandler
@ -54,3 +70,123 @@ class CoherePassthroughLoggingHandler(BasePassthroughLoggingHandler):
break
complete_streaming_response = stream_chunk_builder(chunks=all_openai_chunks)
return complete_streaming_response
def cohere_passthrough_handler( # noqa: PLR0915
self,
httpx_response: httpx.Response,
response_body: dict,
logging_obj: LiteLLMLoggingObj,
url_route: str,
result: str,
start_time: datetime,
end_time: datetime,
cache_hit: bool,
request_body: dict,
**kwargs,
) -> PassThroughEndpointLoggingTypedDict:
"""
Handle Cohere passthrough logging with route detection and cost tracking.
"""
# Check if this is an embed endpoint
if "/v1/embed" in url_route:
model = request_body.get("model", response_body.get("model", ""))
try:
cohere_embed_config = CohereEmbeddingConfig()
litellm_model_response = litellm.EmbeddingResponse()
handler_instance = CoherePassthroughLoggingHandler()
input_texts = request_body.get("texts", [])
if not input_texts:
input_texts = request_body.get("input", [])
# Transform the response
litellm_model_response = cohere_embed_config._transform_response(
response=httpx_response,
api_key="",
logging_obj=logging_obj,
data=request_body,
model_response=litellm_model_response,
model=model,
encoding=litellm.encoding,
input=input_texts,
)
# Calculate cost using LiteLLM's cost calculator
response_cost = litellm.completion_cost(
completion_response=litellm_model_response,
model=model,
custom_llm_provider="cohere",
call_type="aembedding",
)
# Set the calculated cost in _hidden_params to prevent recalculation
if not hasattr(litellm_model_response, "_hidden_params"):
litellm_model_response._hidden_params = {}
litellm_model_response._hidden_params["response_cost"] = response_cost
kwargs["response_cost"] = response_cost
kwargs["model"] = model
kwargs["custom_llm_provider"] = "cohere"
# Extract user information for tracking
passthrough_logging_payload: Optional[
PassthroughStandardLoggingPayload
] = kwargs.get("passthrough_logging_payload")
if passthrough_logging_payload:
user = handler_instance._get_user_from_metadata(
passthrough_logging_payload=passthrough_logging_payload,
)
if user:
kwargs.setdefault("litellm_params", {})
kwargs["litellm_params"].update(
{"proxy_server_request": {"body": {"user": user}}}
)
# Create standard logging object
if litellm_model_response is not None:
get_standard_logging_object_payload(
kwargs=kwargs,
init_response_obj=litellm_model_response,
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
status="success",
)
# Update logging object with cost information
logging_obj.model_call_details["model"] = model
logging_obj.model_call_details["custom_llm_provider"] = "cohere"
logging_obj.model_call_details["response_cost"] = response_cost
return {
"result": litellm_model_response,
"kwargs": kwargs,
}
except Exception:
# For other routes (e.g., /v2/chat), fall back to chat handler
return super().passthrough_chat_handler(
httpx_response=httpx_response,
response_body=response_body,
logging_obj=logging_obj,
url_route=url_route,
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
request_body=request_body,
**kwargs,
)
# For non-embed routes (e.g., /v2/chat), fall back to chat handler
return super().passthrough_chat_handler(
httpx_response=httpx_response,
response_body=response_body,
logging_obj=logging_obj,
url_route=url_route,
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
request_body=request_body,
**kwargs,
)

3
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -148,6 +148,8 @@ class VertexPassthroughLoggingHandler:
logging_obj.model = model
logging_obj.model_call_details["model"] = logging_obj.model
logging_obj.model_call_details["custom_llm_provider"] = "vertex_ai"
logging_obj.custom_llm_provider = "vertex_ai"
response_cost = litellm.completion_cost(
completion_response=litellm_prediction_response,
model=model,
@ -156,6 +158,7 @@ class VertexPassthroughLoggingHandler:
kwargs["response_cost"] = response_cost
kwargs["model"] = model
kwargs["custom_llm_provider"] = "vertex_ai"
logging_obj.model_call_details["response_cost"] = response_cost
return {

4
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -48,7 +48,7 @@ class PassThroughEndpointLogging:
self.TRACKED_ANTHROPIC_ROUTES = ["/messages"]
# Cohere
self.TRACKED_COHERE_ROUTES = ["/v2/chat"]
self.TRACKED_COHERE_ROUTES = ["/v2/chat", "/v1/embed"]
self.assemblyai_passthrough_logging_handler = (
AssemblyAIPassthroughLoggingHandler()
)
@ -177,7 +177,7 @@ class PassThroughEndpointLogging:
kwargs = anthropic_passthrough_logging_handler_result["kwargs"]
elif self.is_cohere_route(url_route):
cohere_passthrough_logging_handler_result = (
cohere_passthrough_logging_handler.passthrough_chat_handler(
cohere_passthrough_logging_handler.cohere_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body or {},
logging_obj=logging_obj,

19
litellm/proxy/proxy_config.yaml
View file

@ -1,22 +1,7 @@
model_list:
- model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1
- model_name: qwen-25vl-72b
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
- model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
- model_name: bedrock/*
litellm_params:
model: bedrock/*
custom_llm_provider: bedrock
aws_region_name: us-west-2
- model_name: runwayml/*
litellm_params:
model: runwayml/*
model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z

43
litellm/proxy/proxy_server.py
View file

@ -362,6 +362,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
)
from litellm.proxy.prompts.prompt_endpoints import router as prompts_router
from litellm.proxy.public_endpoints import router as public_endpoints_router
from litellm.proxy.rag_endpoints.endpoints import router as rag_router
from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
from litellm.proxy.response_api_endpoints.endpoints import router as response_router
from litellm.proxy.route_llm_request import route_request
@ -1240,7 +1241,7 @@ def cost_tracking():
global prisma_client
if prisma_client is not None:
litellm.logging_callback_manager.add_litellm_callback(_ProxyDBLogger())
litellm.logging_callback_manager.add_litellm_async_success_callback(_ProxyDBLogger())
async def update_cache( # noqa: PLR0915
token: Optional[str],
@ -2010,6 +2011,16 @@ class ProxyConfig:
)
print(f"\033[32m {search_tool_name} ({search_provider})\033[0m") # noqa
# Handle os.environ/ variables in litellm_params
litellm_params = search_tool.get("litellm_params", {})
if litellm_params:
for k, v in litellm_params.items():
if isinstance(v, str) and v.startswith("os.environ/"):
_v = v.replace("os.environ/", "")
v = get_secret(_v)
litellm_params[k] = v
search_tool["litellm_params"] = litellm_params
# Cast to SearchToolTypedDict for type safety
try:
search_tool_typed: SearchToolTypedDict = SearchToolTypedDict(**search_tool) # type: ignore
@ -3761,6 +3772,7 @@ class ProxyConfig:
async def _init_search_tools_in_db(self, prisma_client: PrismaClient):
"""
Initialize search tools from database into the router on startup.
Only updates router if there are tools in the database, otherwise preserves config-loaded tools.
"""
global llm_router
@ -3778,17 +3790,24 @@ class ProxyConfig:
f"Loading {len(search_tools)} search tool(s) from database into router"
)
if llm_router is not None:
# Add search tools to the router
await SearchAPIRouter.update_router_search_tools(
router_instance=llm_router, search_tools=search_tools
)
verbose_proxy_logger.info(
f"Successfully loaded {len(search_tools)} search tool(s) into router"
)
# Only update router if there are tools in the database
# This prevents overwriting config-loaded tools with an empty list
if len(search_tools) > 0:
if llm_router is not None:
# Add search tools to the router
await SearchAPIRouter.update_router_search_tools(
router_instance=llm_router, search_tools=search_tools
)
verbose_proxy_logger.info(
f"Successfully loaded {len(search_tools)} search tool(s) into router"
)
else:
verbose_proxy_logger.debug(
"Router not initialized yet, search tools will be added when router is created"
)
else:
verbose_proxy_logger.debug(
"Router not initialized yet, search tools will be added when router is created"
"No search tools found in database, keeping config-loaded search tools (if any)"
)
except Exception as e:
@ -5465,6 +5484,7 @@ async def audio_transcriptions(
file_object = io.BytesIO(file_content)
file_object.name = file.filename
data["file"] = file_object
try:
### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook(
@ -5482,7 +5502,7 @@ async def audio_transcriptions(
)
response = await llm_call
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
raise e
finally:
file_object.close() # close the file read in by io library
@ -10138,6 +10158,7 @@ app.include_router(batches_router)
app.include_router(public_endpoints_router)
app.include_router(rerank_router)
app.include_router(ocr_router)
app.include_router(rag_router)
app.include_router(video_router)
app.include_router(container_router)
app.include_router(search_router)

Some files were not shown because too many files have changed in this diff Show more