mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
Add day 0 support for anthropic new feat (#17091)
* Added tool search support for anthropic * Add programtic tool calling support * Add tool use input examples support * Add anthropic effort param support * Add anthropic effort param support * Add blog for new features * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * Add better handling * Add better handling
This commit is contained in:
Sameer Kankute
GitHub
parent
3249f6dd2d
commit
67622fb040
13 changed files with 3420 additions and 57 deletions
655
docs/my-website/blog/anthropic_advanced_features/index.md
Normal file
655
docs/my-website/blog/anthropic_advanced_features/index.md
Normal file
|
|
@ -0,0 +1,655 @@
|
|||
---
|
||||
slug: anthropic_advanced_features
|
||||
title: "Advanced Anthropic Features in LiteLLM: Tool Search, Programmatic Tool Calling, Input Examples, and Effort Control"
|
||||
date: 2025-01-25T10:00:00
|
||||
authors:
|
||||
- name: Sameer Kankute
|
||||
title: SWE @ LiteLLM (LLM Translation)
|
||||
url: https://www.linkedin.com/in/sameer-kankute/
|
||||
image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
|
||||
- name: Krrish Dholakia
|
||||
title: "CEO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
:::info
|
||||
|
||||
This guide covers Anthropic's latest advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter.
|
||||
|
||||
:::
|
||||
|
||||
We're excited to announce support for Anthropic's latest advanced features in LiteLLM! These powerful capabilities enable you to build more efficient, scalable, and cost-effective AI applications with Claude.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Tool Search](#tool-search)
|
||||
2. [Programmatic Tool Calling](#programmatic-tool-calling)
|
||||
3. [Tool Input Examples](#tool-input-examples)
|
||||
4. [Effort Parameter: Control Token Usage](#effort-parameter)
|
||||
5. [Cost Tracking: Monitor Tool Search Usage](#cost-tracking)
|
||||
6. [Combining Features](#combining-features)
|
||||
|
||||
---
|
||||
|
||||
## Tool Search {#tool-search}
|
||||
|
||||
### Usage Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Configure your API key
|
||||
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
|
||||
|
||||
# Define your tools with defer_loading
|
||||
tools = [
|
||||
# Tool search tool (regex variant)
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
# Deferred tools - loaded on-demand
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location. Returns temperature and conditions.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "Temperature unit"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True # Load on-demand
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_files",
|
||||
"description": "Search through files in the workspace using keywords",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"file_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute SQL queries against the database",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {"type": "string"}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
|
||||
# Make a request - Claude will search for and use relevant tools
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "What's the weather like in San Francisco?"
|
||||
}],
|
||||
tools=tools
|
||||
)
|
||||
|
||||
print("Claude's response:", response.choices[0].message.content)
|
||||
print("Tool calls:", response.choices[0].message.tool_calls)
|
||||
|
||||
# Check tool search usage
|
||||
if hasattr(response.usage, 'server_tool_use'):
|
||||
print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}")
|
||||
```
|
||||
|
||||
### BM25 Variant (Natural Language Search)
|
||||
|
||||
For natural language queries instead of regex patterns:
|
||||
|
||||
```python
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119", # Natural language variant
|
||||
"name": "tool_search_tool_bm25"
|
||||
},
|
||||
# ... your deferred tools
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Programmatic Tool Calling {#programmatic-tool-calling}
|
||||
|
||||
### Usage Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import json
|
||||
|
||||
# Define tools that can be called programmatically
|
||||
tools = [
|
||||
# Code execution tool (required for programmatic calling)
|
||||
{
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
},
|
||||
# Tool that can be called from code
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {
|
||||
"type": "string",
|
||||
"description": "SQL query to execute"
|
||||
}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"] # Enable programmatic calling
|
||||
}
|
||||
]
|
||||
|
||||
# First request
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue"
|
||||
}],
|
||||
tools=tools
|
||||
)
|
||||
|
||||
print("Claude's response:", response.choices[0].message)
|
||||
|
||||
# Handle tool calls
|
||||
messages = [
|
||||
{"role": "user", "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue"},
|
||||
{"role": "assistant", "content": response.choices[0].message.content, "tool_calls": response.choices[0].message.tool_calls}
|
||||
]
|
||||
|
||||
# Process each tool call
|
||||
for tool_call in response.choices[0].message.tool_calls:
|
||||
# Check if it's a programmatic call
|
||||
if hasattr(tool_call, 'caller') and tool_call.caller:
|
||||
print(f"Programmatic call to {tool_call.function.name}")
|
||||
print(f"Called from: {tool_call.caller}")
|
||||
|
||||
# Simulate tool execution
|
||||
if tool_call.function.name == "query_database":
|
||||
args = json.loads(tool_call.function.arguments)
|
||||
# Simulate database query
|
||||
result = json.dumps([
|
||||
{"region": "West", "revenue": 150000},
|
||||
{"region": "East", "revenue": 180000},
|
||||
{"region": "Central", "revenue": 120000}
|
||||
])
|
||||
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tool_call.id,
|
||||
"content": result
|
||||
}]
|
||||
})
|
||||
|
||||
# Get final response
|
||||
final_response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=messages,
|
||||
tools=tools
|
||||
)
|
||||
|
||||
print("\nFinal answer:", final_response.choices[0].message.content)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tool Input Examples {#tool-input-examples}
|
||||
|
||||
### Usage Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_calendar_event",
|
||||
"description": "Create a new calendar event with attendees and reminders",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"start_time": {
|
||||
"type": "string",
|
||||
"description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS"
|
||||
},
|
||||
"duration_minutes": {"type": "integer"},
|
||||
"attendees": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"email": {"type": "string"},
|
||||
"optional": {"type": "boolean"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"reminders": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"minutes_before": {"type": "integer"},
|
||||
"method": {"type": "string", "enum": ["email", "popup"]}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["title", "start_time", "duration_minutes"]
|
||||
}
|
||||
},
|
||||
# Provide concrete examples
|
||||
"input_examples": [
|
||||
{
|
||||
"title": "Team Standup",
|
||||
"start_time": "2025-01-15T09:00:00",
|
||||
"duration_minutes": 30,
|
||||
"attendees": [
|
||||
{"email": "alice@company.com", "optional": False},
|
||||
{"email": "bob@company.com", "optional": False}
|
||||
],
|
||||
"reminders": [
|
||||
{"minutes_before": 15, "method": "popup"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Lunch Break",
|
||||
"start_time": "2025-01-15T12:00:00",
|
||||
"duration_minutes": 60
|
||||
# Demonstrates optional fields can be omitted
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com"
|
||||
}],
|
||||
tools=tools
|
||||
)
|
||||
|
||||
print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Effort Parameter: Control Token Usage {#effort-parameter}
|
||||
|
||||
### Usage Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
message = "Analyze the trade-offs between microservices and monolithic architectures"
|
||||
|
||||
# High effort (default) - Maximum capability
|
||||
response_high = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{"role": "user", "content": message}],
|
||||
output_config={"effort": "high"}
|
||||
)
|
||||
|
||||
print("High effort response:")
|
||||
print(response_high.choices[0].message.content)
|
||||
print(f"Tokens used: {response_high.usage.completion_tokens}\n")
|
||||
|
||||
# Medium effort - Balanced approach
|
||||
response_medium = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{"role": "user", "content": message}],
|
||||
output_config={"effort": "medium"}
|
||||
)
|
||||
|
||||
print("Medium effort response:")
|
||||
print(response_medium.choices[0].message.content)
|
||||
print(f"Tokens used: {response_medium.usage.completion_tokens}\n")
|
||||
|
||||
# Low effort - Maximum efficiency
|
||||
response_low = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{"role": "user", "content": message}],
|
||||
output_config={"effort": "low"}
|
||||
)
|
||||
|
||||
print("Low effort response:")
|
||||
print(response_low.choices[0].message.content)
|
||||
print(f"Tokens used: {response_low.usage.completion_tokens}\n")
|
||||
|
||||
# Compare token usage
|
||||
print("Token Comparison:")
|
||||
print(f"High: {response_high.usage.completion_tokens} tokens")
|
||||
print(f"Medium: {response_medium.usage.completion_tokens} tokens")
|
||||
print(f"Low: {response_low.usage.completion_tokens} tokens")
|
||||
```
|
||||
|
||||
### Effort with Tool Use
|
||||
|
||||
Lower effort affects both explanations and tool calls:
|
||||
|
||||
```python
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Low effort = fewer tool calls, more direct
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Check weather in San Francisco, New York, and London"
|
||||
}],
|
||||
tools=tools,
|
||||
output_config={"effort": "low"} # May combine into fewer calls
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cost Tracking: Monitor Tool Search Usage {#cost-tracking}
|
||||
|
||||
### Understanding Tool Search Costs
|
||||
|
||||
Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs.
|
||||
|
||||
### Tracking Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
# ... 100 deferred tools
|
||||
]
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Find and use the weather tool for San Francisco"
|
||||
}],
|
||||
tools=tools
|
||||
)
|
||||
|
||||
# Standard token usage
|
||||
print("Token Usage:")
|
||||
print(f" Input tokens: {response.usage.prompt_tokens}")
|
||||
print(f" Output tokens: {response.usage.completion_tokens}")
|
||||
print(f" Total tokens: {response.usage.total_tokens}")
|
||||
|
||||
# Tool search specific usage
|
||||
if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use:
|
||||
print(f"\nTool Search Usage:")
|
||||
print(f" Search requests: {response.usage.server_tool_use.tool_search_requests}")
|
||||
|
||||
# Calculate cost (example pricing)
|
||||
input_cost = response.usage.prompt_tokens * 0.000003 # $3 per 1M tokens
|
||||
output_cost = response.usage.completion_tokens * 0.000015 # $15 per 1M tokens
|
||||
search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001 # Example
|
||||
|
||||
total_cost = input_cost + output_cost + search_cost
|
||||
|
||||
print(f"\nCost Breakdown:")
|
||||
print(f" Input tokens: ${input_cost:.6f}")
|
||||
print(f" Output tokens: ${output_cost:.6f}")
|
||||
print(f" Tool searches: ${search_cost:.6f}")
|
||||
print(f" Total: ${total_cost:.6f}")
|
||||
```
|
||||
|
||||
### Cost Optimization Tips
|
||||
|
||||
1. **Keep frequently used tools non-deferred** (3-5 tools)
|
||||
2. **Use tool search for large catalogs** (10+ tools)
|
||||
3. **Monitor search requests** to identify optimization opportunities
|
||||
4. **Combine with effort parameter** for maximum efficiency
|
||||
|
||||
```python
|
||||
# Optimized for cost
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Simple query"}],
|
||||
tools=tools_with_search,
|
||||
output_config={"effort": "low"} # Reduce output tokens
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Combining Features {#combining-features}
|
||||
|
||||
### The Power of Integration
|
||||
|
||||
These features work together seamlessly. Here's a real-world example combining all of them:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import json
|
||||
|
||||
# Large tool catalog with search, programmatic calling, and examples
|
||||
tools = [
|
||||
# Enable tool search
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
# Enable programmatic calling
|
||||
{
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
},
|
||||
# Database tool with all features
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute SQL queries against the analytics database. Returns JSON array of results.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {
|
||||
"type": "string",
|
||||
"description": "SQL SELECT statement"
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum rows to return"
|
||||
}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True, # Tool search
|
||||
"allowed_callers": ["code_execution_20250825"], # Programmatic calling
|
||||
"input_examples": [ # Input examples
|
||||
{
|
||||
"sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region",
|
||||
"limit": 100
|
||||
}
|
||||
]
|
||||
},
|
||||
# ... 50 more tools with defer_loading
|
||||
]
|
||||
|
||||
# Make request with effort control
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Analyze sales by region for the last quarter and identify top performers"
|
||||
}],
|
||||
tools=tools,
|
||||
output_config={"effort": "medium"} # Balanced efficiency
|
||||
)
|
||||
|
||||
# Track comprehensive usage
|
||||
print("Complete Usage Metrics:")
|
||||
print(f" Input tokens: {response.usage.prompt_tokens}")
|
||||
print(f" Output tokens: {response.usage.completion_tokens}")
|
||||
print(f" Total tokens: {response.usage.total_tokens}")
|
||||
|
||||
if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use:
|
||||
print(f" Tool searches: {response.usage.server_tool_use.tool_search_requests}")
|
||||
|
||||
print(f"\nResponse: {response.choices[0].message.content}")
|
||||
```
|
||||
|
||||
### Real-World Benefits
|
||||
|
||||
This combination enables:
|
||||
|
||||
1. **Massive scale** - Handle 1000+ tools efficiently
|
||||
2. **Low latency** - Programmatic calling reduces round trips
|
||||
3. **High accuracy** - Input examples ensure correct tool usage
|
||||
4. **Cost control** - Effort parameter optimizes token spend
|
||||
5. **Full visibility** - Track all usage metrics
|
||||
|
||||
---
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
pip install litellm --upgrade
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```python
|
||||
import os
|
||||
import litellm
|
||||
|
||||
# Set your API key
|
||||
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
|
||||
|
||||
# LiteLLM automatically handles beta headers for all features
|
||||
```
|
||||
|
||||
### Supported Models
|
||||
|
||||
| Feature | Supported Models |
|
||||
|---------|-----------------|
|
||||
| Tool Search | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Input Examples | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Effort Parameter | Claude Opus 4.5 only |
|
||||
|
||||
### Supported Endpoints
|
||||
|
||||
**Note**: All features are supported on the `/chat/completions` endpoint only.
|
||||
|
||||
| Feature | Supported Models |
|
||||
|---------|-----------------|
|
||||
| Tool Search | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Input Examples | Claude Opus 4.5, Sonnet 4.5 |
|
||||
| Effort Parameter | Claude Opus 4.5 only |
|
||||
|
||||
### Provider Support
|
||||
|
||||
All features work across:
|
||||
- ✅ Standard Anthropic API
|
||||
- ✅ Azure Anthropic
|
||||
- ✅ Vertex AI Anthropic
|
||||
- ✅ LiteLLM Proxy
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
These advanced Anthropic features in LiteLLM enable you to build more sophisticated, efficient, and cost-effective AI applications:
|
||||
|
||||
- **Tool Search** scales to thousands of tools
|
||||
- **Programmatic Tool Calling** reduces latency and tokens
|
||||
- **Input Examples** improve accuracy
|
||||
- **Effort Parameter** controls costs
|
||||
|
||||
All features work seamlessly together and are supported across all Anthropic providers through LiteLLM's unified interface.
|
||||
|
||||
### Resources
|
||||
|
||||
- [LiteLLM Documentation](https://docs.litellm.ai/)
|
||||
- [Anthropic Tool Search Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_search)
|
||||
- [Anthropic Programmatic Tool Calling Docs](https://docs.litellm.ai/docs/providers/anthropic_programmatic_tool_calling)
|
||||
- [Anthropic Input Examples Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_input_examples)
|
||||
- [Anthropic Effort Parameter Docs](https://docs.litellm.ai/docs/providers/anthropic_effort)
|
||||
|
||||
### Get Started Today
|
||||
|
||||
```bash
|
||||
pip install litellm --upgrade
|
||||
```
|
||||
|
||||
Happy building! 🚀
|
||||
|
||||
276
docs/my-website/docs/providers/anthropic_effort.md
Normal file
276
docs/my-website/docs/providers/anthropic_effort.md
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
# Anthropic Effort Parameter
|
||||
|
||||
Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency.
|
||||
|
||||
## Overview
|
||||
|
||||
The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
|
||||
|
||||
**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected).
|
||||
|
||||
## How Effort Works
|
||||
|
||||
By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability.
|
||||
|
||||
**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely.
|
||||
|
||||
The effort parameter affects **all tokens** in the response, including:
|
||||
- Text responses and explanations
|
||||
- Tool calls and function arguments
|
||||
- Extended thinking (when enabled)
|
||||
|
||||
This approach has two major advantages:
|
||||
1. It doesn't require thinking to be enabled in order to use it.
|
||||
2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls.
|
||||
|
||||
This gives a much greater degree of control over efficiency.
|
||||
|
||||
## Effort Levels
|
||||
|
||||
| Level | Description | Typical use case |
|
||||
|-------|-------------|------------------|
|
||||
| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
|
||||
| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
|
||||
| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Using LiteLLM SDK
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python" label="Python">
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Analyze the trade-offs between microservices and monolithic architectures"
|
||||
}],
|
||||
output_config={
|
||||
"effort": "medium"
|
||||
}
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="typescript" label="TypeScript">
|
||||
|
||||
```typescript
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
|
||||
const client = new Anthropic({
|
||||
apiKey: process.env.ANTHROPIC_API_KEY,
|
||||
});
|
||||
|
||||
const response = await client.messages.create({
|
||||
model: "claude-opus-4-5-20251101",
|
||||
max_tokens: 4096,
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: "Analyze the trade-offs between microservices and monolithic architectures"
|
||||
}],
|
||||
output_config: {
|
||||
effort: "medium"
|
||||
}
|
||||
});
|
||||
|
||||
console.log(response.content[0].text);
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Using LiteLLM Proxy
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_API_KEY" \
|
||||
-d '{
|
||||
"model": "anthropic/claude-opus-4-5-20251101",
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": "Analyze the trade-offs between microservices and monolithic architectures"
|
||||
}],
|
||||
"output_config": {
|
||||
"effort": "medium"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
### Direct Anthropic API Call
|
||||
|
||||
```bash
|
||||
curl https://api.anthropic.com/v1/messages \
|
||||
--header "x-api-key: $ANTHROPIC_API_KEY" \
|
||||
--header "anthropic-version: 2023-06-01" \
|
||||
--header "anthropic-beta: effort-2025-11-24" \
|
||||
--header "content-type: application/json" \
|
||||
--data '{
|
||||
"model": "claude-opus-4-5-20251101",
|
||||
"max_tokens": 4096,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": "Analyze the trade-offs between microservices and monolithic architectures"
|
||||
}],
|
||||
"output_config": {
|
||||
"effort": "medium"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
## Model Compatibility
|
||||
|
||||
The effort parameter is currently only supported by:
|
||||
- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
|
||||
|
||||
## When Should I Adjust the Effort Parameter?
|
||||
|
||||
- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority.
|
||||
|
||||
- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort.
|
||||
|
||||
- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend.
|
||||
|
||||
## Effort with Tool Use
|
||||
|
||||
When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to:
|
||||
- Combine multiple operations into fewer tool calls
|
||||
- Make fewer tool calls
|
||||
- Proceed directly to action
|
||||
|
||||
Example with tools:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Check the weather in multiple cities"
|
||||
}],
|
||||
tools=[{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
}],
|
||||
output_config={
|
||||
"effort": "low" # Will make fewer tool calls
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
## Effort with Extended Thinking
|
||||
|
||||
The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Solve this complex problem"
|
||||
}],
|
||||
thinking={
|
||||
"type": "enabled",
|
||||
"budget_tokens": 5000
|
||||
},
|
||||
output_config={
|
||||
"effort": "medium" # Affects both thinking and response tokens
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs.
|
||||
|
||||
2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency.
|
||||
|
||||
3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses.
|
||||
|
||||
4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases.
|
||||
|
||||
5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity.
|
||||
|
||||
## Provider Support
|
||||
|
||||
The effort parameter is supported across all Anthropic-compatible providers:
|
||||
|
||||
- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5)
|
||||
- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5)
|
||||
- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5)
|
||||
|
||||
LiteLLM automatically handles the beta header injection for all providers.
|
||||
|
||||
## Usage and Pricing
|
||||
|
||||
Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs:
|
||||
|
||||
```python
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-opus-4-5-20251101",
|
||||
messages=[{"role": "user", "content": "Analyze this"}],
|
||||
output_config={"effort": "low"}
|
||||
)
|
||||
|
||||
print(f"Output tokens: {response.usage.completion_tokens}")
|
||||
print(f"Total tokens: {response.usage.total_tokens}")
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Beta header not being added
|
||||
|
||||
LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header:
|
||||
|
||||
1. Ensure you're using `output_config` with an `effort` field
|
||||
2. Verify the model is Claude Opus 4.5
|
||||
3. Check that LiteLLM version supports this feature
|
||||
|
||||
### Invalid effort value error
|
||||
|
||||
Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
|
||||
|
||||
```python
|
||||
# ❌ This will raise an error
|
||||
output_config={"effort": "very_low"}
|
||||
|
||||
# ✅ Use one of the valid values
|
||||
output_config={"effort": "low"}
|
||||
```
|
||||
|
||||
### Model not supported
|
||||
|
||||
Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
|
||||
|
||||
## Related Features
|
||||
|
||||
- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process
|
||||
- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions
|
||||
- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools
|
||||
- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort)
|
||||
- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic)
|
||||
- [Cost Optimization Best Practices](/docs/guides/cost_optimization)
|
||||
|
||||
|
|
@ -0,0 +1,430 @@
|
|||
# Anthropic Programmatic Tool Calling
|
||||
|
||||
Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window.
|
||||
|
||||
:::info
|
||||
Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field.
|
||||
|
||||
This feature requires the code execution tool to be enabled.
|
||||
:::
|
||||
|
||||
## Model Compatibility
|
||||
|
||||
Programmatic tool calling is available on the following models:
|
||||
|
||||
| Model | Tool Version |
|
||||
|-------|--------------|
|
||||
| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` |
|
||||
| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` |
|
||||
|
||||
## Quick Start
|
||||
|
||||
Here's a simple example where Claude programmatically queries a database multiple times and aggregates results:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue"
|
||||
}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {
|
||||
"type": "string",
|
||||
"description": "SQL query to execute"
|
||||
}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When you configure a tool to be callable from code execution and Claude decides to use that tool:
|
||||
|
||||
1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic
|
||||
2. Claude runs this code in a sandboxed container via code execution
|
||||
3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field
|
||||
4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window)
|
||||
5. Once all code execution completes, Claude receives the final output and continues working on the task
|
||||
|
||||
This approach is particularly useful for:
|
||||
|
||||
- **Large data processing**: Filter or aggregate tool results before they reach Claude's context
|
||||
- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls
|
||||
- **Conditional logic**: Make decisions based on intermediate tool results
|
||||
|
||||
## The `allowed_callers` Field
|
||||
|
||||
The `allowed_callers` field specifies which contexts can invoke a tool:
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query against the database",
|
||||
"parameters": {...}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
```
|
||||
|
||||
**Possible values:**
|
||||
|
||||
- `["direct"]` - Only Claude can call this tool directly (default if omitted)
|
||||
- `["code_execution_20250825"]` - Only callable from within code execution
|
||||
- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution
|
||||
|
||||
:::tip
|
||||
We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool.
|
||||
:::
|
||||
|
||||
## The `caller` Field in Responses
|
||||
|
||||
Every tool use block includes a `caller` field indicating how it was invoked:
|
||||
|
||||
**Direct invocation (traditional tool use):**
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_abc123",
|
||||
"name": "query_database",
|
||||
"input": {"sql": "<sql>"},
|
||||
"caller": {"type": "direct"}
|
||||
}
|
||||
```
|
||||
|
||||
**Programmatic invocation:**
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_xyz789",
|
||||
"name": "query_database",
|
||||
"input": {"sql": "<sql>"},
|
||||
"caller": {
|
||||
"type": "code_execution_20250825",
|
||||
"tool_id": "srvtoolu_abc123"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `tool_id` references the code execution tool that made the programmatic call.
|
||||
|
||||
## Container Lifecycle
|
||||
|
||||
Programmatic tool calling uses code execution containers:
|
||||
|
||||
- **Container creation**: A new container is created for each session unless you reuse an existing one
|
||||
- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change)
|
||||
- **Container ID**: Pass the `container` parameter to reuse an existing container
|
||||
- **Reuse**: Pass the container ID to maintain state across requests
|
||||
|
||||
```python
|
||||
# First request - creates a new container
|
||||
response1 = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Query the database"}],
|
||||
tools=[...]
|
||||
)
|
||||
|
||||
# Get container ID from response (if available in response metadata)
|
||||
container_id = response1.get("container", {}).get("id")
|
||||
|
||||
# Second request - reuse the same container
|
||||
response2 = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[...],
|
||||
tools=[...],
|
||||
container=container_id # Reuse container
|
||||
)
|
||||
```
|
||||
|
||||
:::warning
|
||||
When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it.
|
||||
:::
|
||||
|
||||
## Example Workflow
|
||||
|
||||
### Step 1: Initial Request
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue"
|
||||
}],
|
||||
tools=[
|
||||
{
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {"type": "string", "description": "SQL query to execute"}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### Step 2: API Response with Tool Call
|
||||
|
||||
Claude writes code that calls your tool. The response includes:
|
||||
|
||||
```python
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll query the purchase history and analyze the results."
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_abc123",
|
||||
"name": "code_execution",
|
||||
"input": {
|
||||
"code": "results = await query_database('<sql>')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_def456",
|
||||
"name": "query_database",
|
||||
"input": {"sql": "<sql>"},
|
||||
"caller": {
|
||||
"type": "code_execution_20250825",
|
||||
"tool_id": "srvtoolu_abc123"
|
||||
}
|
||||
}
|
||||
],
|
||||
"stop_reason": "tool_use"
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3: Provide Tool Result
|
||||
|
||||
```python
|
||||
# Add assistant's response and tool result to conversation
|
||||
messages = [
|
||||
{"role": "user", "content": "Query customer purchase history..."},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": response.choices[0].message.content,
|
||||
"tool_calls": response.choices[0].message.tool_calls
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_def456",
|
||||
"content": '[{"customer_id": "C1", "revenue": 45000}, ...]'
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
# Continue the conversation
|
||||
response2 = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=messages,
|
||||
tools=[...]
|
||||
)
|
||||
```
|
||||
|
||||
### Step 4: Final Response
|
||||
|
||||
Once code execution completes, Claude provides the final response:
|
||||
|
||||
```python
|
||||
{
|
||||
"content": [
|
||||
{
|
||||
"type": "code_execution_tool_result",
|
||||
"tool_use_id": "srvtoolu_abc123",
|
||||
"content": {
|
||||
"type": "code_execution_result",
|
||||
"stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...",
|
||||
"stderr": "",
|
||||
"return_code": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..."
|
||||
}
|
||||
],
|
||||
"stop_reason": "end_turn"
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Patterns
|
||||
|
||||
### Batch Processing with Loops
|
||||
|
||||
Claude can write code that processes multiple items efficiently:
|
||||
|
||||
```python
|
||||
# Claude writes code like this:
|
||||
regions = ["West", "East", "Central", "North", "South"]
|
||||
results = {}
|
||||
for region in regions:
|
||||
data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'")
|
||||
results[region] = data[0]["total"]
|
||||
|
||||
top_region = max(results.items(), key=lambda x: x[1])
|
||||
print(f"Top region: {top_region[0]} with ${top_region[1]:,}")
|
||||
```
|
||||
|
||||
This pattern:
|
||||
- Reduces model round-trips from N (one per region) to 1
|
||||
- Processes large result sets programmatically before returning to Claude
|
||||
- Saves tokens by only returning aggregated conclusions
|
||||
|
||||
### Early Termination
|
||||
|
||||
Claude can stop processing as soon as success criteria are met:
|
||||
|
||||
```python
|
||||
endpoints = ["us-east", "eu-west", "apac"]
|
||||
for endpoint in endpoints:
|
||||
status = await check_health(endpoint)
|
||||
if status == "healthy":
|
||||
print(f"Found healthy endpoint: {endpoint}")
|
||||
break # Stop early
|
||||
```
|
||||
|
||||
### Data Filtering
|
||||
|
||||
```python
|
||||
logs = await fetch_logs(server_id)
|
||||
errors = [log for log in logs if "ERROR" in log]
|
||||
print(f"Found {len(errors)} errors")
|
||||
for error in errors[-10:]: # Only return last 10 errors
|
||||
print(error)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Tool Design
|
||||
|
||||
- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.)
|
||||
- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing
|
||||
- **Keep responses concise**: Return only necessary data to minimize processing overhead
|
||||
|
||||
### When to Use Programmatic Calling
|
||||
|
||||
**Good use cases:**
|
||||
|
||||
- Processing large datasets where you only need aggregates or summaries
|
||||
- Multi-step workflows with 3+ dependent tool calls
|
||||
- Operations requiring filtering, sorting, or transformation of tool results
|
||||
- Tasks where intermediate data shouldn't influence Claude's reasoning
|
||||
- Parallel operations across many items (e.g., checking 50 endpoints)
|
||||
|
||||
**Less ideal use cases:**
|
||||
|
||||
- Single tool calls with simple responses
|
||||
- Tools that need immediate user feedback
|
||||
- Very fast operations where code execution overhead would outweigh the benefit
|
||||
|
||||
## Token Efficiency
|
||||
|
||||
Programmatic tool calling can significantly reduce token consumption:
|
||||
|
||||
- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is
|
||||
- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens
|
||||
- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns
|
||||
|
||||
For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary.
|
||||
|
||||
## Provider Support
|
||||
|
||||
LiteLLM supports programmatic tool calling across all Anthropic-compatible providers:
|
||||
|
||||
- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
|
||||
- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
|
||||
- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
|
||||
|
||||
The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field.
|
||||
|
||||
## Limitations
|
||||
|
||||
### Feature Incompatibilities
|
||||
|
||||
- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling
|
||||
- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice`
|
||||
- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling
|
||||
|
||||
### Tool Restrictions
|
||||
|
||||
The following tools cannot currently be called programmatically:
|
||||
|
||||
- Web search
|
||||
- Web fetch
|
||||
- Tools provided by an MCP connector
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**"Tool not allowed" error**
|
||||
|
||||
- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]`
|
||||
- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5)
|
||||
|
||||
**Container expiration**
|
||||
|
||||
- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes)
|
||||
- Consider implementing faster tool execution
|
||||
|
||||
**Beta header not added**
|
||||
|
||||
- LiteLLM automatically adds the beta header when it detects `allowed_callers`
|
||||
- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20`
|
||||
|
||||
## Related Features
|
||||
|
||||
- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
|
||||
- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation
|
||||
|
||||
438
docs/my-website/docs/providers/anthropic_tool_input_examples.md
Normal file
438
docs/my-website/docs/providers/anthropic_tool_input_examples.md
Normal file
|
|
@ -0,0 +1,438 @@
|
|||
# Anthropic Tool Input Examples
|
||||
|
||||
Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs.
|
||||
|
||||
:::info
|
||||
Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field.
|
||||
:::
|
||||
|
||||
## When to Use Input Examples
|
||||
|
||||
Input examples are most helpful for:
|
||||
|
||||
- **Complex nested objects**: Tools with deeply nested parameter structures
|
||||
- **Optional parameters**: Showing when optional parameters should be included
|
||||
- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.)
|
||||
- **Enum values**: Illustrating valid enum choices in context
|
||||
- **Edge cases**: Showing how to handle special cases
|
||||
|
||||
:::tip
|
||||
**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient.
|
||||
:::
|
||||
|
||||
## Quick Start
|
||||
|
||||
Add an `input_examples` field to your tool definition with an array of example input objects:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather like in San Francisco?"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
"description": "The unit of temperature"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{
|
||||
"location": "San Francisco, CA",
|
||||
"unit": "fahrenheit"
|
||||
},
|
||||
{
|
||||
"location": "Tokyo, Japan",
|
||||
"unit": "celsius"
|
||||
},
|
||||
{
|
||||
"location": "New York, NY" # 'unit' is optional
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
When you provide `input_examples`:
|
||||
|
||||
1. **LiteLLM detects** the `input_examples` field in your tool definition
|
||||
2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected
|
||||
3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema
|
||||
4. **Claude learns patterns**: The model uses examples to understand proper tool usage
|
||||
5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats
|
||||
|
||||
## Example Formats
|
||||
|
||||
### Simple Tool with Examples
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "send_email",
|
||||
"description": "Send an email to a recipient",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"to": {"type": "string", "description": "Email address"},
|
||||
"subject": {"type": "string"},
|
||||
"body": {"type": "string"}
|
||||
},
|
||||
"required": ["to", "subject", "body"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{
|
||||
"to": "user@example.com",
|
||||
"subject": "Meeting Reminder",
|
||||
"body": "Don't forget our meeting tomorrow at 2 PM."
|
||||
},
|
||||
{
|
||||
"to": "team@company.com",
|
||||
"subject": "Weekly Update",
|
||||
"body": "Here's this week's progress report..."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Complex Nested Objects
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_calendar_event",
|
||||
"description": "Create a new calendar event",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"start": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"date": {"type": "string"},
|
||||
"time": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"attendees": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"email": {"type": "string"},
|
||||
"optional": {"type": "boolean"}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["title", "start"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{
|
||||
"title": "Team Standup",
|
||||
"start": {
|
||||
"date": "2025-01-15",
|
||||
"time": "09:00"
|
||||
},
|
||||
"attendees": [
|
||||
{"email": "alice@example.com", "optional": False},
|
||||
{"email": "bob@example.com", "optional": True}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Lunch Break",
|
||||
"start": {
|
||||
"date": "2025-01-15",
|
||||
"time": "12:00"
|
||||
}
|
||||
# No attendees - showing optional field
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Format-Sensitive Parameters
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_flights",
|
||||
"description": "Search for available flights",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"origin": {"type": "string", "description": "Airport code"},
|
||||
"destination": {"type": "string", "description": "Airport code"},
|
||||
"date": {"type": "string", "description": "Date in YYYY-MM-DD format"},
|
||||
"passengers": {"type": "integer"}
|
||||
},
|
||||
"required": ["origin", "destination", "date"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{
|
||||
"origin": "SFO",
|
||||
"destination": "JFK",
|
||||
"date": "2025-03-15",
|
||||
"passengers": 2
|
||||
},
|
||||
{
|
||||
"origin": "LAX",
|
||||
"destination": "ORD",
|
||||
"date": "2025-04-20",
|
||||
"passengers": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Requirements and Limitations
|
||||
|
||||
### Schema Validation
|
||||
|
||||
- Each example **must be valid** according to the tool's `input_schema`
|
||||
- Invalid examples will return a **400 error** from Anthropic
|
||||
- Validation happens server-side (LiteLLM passes examples through)
|
||||
|
||||
### Server-Side Tools Not Supported
|
||||
|
||||
Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`:
|
||||
|
||||
- `web_search` (web search tool)
|
||||
- `code_execution` (code execution tool)
|
||||
- `computer_use` (computer use tool)
|
||||
- `bash_tool` (bash execution tool)
|
||||
- `text_editor` (text editor tool)
|
||||
|
||||
### Token Costs
|
||||
|
||||
Examples add to your prompt tokens:
|
||||
|
||||
- **Simple examples**: ~20-50 tokens per example
|
||||
- **Complex nested objects**: ~100-200 tokens per example
|
||||
- **Trade-off**: Higher token cost for better tool call accuracy
|
||||
|
||||
### Model Compatibility
|
||||
|
||||
Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header:
|
||||
|
||||
- Claude Opus 4.5 (`claude-opus-4-5-20251101`)
|
||||
- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`)
|
||||
- Claude Opus 4.1 (`claude-opus-4-1-20250805`)
|
||||
|
||||
:::note
|
||||
On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples.
|
||||
:::
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Show Diverse Examples
|
||||
|
||||
Include examples that demonstrate different use cases:
|
||||
|
||||
```python
|
||||
"input_examples": [
|
||||
{"location": "San Francisco, CA", "unit": "fahrenheit"}, # US city
|
||||
{"location": "Tokyo, Japan", "unit": "celsius"}, # International
|
||||
{"location": "New York, NY"} # Optional param omitted
|
||||
]
|
||||
```
|
||||
|
||||
### 2. Demonstrate Optional Parameters
|
||||
|
||||
Show when optional parameters should and shouldn't be included:
|
||||
|
||||
```python
|
||||
"input_examples": [
|
||||
{
|
||||
"query": "machine learning",
|
||||
"filters": {"year": 2024, "category": "research"} # With optional filters
|
||||
},
|
||||
{
|
||||
"query": "artificial intelligence" # Without optional filters
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### 3. Illustrate Format Requirements
|
||||
|
||||
Make format expectations clear through examples:
|
||||
|
||||
```python
|
||||
"input_examples": [
|
||||
{
|
||||
"phone": "+1-555-123-4567", # Shows expected phone format
|
||||
"date": "2025-01-15", # Shows date format (YYYY-MM-DD)
|
||||
"time": "14:30" # Shows time format (HH:MM)
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### 4. Keep Examples Realistic
|
||||
|
||||
Use realistic, production-like examples rather than placeholder data:
|
||||
|
||||
```python
|
||||
# ✅ Good - realistic examples
|
||||
"input_examples": [
|
||||
{"email": "alice@company.com", "role": "admin"},
|
||||
{"email": "bob@company.com", "role": "user"}
|
||||
]
|
||||
|
||||
# ❌ Bad - placeholder examples
|
||||
"input_examples": [
|
||||
{"email": "test@test.com", "role": "role1"},
|
||||
{"email": "example@example.com", "role": "role2"}
|
||||
]
|
||||
```
|
||||
|
||||
### 5. Limit Example Count
|
||||
|
||||
Provide 2-5 examples per tool:
|
||||
|
||||
- **Too few** (1): May not show enough variation
|
||||
- **Just right** (2-5): Demonstrates patterns without bloating tokens
|
||||
- **Too many** (10+): Wastes tokens, diminishing returns
|
||||
|
||||
## Integration with Other Features
|
||||
|
||||
Input examples work seamlessly with other Anthropic tool features:
|
||||
|
||||
### With Tool Search
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query",
|
||||
"parameters": {...}
|
||||
},
|
||||
"defer_loading": True, # Tool search
|
||||
"input_examples": [ # Input examples
|
||||
{"sql": "SELECT * FROM users WHERE id = 1"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### With Programmatic Tool Calling
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "fetch_data",
|
||||
"description": "Fetch data from API",
|
||||
"parameters": {...}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"], # Programmatic calling
|
||||
"input_examples": [ # Input examples
|
||||
{"endpoint": "/api/users", "method": "GET"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### All Features Combined
|
||||
|
||||
```python
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "advanced_tool",
|
||||
"description": "A complex tool",
|
||||
"parameters": {...}
|
||||
},
|
||||
"defer_loading": True, # Tool search
|
||||
"allowed_callers": ["code_execution_20250825"], # Programmatic calling
|
||||
"input_examples": [ # Input examples
|
||||
{"param1": "value1", "param2": "value2"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Provider Support
|
||||
|
||||
LiteLLM supports input examples across all Anthropic-compatible providers:
|
||||
|
||||
- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
|
||||
- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
|
||||
- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
|
||||
|
||||
The beta header is automatically added when LiteLLM detects tools with `input_examples` field.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Invalid request" error with examples
|
||||
|
||||
**Problem**: Receiving 400 error when using input examples
|
||||
|
||||
**Solution**: Ensure each example is valid according to your `input_schema`:
|
||||
|
||||
```python
|
||||
# Check that:
|
||||
# 1. All required fields are present in examples
|
||||
# 2. Field types match the schema
|
||||
# 3. Enum values are valid
|
||||
# 4. Nested objects follow the schema structure
|
||||
```
|
||||
|
||||
### Examples not improving tool calls
|
||||
|
||||
**Problem**: Adding examples doesn't seem to help
|
||||
|
||||
**Solution**:
|
||||
1. **Check descriptions first**: Ensure tool descriptions are detailed and clear
|
||||
2. **Review example quality**: Make sure examples are realistic and diverse
|
||||
3. **Verify schema**: Confirm examples actually match your schema
|
||||
4. **Add more variation**: Include examples showing different use cases
|
||||
|
||||
### Token usage too high
|
||||
|
||||
**Problem**: Input examples consuming too many tokens
|
||||
|
||||
**Solution**:
|
||||
1. **Reduce example count**: Use 2-3 examples instead of 5+
|
||||
2. **Simplify examples**: Remove unnecessary fields from examples
|
||||
3. **Consider descriptions**: If descriptions are clear, examples may not be needed
|
||||
|
||||
## When NOT to Use Input Examples
|
||||
|
||||
Skip input examples if:
|
||||
|
||||
- **Tool is simple**: Single parameter tools with clear descriptions
|
||||
- **Schema is self-explanatory**: Well-structured schema with good descriptions
|
||||
- **Token budget is tight**: Examples add 20-200 tokens each
|
||||
- **Server-side tools**: web_search, code_execution, etc. don't support examples
|
||||
|
||||
## Related Features
|
||||
|
||||
- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
|
||||
- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution
|
||||
- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation
|
||||
|
||||
397
docs/my-website/docs/providers/anthropic_tool_search.md
Normal file
397
docs/my-website/docs/providers/anthropic_tool_search.md
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
# Anthropic Tool Search
|
||||
|
||||
Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
|
||||
|
||||
## Benefits
|
||||
|
||||
- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
|
||||
- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
|
||||
- **On-demand loading**: Tools are only loaded when Claude needs them
|
||||
|
||||
## Supported Models
|
||||
|
||||
Tool search is available on:
|
||||
- Claude Opus 4.5
|
||||
- Claude Sonnet 4.5
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
- Anthropic API (direct)
|
||||
- Azure Anthropic (Microsoft Foundry)
|
||||
- Google Cloud Vertex AI
|
||||
- Amazon Bedrock (invoke API only, not converse API)
|
||||
|
||||
## Tool Search Variants
|
||||
|
||||
LiteLLM supports both tool search variants:
|
||||
|
||||
### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
|
||||
|
||||
Claude constructs regex patterns to search for tools.
|
||||
|
||||
### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
|
||||
|
||||
Claude uses natural language queries to search for tools using the BM25 algorithm.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Basic Example with Regex Tool Search
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the weather in San Francisco?"}
|
||||
],
|
||||
tools=[
|
||||
# Tool search tool (regex variant)
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
# Deferred tool - will be loaded on-demand
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather at a specific location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"]
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True # Mark for deferred loading
|
||||
},
|
||||
# Another deferred tool
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_files",
|
||||
"description": "Search through files in the workspace",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"file_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### BM25 Tool Search Example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[
|
||||
{"role": "user", "content": "Search for Python files containing 'authentication'"}
|
||||
],
|
||||
tools=[
|
||||
# Tool search tool (BM25 variant)
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
},
|
||||
# Deferred tools...
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "search_codebase",
|
||||
"description": "Search through codebase files by content and filename",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"file_pattern": {"type": "string"}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Using with Azure Anthropic
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="azure_anthropic/claude-sonnet-4-5",
|
||||
api_base="https://<your-resource>.services.ai.azure.com/anthropic",
|
||||
api_key="your-azure-api-key",
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather like?"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get current weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Using with Vertex AI
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="vertex_ai/claude-sonnet-4-5",
|
||||
vertex_project="your-project-id",
|
||||
vertex_location="us-central1",
|
||||
messages=[
|
||||
{"role": "user", "content": "Search my documents"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
},
|
||||
# Your deferred tools...
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Streaming Support
|
||||
|
||||
Tool search works with streaming:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[
|
||||
{"role": "user", "content": "Get the weather"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
],
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if chunk.choices[0].delta.content:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
## LiteLLM Proxy
|
||||
|
||||
Tool search works automatically through the LiteLLM proxy:
|
||||
|
||||
### Proxy Config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5-20250929
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
### Client Request
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your-litellm-proxy-key",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="claude-sonnet",
|
||||
messages=[
|
||||
{"role": "user", "content": "What's the weather?"}
|
||||
],
|
||||
tools=[
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
## Important Notes
|
||||
|
||||
### Beta Header
|
||||
|
||||
LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it.
|
||||
|
||||
### Deferred Loading
|
||||
|
||||
- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
|
||||
- At least one tool must be non-deferred (the tool search tool itself)
|
||||
- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
|
||||
|
||||
### Tool Descriptions
|
||||
|
||||
Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
|
||||
- Tool names
|
||||
- Tool descriptions
|
||||
- Argument names
|
||||
- Argument descriptions
|
||||
|
||||
### Usage Tracking
|
||||
|
||||
Tool search requests are tracked in the usage object:
|
||||
|
||||
```python
|
||||
response = litellm.completion(
|
||||
model="anthropic/claude-sonnet-4-5-20250929",
|
||||
messages=[{"role": "user", "content": "Search for tools"}],
|
||||
tools=[...]
|
||||
)
|
||||
|
||||
# Check tool search usage
|
||||
if response.usage.server_tool_use:
|
||||
print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### All Tools Deferred
|
||||
|
||||
```python
|
||||
# ❌ This will fail - at least one tool must be non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
|
||||
# ✅ Correct - tool search tool is non-deferred
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {...},
|
||||
"defer_loading": True
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### Missing Tool Definition
|
||||
|
||||
If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
|
||||
|
||||
2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
|
||||
|
||||
3. **Choose the right variant**:
|
||||
- Use **regex** for exact pattern matching (faster)
|
||||
- Use **BM25** for natural language semantic search
|
||||
|
||||
4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
|
||||
|
||||
5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
|
||||
|
||||
## When to Use Tool Search
|
||||
|
||||
**Good use cases:**
|
||||
- 10+ tools available in your system
|
||||
- Tool definitions consuming >10K tokens
|
||||
- Experiencing tool selection accuracy issues
|
||||
- Building systems with multiple tool categories
|
||||
- Tool library growing over time
|
||||
|
||||
**When traditional tool calling is better:**
|
||||
- Less than 10 tools total
|
||||
- All tools are frequently used
|
||||
- Very small tool definitions (<100 tokens total)
|
||||
|
||||
## Limitations
|
||||
|
||||
- Not compatible with tool use examples
|
||||
- Requires Claude Opus 4.5 or Sonnet 4.5
|
||||
- On Bedrock, only available via invoke API (not converse API)
|
||||
- Maximum 10,000 tools in catalog
|
||||
- Returns 3-5 most relevant tools per search
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
|
||||
- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)
|
||||
|
||||
|
|
@ -42,6 +42,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolCallChunk,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
|
|
@ -550,15 +551,18 @@ class ModelResponseIterator:
|
|||
if "text" in content_block["delta"]:
|
||||
text = content_block["delta"]["text"]
|
||||
elif "partial_json" in content_block["delta"]:
|
||||
tool_use = {
|
||||
"id": None,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": None,
|
||||
"arguments": content_block["delta"]["partial_json"],
|
||||
tool_use = cast(
|
||||
ChatCompletionToolCallChunk,
|
||||
{
|
||||
"id": None,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": None,
|
||||
"arguments": content_block["delta"]["partial_json"],
|
||||
},
|
||||
"index": self.tool_index,
|
||||
},
|
||||
"index": self.tool_index,
|
||||
}
|
||||
)
|
||||
elif "citation" in content_block["delta"]:
|
||||
provider_specific_fields["citation"] = content_block["delta"]["citation"]
|
||||
elif (
|
||||
|
|
@ -569,7 +573,7 @@ class ModelResponseIterator:
|
|||
ChatCompletionThinkingBlock(
|
||||
type="thinking",
|
||||
thinking=content_block["delta"].get("thinking") or "",
|
||||
signature=content_block["delta"].get("signature"),
|
||||
signature=str(content_block["delta"].get("signature") or ""),
|
||||
)
|
||||
]
|
||||
provider_specific_fields["thinking_blocks"] = thinking_blocks
|
||||
|
|
@ -625,7 +629,7 @@ class ModelResponseIterator:
|
|||
|
||||
return content_block_start
|
||||
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915
|
||||
try:
|
||||
type_chunk = chunk.get("type", "") or ""
|
||||
|
||||
|
|
@ -672,15 +676,32 @@ class ModelResponseIterator:
|
|||
text = content_block_start["content_block"]["text"]
|
||||
elif content_block_start["content_block"]["type"] == "tool_use":
|
||||
self.tool_index += 1
|
||||
tool_use = {
|
||||
"id": content_block_start["content_block"]["id"],
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": content_block_start["content_block"]["name"],
|
||||
"arguments": "",
|
||||
},
|
||||
"index": self.tool_index,
|
||||
}
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=content_block_start["content_block"]["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=content_block_start["content_block"]["name"],
|
||||
arguments="",
|
||||
),
|
||||
index=self.tool_index,
|
||||
)
|
||||
# Include caller information if present (for programmatic tool calling)
|
||||
if "caller" in content_block_start["content_block"]:
|
||||
caller_data = content_block_start["content_block"]["caller"]
|
||||
if caller_data:
|
||||
tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item]
|
||||
elif content_block_start["content_block"]["type"] == "server_tool_use":
|
||||
# Handle server tool use (for tool search)
|
||||
self.tool_index += 1
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=content_block_start["content_block"]["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=content_block_start["content_block"]["name"],
|
||||
arguments="",
|
||||
),
|
||||
index=self.tool_index,
|
||||
)
|
||||
elif (
|
||||
content_block_start["content_block"]["type"] == "redacted_thinking"
|
||||
):
|
||||
|
|
@ -696,17 +717,21 @@ class ModelResponseIterator:
|
|||
# check if tool call content block
|
||||
is_empty = self.check_empty_tool_call_args()
|
||||
if is_empty:
|
||||
tool_use = {
|
||||
"id": None,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": None,
|
||||
"arguments": "{}",
|
||||
},
|
||||
"index": self.tool_index,
|
||||
}
|
||||
tool_use = ChatCompletionToolCallChunk(
|
||||
id=None, # type: ignore[typeddict-item]
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=None, # type: ignore[typeddict-item]
|
||||
arguments="{}",
|
||||
),
|
||||
index=self.tool_index,
|
||||
)
|
||||
# Reset response_format tool tracking when block stops
|
||||
self.is_response_format_tool = False
|
||||
elif type_chunk == "tool_result":
|
||||
# Handle tool_result blocks (for tool search results with tool_reference)
|
||||
# These are automatically handled by Anthropic API, we just pass them through
|
||||
pass
|
||||
elif type_chunk == "message_delta":
|
||||
finish_reason, usage = self._handle_message_delta(chunk)
|
||||
elif type_chunk == "message_start":
|
||||
|
|
|
|||
|
|
@ -54,7 +54,10 @@ from litellm.types.utils import (
|
|||
CompletionTokensDetailsWrapper,
|
||||
)
|
||||
from litellm.types.utils import Message as LitellmMessage
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
|
||||
from litellm.types.utils import (
|
||||
PromptTokensDetailsWrapper,
|
||||
ServerToolUse,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ModelResponse,
|
||||
Usage,
|
||||
|
|
@ -187,7 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
return _tool_choice
|
||||
|
||||
def _map_tool_helper(
|
||||
def _map_tool_helper( # noqa: PLR0915
|
||||
self, tool: ChatCompletionToolParam
|
||||
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
|
||||
returned_tool: Optional[AllAnthropicToolsValues] = None
|
||||
|
|
@ -250,9 +253,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
returned_tool = _computer_tool
|
||||
elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS):
|
||||
function_name = tool.get("name", tool.get("function", {}).get("name"))
|
||||
if function_name is None or not isinstance(function_name, str):
|
||||
function_name_obj = tool.get("name", tool.get("function", {}).get("name"))
|
||||
if function_name_obj is None or not isinstance(function_name_obj, str):
|
||||
raise ValueError("Missing required parameter: name")
|
||||
function_name = function_name_obj
|
||||
|
||||
additional_tool_params = {}
|
||||
for k, v in tool.items():
|
||||
|
|
@ -268,6 +272,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
mcp_server = self._map_openai_mcp_server_tool(
|
||||
cast(OpenAIMcpServerTool, tool)
|
||||
)
|
||||
elif tool["type"] == "tool_search_tool_regex_20251119":
|
||||
# Tool search tool using regex
|
||||
from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex
|
||||
|
||||
tool_name_obj = tool.get("name", "tool_search_tool_regex")
|
||||
if not isinstance(tool_name_obj, str):
|
||||
raise ValueError("Tool search tool must have a valid name")
|
||||
tool_name = tool_name_obj
|
||||
returned_tool = AnthropicToolSearchToolRegex(
|
||||
type="tool_search_tool_regex_20251119",
|
||||
name=tool_name,
|
||||
)
|
||||
elif tool["type"] == "tool_search_tool_bm25_20251119":
|
||||
# Tool search tool using BM25
|
||||
from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25
|
||||
|
||||
tool_name_obj = tool.get("name", "tool_search_tool_bm25")
|
||||
if not isinstance(tool_name_obj, str):
|
||||
raise ValueError("Tool search tool must have a valid name")
|
||||
tool_name = tool_name_obj
|
||||
returned_tool = AnthropicToolSearchToolBM25(
|
||||
type="tool_search_tool_bm25_20251119",
|
||||
name=tool_name,
|
||||
)
|
||||
if returned_tool is None and mcp_server is None:
|
||||
raise ValueError(f"Unsupported tool type: {tool['type']}")
|
||||
|
||||
|
|
@ -275,14 +303,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_cache_control = tool.get("cache_control", None)
|
||||
_cache_control_function = tool.get("function", {}).get("cache_control", None)
|
||||
if returned_tool is not None:
|
||||
if _cache_control is not None:
|
||||
returned_tool["cache_control"] = _cache_control
|
||||
elif _cache_control_function is not None and isinstance(
|
||||
_cache_control_function, dict
|
||||
):
|
||||
returned_tool["cache_control"] = ChatCompletionCachedContent(
|
||||
**_cache_control_function # type: ignore
|
||||
)
|
||||
# Only set cache_control on tools that support it (not tool search tools)
|
||||
tool_type = returned_tool.get("type", "")
|
||||
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
|
||||
if _cache_control is not None:
|
||||
returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item]
|
||||
elif _cache_control_function is not None and isinstance(
|
||||
_cache_control_function, dict
|
||||
):
|
||||
returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item]
|
||||
**_cache_control_function # type: ignore
|
||||
)
|
||||
|
||||
## check if defer_loading is set in the tool
|
||||
_defer_loading = tool.get("defer_loading", None)
|
||||
_defer_loading_function = tool.get("function", {}).get("defer_loading", None)
|
||||
if returned_tool is not None:
|
||||
# Only set defer_loading on tools that support it (not tool search tools or computer tools)
|
||||
tool_type = returned_tool.get("type", "")
|
||||
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
|
||||
if _defer_loading is not None:
|
||||
if not isinstance(_defer_loading, bool):
|
||||
raise ValueError("defer_loading must be a boolean")
|
||||
returned_tool["defer_loading"] = _defer_loading # type: ignore[typeddict-item]
|
||||
elif _defer_loading_function is not None:
|
||||
if not isinstance(_defer_loading_function, bool):
|
||||
raise ValueError("defer_loading must be a boolean")
|
||||
returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item]
|
||||
|
||||
## check if allowed_callers is set in the tool
|
||||
_allowed_callers = tool.get("allowed_callers", None)
|
||||
_allowed_callers_function = tool.get("function", {}).get("allowed_callers", None)
|
||||
if returned_tool is not None:
|
||||
# Only set allowed_callers on tools that support it (not tool search tools or computer tools)
|
||||
tool_type = returned_tool.get("type", "")
|
||||
if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"):
|
||||
if _allowed_callers is not None:
|
||||
if not isinstance(_allowed_callers, list) or not all(
|
||||
isinstance(item, str) for item in _allowed_callers
|
||||
):
|
||||
raise ValueError("allowed_callers must be a list of strings")
|
||||
returned_tool["allowed_callers"] = _allowed_callers # type: ignore[typeddict-item]
|
||||
elif _allowed_callers_function is not None:
|
||||
if not isinstance(_allowed_callers_function, list) or not all(
|
||||
isinstance(item, str) for item in _allowed_callers_function
|
||||
):
|
||||
raise ValueError("allowed_callers must be a list of strings")
|
||||
returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item]
|
||||
|
||||
## check if input_examples is set in the tool
|
||||
_input_examples = tool.get("input_examples", None)
|
||||
_input_examples_function = tool.get("function", {}).get("input_examples", None)
|
||||
if returned_tool is not None:
|
||||
# Only set input_examples on user-defined tools (type "custom" or no type)
|
||||
tool_type = returned_tool.get("type", "")
|
||||
if tool_type == "custom" or (tool_type == "" and "name" in returned_tool):
|
||||
if _input_examples is not None and isinstance(_input_examples, list):
|
||||
returned_tool["input_examples"] = _input_examples # type: ignore[typeddict-item]
|
||||
elif _input_examples_function is not None and isinstance(
|
||||
_input_examples_function, list
|
||||
):
|
||||
returned_tool["input_examples"] = _input_examples_function # type: ignore[typeddict-item]
|
||||
|
||||
return returned_tool, mcp_server
|
||||
|
||||
|
|
@ -334,6 +415,82 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
mcp_servers.append(mcp_server_tool)
|
||||
return anthropic_tools, mcp_servers
|
||||
|
||||
def _detect_tool_search_tools(self, tools: Optional[List]) -> bool:
|
||||
"""Check if tool search tools are present in the tools list."""
|
||||
if not tools:
|
||||
return False
|
||||
|
||||
for tool in tools:
|
||||
tool_type = tool.get("type", "")
|
||||
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _separate_deferred_tools(
|
||||
self, tools: List
|
||||
) -> Tuple[List, List]:
|
||||
"""
|
||||
Separate tools into deferred and non-deferred lists.
|
||||
|
||||
Returns:
|
||||
Tuple of (non_deferred_tools, deferred_tools)
|
||||
"""
|
||||
non_deferred = []
|
||||
deferred = []
|
||||
|
||||
for tool in tools:
|
||||
if tool.get("defer_loading", False):
|
||||
deferred.append(tool)
|
||||
else:
|
||||
non_deferred.append(tool)
|
||||
|
||||
return non_deferred, deferred
|
||||
|
||||
def _expand_tool_references(
|
||||
self,
|
||||
content: List,
|
||||
deferred_tools: List,
|
||||
) -> List:
|
||||
"""
|
||||
Expand tool_reference blocks to full tool definitions.
|
||||
|
||||
When Anthropic's tool search returns results, it includes tool_reference blocks
|
||||
that reference tools by name. This method expands those references to full
|
||||
tool definitions from the deferred_tools catalog.
|
||||
|
||||
Args:
|
||||
content: Response content that may contain tool_reference blocks
|
||||
deferred_tools: List of deferred tools that can be referenced
|
||||
|
||||
Returns:
|
||||
Content with tool_reference blocks expanded to full tool definitions
|
||||
"""
|
||||
if not deferred_tools:
|
||||
return content
|
||||
|
||||
# Create a mapping of tool names to tool definitions
|
||||
tool_map = {}
|
||||
for tool in deferred_tools:
|
||||
tool_name = tool.get("name") or tool.get("function", {}).get("name")
|
||||
if tool_name:
|
||||
tool_map[tool_name] = tool
|
||||
|
||||
# Expand tool references in content
|
||||
expanded_content = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "tool_reference":
|
||||
tool_name = item.get("tool_name")
|
||||
if tool_name and tool_name in tool_map:
|
||||
# Replace reference with full tool definition
|
||||
expanded_content.append(tool_map[tool_name])
|
||||
else:
|
||||
# Keep the reference if we can't find the tool
|
||||
expanded_content.append(item)
|
||||
else:
|
||||
expanded_content.append(item)
|
||||
|
||||
return expanded_content
|
||||
|
||||
def _map_stop_sequences(
|
||||
self, stop: Optional[Union[str, List[str]]]
|
||||
) -> Optional[List[str]]:
|
||||
|
|
@ -822,6 +979,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
"messages": anthropic_messages,
|
||||
**optional_params,
|
||||
}
|
||||
|
||||
## Handle output_config (Anthropic-specific parameter)
|
||||
if "output_config" in optional_params:
|
||||
output_config = optional_params.get("output_config")
|
||||
if output_config and isinstance(output_config, dict):
|
||||
effort = output_config.get("effort")
|
||||
if effort and effort not in ["high", "medium", "low"]:
|
||||
raise ValueError(
|
||||
f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'"
|
||||
)
|
||||
data["output_config"] = output_config
|
||||
|
||||
return data
|
||||
|
||||
|
|
@ -870,18 +1038,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
text_content += content["text"]
|
||||
## TOOL CALLING
|
||||
elif content["type"] == "tool_use":
|
||||
tool_calls.append(
|
||||
ChatCompletionToolCallChunk(
|
||||
id=content["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=content["name"],
|
||||
arguments=json.dumps(content["input"]),
|
||||
),
|
||||
index=idx,
|
||||
)
|
||||
tool_call = ChatCompletionToolCallChunk(
|
||||
id=content["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=content["name"],
|
||||
arguments=json.dumps(content["input"]),
|
||||
),
|
||||
index=idx,
|
||||
)
|
||||
|
||||
# Include caller information if present (for programmatic tool calling)
|
||||
if "caller" in content:
|
||||
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
|
||||
tool_calls.append(tool_call)
|
||||
## SERVER TOOL USE (for tool search)
|
||||
elif content["type"] == "server_tool_use":
|
||||
# Server tool use blocks are for tool search - treat as tool calls
|
||||
tool_call = ChatCompletionToolCallChunk(
|
||||
id=content["id"],
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=content["name"],
|
||||
arguments=json.dumps(content.get("input", {})),
|
||||
),
|
||||
index=idx,
|
||||
)
|
||||
# Include caller information if present (for programmatic tool calling)
|
||||
if "caller" in content:
|
||||
tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item]
|
||||
tool_calls.append(tool_call)
|
||||
## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery)
|
||||
elif content["type"] == "tool_search_tool_result":
|
||||
# This block contains tool_references that were discovered
|
||||
# We don't need to include this in the response as it's internal metadata
|
||||
pass
|
||||
elif content.get("thinking", None) is not None:
|
||||
if thinking_blocks is None:
|
||||
thinking_blocks = []
|
||||
|
|
@ -916,7 +1106,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
return text_content, citations, thinking_blocks, reasoning_content, tool_calls
|
||||
|
||||
def calculate_usage(
|
||||
self, usage_object: dict, reasoning_content: Optional[str]
|
||||
self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None
|
||||
) -> Usage:
|
||||
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
|
||||
prompt_tokens = usage_object.get("input_tokens", 0) or 0
|
||||
|
|
@ -926,6 +1116,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
cache_read_input_tokens: int = 0
|
||||
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
|
||||
web_search_requests: Optional[int] = None
|
||||
tool_search_requests: Optional[int] = None
|
||||
if (
|
||||
"cache_creation_input_tokens" in _usage
|
||||
and _usage["cache_creation_input_tokens"] is not None
|
||||
|
|
@ -946,6 +1137,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
web_search_requests = cast(
|
||||
int, _usage["server_tool_use"]["web_search_requests"]
|
||||
)
|
||||
if (
|
||||
"tool_search_requests" in _usage["server_tool_use"]
|
||||
and _usage["server_tool_use"]["tool_search_requests"] is not None
|
||||
):
|
||||
tool_search_requests = cast(
|
||||
int, _usage["server_tool_use"]["tool_search_requests"]
|
||||
)
|
||||
|
||||
# Count tool_search_requests from content blocks if not in usage
|
||||
# Anthropic doesn't always include tool_search_requests in the usage object
|
||||
if tool_search_requests is None and completion_response is not None:
|
||||
tool_search_count = 0
|
||||
for content in completion_response.get("content", []):
|
||||
if content.get("type") == "server_tool_use":
|
||||
tool_name = content.get("name", "")
|
||||
if "tool_search" in tool_name:
|
||||
tool_search_count += 1
|
||||
if tool_search_count > 0:
|
||||
tool_search_requests = tool_search_count
|
||||
|
||||
if "cache_creation" in _usage and _usage["cache_creation"] is not None:
|
||||
cache_creation_token_details = CacheCreationTokenDetails(
|
||||
|
|
@ -982,8 +1192,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
completion_tokens_details=completion_token_details,
|
||||
server_tool_use=(
|
||||
ServerToolUse(web_search_requests=web_search_requests)
|
||||
if web_search_requests is not None
|
||||
ServerToolUse(
|
||||
web_search_requests=web_search_requests,
|
||||
tool_search_requests=tool_search_requests,
|
||||
)
|
||||
if (web_search_requests is not None or tool_search_requests is not None)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
|
@ -1077,6 +1290,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
usage = self.calculate_usage(
|
||||
usage_object=completion_response["usage"],
|
||||
reasoning_content=reasoning_content,
|
||||
completion_response=completion_response,
|
||||
)
|
||||
setattr(model_response, "usage", usage) # type: ignore
|
||||
|
||||
|
|
|
|||
|
|
@ -88,6 +88,86 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
return True
|
||||
return False
|
||||
|
||||
def is_tool_search_used(self, tools: Optional[List]) -> bool:
|
||||
"""
|
||||
Check if tool search tools are present in the tools list.
|
||||
"""
|
||||
if not tools:
|
||||
return False
|
||||
|
||||
for tool in tools:
|
||||
tool_type = tool.get("type", "")
|
||||
if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]:
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool:
|
||||
"""
|
||||
Check if programmatic tool calling is being used (tools with allowed_callers field).
|
||||
|
||||
Returns True if any tool has allowed_callers containing 'code_execution_20250825'.
|
||||
"""
|
||||
if not tools:
|
||||
return False
|
||||
|
||||
for tool in tools:
|
||||
# Check top-level allowed_callers
|
||||
allowed_callers = tool.get("allowed_callers", None)
|
||||
if allowed_callers and isinstance(allowed_callers, list):
|
||||
if "code_execution_20250825" in allowed_callers:
|
||||
return True
|
||||
|
||||
# Check function.allowed_callers for OpenAI format tools
|
||||
function = tool.get("function", {})
|
||||
if isinstance(function, dict):
|
||||
function_allowed_callers = function.get("allowed_callers", None)
|
||||
if function_allowed_callers and isinstance(function_allowed_callers, list):
|
||||
if "code_execution_20250825" in function_allowed_callers:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def is_input_examples_used(self, tools: Optional[List]) -> bool:
|
||||
"""
|
||||
Check if input_examples is being used in any tools.
|
||||
|
||||
Returns True if any tool has input_examples field.
|
||||
"""
|
||||
if not tools:
|
||||
return False
|
||||
|
||||
for tool in tools:
|
||||
# Check top-level input_examples
|
||||
input_examples = tool.get("input_examples", None)
|
||||
if input_examples and isinstance(input_examples, list) and len(input_examples) > 0:
|
||||
return True
|
||||
|
||||
# Check function.input_examples for OpenAI format tools
|
||||
function = tool.get("function", {})
|
||||
if isinstance(function, dict):
|
||||
function_input_examples = function.get("input_examples", None)
|
||||
if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def is_effort_used(self, optional_params: Optional[dict]) -> bool:
|
||||
"""
|
||||
Check if effort parameter is being used via output_config.
|
||||
|
||||
Returns True if output_config with effort field is present.
|
||||
"""
|
||||
if not optional_params:
|
||||
return False
|
||||
|
||||
output_config = optional_params.get("output_config")
|
||||
if output_config and isinstance(output_config, dict):
|
||||
effort = output_config.get("effort")
|
||||
if effort and isinstance(effort, str):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _get_user_anthropic_beta_headers(
|
||||
self, anthropic_beta_header: Optional[str]
|
||||
) -> Optional[List[str]]:
|
||||
|
|
@ -122,6 +202,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
pdf_used: bool = False,
|
||||
file_id_used: bool = False,
|
||||
mcp_server_used: bool = False,
|
||||
tool_search_used: bool = False,
|
||||
programmatic_tool_calling_used: bool = False,
|
||||
input_examples_used: bool = False,
|
||||
effort_used: bool = False,
|
||||
is_vertex_request: bool = False,
|
||||
user_anthropic_beta_headers: Optional[List[str]] = None,
|
||||
) -> dict:
|
||||
|
|
@ -138,6 +222,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
betas.add("code-execution-2025-05-22")
|
||||
if mcp_server_used:
|
||||
betas.add("mcp-client-2025-04-04")
|
||||
# Tool search, programmatic tool calling, and input_examples all use the same beta header
|
||||
if tool_search_used or programmatic_tool_calling_used or input_examples_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
|
||||
betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
|
||||
|
||||
# Effort parameter uses a separate beta header
|
||||
if effort_used:
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER
|
||||
betas.add(ANTHROPIC_EFFORT_BETA_HEADER)
|
||||
|
||||
headers = {
|
||||
"anthropic-version": anthropic_version or "2023-06-01",
|
||||
|
|
@ -182,6 +275,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
)
|
||||
pdf_used = self.is_pdf_used(messages=messages)
|
||||
file_id_used = self.is_file_id_used(messages=messages)
|
||||
tool_search_used = self.is_tool_search_used(tools=tools)
|
||||
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
|
||||
input_examples_used = self.is_input_examples_used(tools=tools)
|
||||
effort_used = self.is_effort_used(optional_params=optional_params)
|
||||
user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
|
||||
anthropic_beta_header=headers.get("anthropic-beta")
|
||||
)
|
||||
|
|
@ -194,6 +291,10 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
is_vertex_request=optional_params.get("is_vertex_request", False),
|
||||
user_anthropic_beta_headers=user_anthropic_beta_headers,
|
||||
mcp_server_used=mcp_server_used,
|
||||
tool_search_used=tool_search_used,
|
||||
programmatic_tool_calling_used=programmatic_tool_calling_used,
|
||||
input_examples_used=input_examples_used,
|
||||
effort_used=effort_used,
|
||||
)
|
||||
|
||||
headers = {**headers, **anthropic_headers}
|
||||
|
|
|
|||
|
|
@ -645,7 +645,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
type="tool_use",
|
||||
id=choice.delta.tool_calls[0].id or str(uuid.uuid4()),
|
||||
name=choice.delta.tool_calls[0].function.name or "",
|
||||
input={},
|
||||
input={}, # type: ignore[typeddict-item]
|
||||
)
|
||||
elif isinstance(choice, StreamingChoices) and hasattr(
|
||||
choice.delta, "thinking_blocks"
|
||||
|
|
|
|||
|
|
@ -36,12 +36,20 @@ class AnthropicOutputSchema(TypedDict, total=False):
|
|||
schema: Required[dict]
|
||||
|
||||
|
||||
class AnthropicOutputConfig(TypedDict, total=False):
|
||||
"""Configuration for controlling Claude's output behavior."""
|
||||
effort: Literal["high", "medium", "low"]
|
||||
|
||||
|
||||
class AnthropicMessagesTool(TypedDict, total=False):
|
||||
name: Required[str]
|
||||
description: str
|
||||
input_schema: Optional[AnthropicInputSchema]
|
||||
type: Literal["custom"]
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
defer_loading: bool
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AnthropicComputerTool(TypedDict, total=False):
|
||||
|
|
@ -67,24 +75,78 @@ class AnthropicWebSearchTool(TypedDict, total=False):
|
|||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
max_uses: Optional[int]
|
||||
user_location: Optional[AnthropicWebSearchUserLocation]
|
||||
defer_loading: Optional[bool]
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_editor
|
||||
type: Required[str]
|
||||
name: Required[str]
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
defer_loading: Optional[bool]
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AnthropicCodeExecutionTool(TypedDict, total=False):
|
||||
type: Required[str]
|
||||
name: Required[Literal["code_execution"]]
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
defer_loading: Optional[bool]
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AnthropicMemoryTool(TypedDict, total=False):
|
||||
type: Required[str]
|
||||
name: Required[Literal["memory"]]
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
defer_loading: Optional[bool]
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class AnthropicToolSearchToolRegex(TypedDict, total=False):
|
||||
"""Tool search tool using regex patterns for tool discovery."""
|
||||
type: Required[Literal["tool_search_tool_regex_20251119"]]
|
||||
name: Required[str]
|
||||
|
||||
|
||||
class AnthropicToolSearchToolBM25(TypedDict, total=False):
|
||||
"""Tool search tool using BM25 algorithm for tool discovery."""
|
||||
type: Required[Literal["tool_search_tool_bm25_20251119"]]
|
||||
name: Required[str]
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
defer_loading: Optional[bool]
|
||||
allowed_callers: Optional[List[str]]
|
||||
input_examples: Optional[List[Dict[str, Any]]]
|
||||
|
||||
|
||||
class ToolReference(TypedDict, total=False):
|
||||
"""Reference to a tool that should be expanded from deferred tools."""
|
||||
type: Required[Literal["tool_reference"]]
|
||||
tool_name: Required[str]
|
||||
|
||||
|
||||
class DirectToolCaller(TypedDict, total=False):
|
||||
"""Indicates a tool was called directly by Claude."""
|
||||
type: Required[Literal["direct"]]
|
||||
|
||||
|
||||
class CodeExecutionToolCaller(TypedDict, total=False):
|
||||
"""Indicates a tool was called programmatically from code execution."""
|
||||
type: Required[Literal["code_execution_20250825"]]
|
||||
tool_id: Required[str] # ID of the code execution tool that made the call
|
||||
|
||||
|
||||
ToolCaller = Union[DirectToolCaller, CodeExecutionToolCaller]
|
||||
|
||||
|
||||
class AnthropicContainer(TypedDict, total=False):
|
||||
"""Container metadata for code execution."""
|
||||
id: Required[str]
|
||||
expires_at: Optional[str] # ISO 8601 timestamp
|
||||
|
||||
|
||||
AllAnthropicToolsValues = Union[
|
||||
|
|
@ -94,6 +156,8 @@ AllAnthropicToolsValues = Union[
|
|||
AnthropicWebSearchTool,
|
||||
AnthropicCodeExecutionTool,
|
||||
AnthropicMemoryTool,
|
||||
AnthropicToolSearchToolRegex,
|
||||
AnthropicToolSearchToolBM25,
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -121,6 +185,7 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False):
|
|||
name: str
|
||||
input: dict
|
||||
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]
|
||||
caller: Optional[ToolCaller]
|
||||
|
||||
|
||||
AnthropicMessagesAssistantMessageValues = Union[
|
||||
|
|
@ -372,6 +437,7 @@ class ToolUseBlock(TypedDict):
|
|||
name: str
|
||||
|
||||
type: Literal["tool_use"]
|
||||
caller: Optional[ToolCaller]
|
||||
|
||||
|
||||
class TextBlock(TypedDict):
|
||||
|
|
@ -565,3 +631,11 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
|
|||
WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10"
|
||||
CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27"
|
||||
STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13"
|
||||
ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
|
||||
|
||||
|
||||
# Tool search beta header constant
|
||||
ANTHROPIC_TOOL_SEARCH_BETA_HEADER = "advanced-tool-use-2025-11-20"
|
||||
|
||||
# Effort beta header constant
|
||||
ANTHROPIC_EFFORT_BETA_HEADER = "effort-2025-11-24"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,17 @@
|
|||
from enum import Enum
|
||||
from os import PathLike
|
||||
from typing import IO, Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Union
|
||||
from typing import (
|
||||
IO,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
import httpx
|
||||
from openai._legacy_response import (
|
||||
|
|
|
|||
|
|
@ -999,7 +999,8 @@ class PromptTokensDetailsWrapper(
|
|||
|
||||
|
||||
class ServerToolUse(BaseModel):
|
||||
web_search_requests: Optional[int]
|
||||
web_search_requests: Optional[int] = None
|
||||
tool_search_requests: Optional[int] = None
|
||||
|
||||
|
||||
class Usage(CompletionUsage):
|
||||
|
|
|
|||
|
|
@ -556,3 +556,744 @@ def test_anthropic_structured_output_beta_header():
|
|||
"structured-outputs-2025-11-13"
|
||||
in response["raw_request_headers"]["anthropic-beta"]
|
||||
)
|
||||
|
||||
|
||||
# ============ Tool Search Tests ============
|
||||
|
||||
|
||||
def test_tool_search_regex_detection():
|
||||
"""Test that tool search regex tools are properly detected"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
config = AnthropicModelInfo()
|
||||
|
||||
# Test with tool search regex tool
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
}
|
||||
]
|
||||
assert config.is_tool_search_used(tools) is True
|
||||
|
||||
# Test without tool search
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"}
|
||||
}
|
||||
]
|
||||
assert config.is_tool_search_used(tools) is False
|
||||
|
||||
|
||||
def test_tool_search_bm25_detection():
|
||||
"""Test that tool search BM25 tools are properly detected"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
config = AnthropicModelInfo()
|
||||
|
||||
# Test with tool search BM25 tool
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
}
|
||||
]
|
||||
assert config.is_tool_search_used(tools) is True
|
||||
|
||||
|
||||
def test_tool_search_beta_header():
|
||||
"""Test that tool search beta header is automatically added"""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
config = AnthropicModelInfo()
|
||||
|
||||
headers = config.get_anthropic_headers(
|
||||
api_key="test-key",
|
||||
tool_search_used=True,
|
||||
)
|
||||
|
||||
assert "anthropic-beta" in headers
|
||||
assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"]
|
||||
|
||||
|
||||
def test_tool_search_regex_mapping():
|
||||
"""Test that tool search regex tools are properly mapped"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
tool = {
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
}
|
||||
|
||||
mapped_tool, mcp_server = config._map_tool_helper(tool)
|
||||
|
||||
assert mapped_tool is not None
|
||||
assert mapped_tool["type"] == "tool_search_tool_regex_20251119"
|
||||
assert mapped_tool["name"] == "tool_search_tool_regex"
|
||||
assert mcp_server is None
|
||||
|
||||
|
||||
def test_tool_search_bm25_mapping():
|
||||
"""Test that tool search BM25 tools are properly mapped"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
tool = {
|
||||
"type": "tool_search_tool_bm25_20251119",
|
||||
"name": "tool_search_tool_bm25"
|
||||
}
|
||||
|
||||
mapped_tool, mcp_server = config._map_tool_helper(tool)
|
||||
|
||||
assert mapped_tool is not None
|
||||
assert mapped_tool["type"] == "tool_search_tool_bm25_20251119"
|
||||
assert mapped_tool["name"] == "tool_search_tool_bm25"
|
||||
assert mcp_server is None
|
||||
|
||||
|
||||
def test_deferred_tools_separation():
|
||||
"""Test that deferred and non-deferred tools are properly separated"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "tool_search_tool_regex_20251119",
|
||||
"name": "tool_search_tool_regex"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
"defer_loading": True
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "search_files"},
|
||||
"defer_loading": False
|
||||
}
|
||||
]
|
||||
|
||||
non_deferred, deferred = config._separate_deferred_tools(tools)
|
||||
|
||||
assert len(non_deferred) == 2 # tool_search and search_files
|
||||
assert len(deferred) == 1 # get_weather
|
||||
|
||||
|
||||
def test_server_tool_use_in_response():
|
||||
"""Test that server_tool_use blocks are parsed correctly"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
completion_response = {
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01ABC123",
|
||||
"name": "tool_search_tool_regex",
|
||||
"input": {"query": "weather"}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content(
|
||||
completion_response
|
||||
)
|
||||
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["id"] == "srvtoolu_01ABC123"
|
||||
assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex"
|
||||
|
||||
|
||||
def test_tool_search_usage_tracking():
|
||||
"""Test that tool_search_requests are tracked in usage"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
usage_object = {
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
"server_tool_use": {
|
||||
"tool_search_requests": 2
|
||||
}
|
||||
}
|
||||
|
||||
usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None)
|
||||
|
||||
assert usage.server_tool_use is not None
|
||||
assert usage.server_tool_use.tool_search_requests == 2
|
||||
|
||||
|
||||
def test_tool_reference_expansion():
|
||||
"""Test that tool_reference blocks are expanded correctly"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
deferred_tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
content = [
|
||||
{"type": "text", "text": "I'll search for tools"},
|
||||
{"type": "tool_reference", "tool_name": "get_weather"}
|
||||
]
|
||||
|
||||
expanded = config._expand_tool_references(content, deferred_tools)
|
||||
|
||||
assert len(expanded) == 2
|
||||
assert expanded[0]["type"] == "text"
|
||||
assert expanded[1]["type"] == "function"
|
||||
assert expanded[1]["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_defer_loading_preserved_in_transformation():
|
||||
"""Test that defer_loading parameter is preserved when transforming tools"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"defer_loading": True
|
||||
}
|
||||
|
||||
mapped_tool, mcp_server = config._map_tool_helper(tool)
|
||||
|
||||
assert mapped_tool is not None
|
||||
assert mapped_tool.get("defer_loading") is True
|
||||
assert mapped_tool["name"] == "get_weather"
|
||||
assert mcp_server is None
|
||||
|
||||
|
||||
def test_tool_search_complete_response_parsing():
|
||||
"""Test parsing a complete tool search response with server_tool_use and tool_search_tool_result blocks"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Simulating actual Anthropic API response with tool search
|
||||
completion_response = {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll search for weather-related tools that can help you."
|
||||
},
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ",
|
||||
"name": "tool_search_tool_regex",
|
||||
"input": {"pattern": "weather", "limit": 5},
|
||||
"caller": {"type": "direct"}
|
||||
},
|
||||
{
|
||||
"type": "tool_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ",
|
||||
"content": {
|
||||
"type": "tool_search_tool_search_result",
|
||||
"tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Great! I found a weather tool."
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ",
|
||||
"name": "get_weather",
|
||||
"input": {"location": "San Francisco"}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 1639,
|
||||
"output_tokens": 170,
|
||||
"server_tool_use": {"web_search_requests": 0}
|
||||
}
|
||||
}
|
||||
|
||||
# Extract content
|
||||
text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content(
|
||||
completion_response
|
||||
)
|
||||
|
||||
# Verify text extraction (should concatenate both text blocks)
|
||||
assert "I'll search for weather-related tools" in text
|
||||
assert "Great! I found a weather tool" in text
|
||||
|
||||
# Verify tool calls (should have both server_tool_use and tool_use)
|
||||
assert len(tool_calls) == 2
|
||||
assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex"
|
||||
assert tool_calls[1]["function"]["name"] == "get_weather"
|
||||
|
||||
# Verify usage calculation counts tool_search_requests from content
|
||||
usage = config.calculate_usage(
|
||||
usage_object=completion_response["usage"],
|
||||
reasoning_content=None,
|
||||
completion_response=completion_response
|
||||
)
|
||||
|
||||
assert usage.server_tool_use is not None
|
||||
assert usage.server_tool_use.web_search_requests == 0
|
||||
assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks
|
||||
|
||||
|
||||
def test_allowed_callers_field_preservation():
|
||||
"""Test that allowed_callers field is preserved during tool transformation."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Test with top-level allowed_callers
|
||||
tool_with_allowed_callers = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {"type": "string"}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers)
|
||||
assert transformed_tool is not None
|
||||
assert "allowed_callers" in transformed_tool
|
||||
assert transformed_tool["allowed_callers"] == ["code_execution_20250825"]
|
||||
|
||||
|
||||
def test_programmatic_tool_calling_beta_header():
|
||||
"""Test that beta header is automatically added when programmatic tool calling is detected."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
model_info = AnthropicModelInfo()
|
||||
|
||||
# Test detection with allowed_callers
|
||||
tools = [
|
||||
{
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
]
|
||||
|
||||
is_programmatic = model_info.is_programmatic_tool_calling_used(tools)
|
||||
assert is_programmatic is True
|
||||
|
||||
# Test header generation
|
||||
headers = model_info.get_anthropic_headers(
|
||||
api_key="test-key",
|
||||
programmatic_tool_calling_used=True
|
||||
)
|
||||
|
||||
assert "anthropic-beta" in headers
|
||||
assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"]
|
||||
|
||||
|
||||
def test_caller_field_in_response():
|
||||
"""Test that caller field is correctly parsed from tool_use blocks."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Mock response with programmatic tool call
|
||||
completion_response = {
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll query the database."
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_123",
|
||||
"name": "query_database",
|
||||
"input": {"sql": "SELECT * FROM users"},
|
||||
"caller": {
|
||||
"type": "code_execution_20250825",
|
||||
"tool_id": "srvtoolu_abc"
|
||||
}
|
||||
}
|
||||
],
|
||||
"stop_reason": "tool_use",
|
||||
"usage": {"input_tokens": 100, "output_tokens": 50}
|
||||
}
|
||||
|
||||
text, citations, thinking, reasoning, tool_calls = config.extract_response_content(completion_response)
|
||||
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["id"] == "toolu_123"
|
||||
assert tool_calls[0]["function"]["name"] == "query_database"
|
||||
assert "caller" in tool_calls[0]
|
||||
assert tool_calls[0]["caller"]["type"] == "code_execution_20250825"
|
||||
assert tool_calls[0]["caller"]["tool_id"] == "srvtoolu_abc"
|
||||
|
||||
|
||||
def test_code_execution_20250825_tool_type():
|
||||
"""Test that code_execution_20250825 tool type is handled correctly."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
tool = {
|
||||
"type": "code_execution_20250825",
|
||||
"name": "code_execution"
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool)
|
||||
assert transformed_tool is not None
|
||||
assert transformed_tool["type"] == "code_execution_20250825"
|
||||
assert transformed_tool["name"] == "code_execution"
|
||||
|
||||
|
||||
def test_allowed_callers_in_function_field():
|
||||
"""Test that allowed_callers in function field is also preserved."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Test with function.allowed_callers
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {"type": "string"}
|
||||
},
|
||||
"required": ["sql"]
|
||||
},
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool)
|
||||
assert transformed_tool is not None
|
||||
assert "allowed_callers" in transformed_tool
|
||||
assert transformed_tool["allowed_callers"] == ["code_execution_20250825"]
|
||||
|
||||
|
||||
def test_input_examples_field_preservation():
|
||||
"""Test that input_examples field is preserved during tool transformation."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Test with top-level input_examples
|
||||
tool_with_examples = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{"location": "San Francisco, CA", "unit": "fahrenheit"},
|
||||
{"location": "Tokyo, Japan", "unit": "celsius"}
|
||||
]
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool_with_examples)
|
||||
assert transformed_tool is not None
|
||||
assert "input_examples" in transformed_tool
|
||||
assert len(transformed_tool["input_examples"]) == 2
|
||||
assert transformed_tool["input_examples"][0]["location"] == "San Francisco, CA"
|
||||
|
||||
|
||||
def test_input_examples_beta_header():
|
||||
"""Test that beta header is automatically added when input_examples is detected."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
model_info = AnthropicModelInfo()
|
||||
|
||||
# Test detection with input_examples
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {"type": "object", "properties": {}}
|
||||
},
|
||||
"input_examples": [
|
||||
{"location": "San Francisco, CA"}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
is_examples_used = model_info.is_input_examples_used(tools)
|
||||
assert is_examples_used is True
|
||||
|
||||
# Test header generation
|
||||
headers = model_info.get_anthropic_headers(
|
||||
api_key="test-key",
|
||||
input_examples_used=True
|
||||
)
|
||||
|
||||
assert "anthropic-beta" in headers
|
||||
assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"]
|
||||
|
||||
|
||||
def test_input_examples_in_function_field():
|
||||
"""Test that input_examples in function field is also preserved."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Test with function.input_examples
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
},
|
||||
"input_examples": [
|
||||
{"location": "Paris, France"},
|
||||
{"location": "London, UK"}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool)
|
||||
assert transformed_tool is not None
|
||||
assert "input_examples" in transformed_tool
|
||||
assert len(transformed_tool["input_examples"]) == 2
|
||||
|
||||
|
||||
def test_input_examples_with_other_features():
|
||||
"""Test that input_examples works alongside other tool features."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Tool with input_examples, defer_loading, and allowed_callers
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "query_database",
|
||||
"description": "Execute a SQL query",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sql": {"type": "string"}
|
||||
},
|
||||
"required": ["sql"]
|
||||
}
|
||||
},
|
||||
"input_examples": [
|
||||
{"sql": "SELECT * FROM users WHERE id = 1"}
|
||||
],
|
||||
"defer_loading": True,
|
||||
"allowed_callers": ["code_execution_20250825"]
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool)
|
||||
assert transformed_tool is not None
|
||||
assert "input_examples" in transformed_tool
|
||||
assert "defer_loading" in transformed_tool
|
||||
assert "allowed_callers" in transformed_tool
|
||||
assert transformed_tool["defer_loading"] is True
|
||||
assert transformed_tool["allowed_callers"] == ["code_execution_20250825"]
|
||||
|
||||
|
||||
def test_input_examples_empty_list_not_added():
|
||||
"""Test that empty input_examples list is not added to transformed tool."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
# Tool with empty input_examples
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
},
|
||||
"input_examples": []
|
||||
}
|
||||
|
||||
transformed_tool, _ = config._map_tool_helper(tool)
|
||||
assert transformed_tool is not None
|
||||
# Empty list should not be added
|
||||
assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0
|
||||
|
||||
|
||||
# ============ Effort Parameter Tests ============
|
||||
|
||||
|
||||
def test_effort_output_config_preservation():
|
||||
"""Test that output_config with effort is preserved in transformation."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
messages = [{"role": "user", "content": "Analyze this code"}]
|
||||
optional_params = {
|
||||
"output_config": {
|
||||
"effort": "medium"
|
||||
}
|
||||
}
|
||||
|
||||
result = config.transform_request(
|
||||
model="claude-opus-4-5-20251101",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
|
||||
assert "output_config" in result
|
||||
assert result["output_config"]["effort"] == "medium"
|
||||
|
||||
|
||||
def test_effort_beta_header_injection():
|
||||
"""Test that effort beta header is automatically added when output_config is detected."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
model_info = AnthropicModelInfo()
|
||||
|
||||
# Test with effort parameter
|
||||
optional_params = {
|
||||
"output_config": {
|
||||
"effort": "low"
|
||||
}
|
||||
}
|
||||
|
||||
effort_used = model_info.is_effort_used(optional_params=optional_params)
|
||||
assert effort_used is True
|
||||
|
||||
headers = model_info.get_anthropic_headers(
|
||||
api_key="test-key",
|
||||
effort_used=effort_used
|
||||
)
|
||||
|
||||
assert "anthropic-beta" in headers
|
||||
assert "effort-2025-11-24" in headers["anthropic-beta"]
|
||||
|
||||
|
||||
def test_effort_validation():
|
||||
"""Test that only valid effort values are accepted."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
messages = [{"role": "user", "content": "Test"}]
|
||||
|
||||
# Valid values should work
|
||||
for effort in ["high", "medium", "low"]:
|
||||
optional_params = {"output_config": {"effort": effort}}
|
||||
result = config.transform_request(
|
||||
model="claude-opus-4-5-20251101",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
assert result["output_config"]["effort"] == effort
|
||||
|
||||
# Invalid value should raise error
|
||||
with pytest.raises(ValueError, match="Invalid effort value"):
|
||||
optional_params = {"output_config": {"effort": "invalid"}}
|
||||
config.transform_request(
|
||||
model="claude-opus-4-5-20251101",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
|
||||
|
||||
def test_effort_with_claude_opus_45():
|
||||
"""Test effort parameter works with Claude Opus 4.5 model."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
messages = [{"role": "user", "content": "Complex analysis task"}]
|
||||
optional_params = {
|
||||
"output_config": {
|
||||
"effort": "high"
|
||||
}
|
||||
}
|
||||
|
||||
result = config.transform_request(
|
||||
model="claude-opus-4-5-20251101",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
|
||||
assert "output_config" in result
|
||||
assert result["output_config"]["effort"] == "high"
|
||||
assert result["model"] == "claude-opus-4-5-20251101"
|
||||
|
||||
|
||||
def test_effort_with_other_features():
|
||||
"""Test effort works alongside other features (thinking, tools)."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
messages = [{"role": "user", "content": "Use tools efficiently"}]
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_data",
|
||||
"description": "Get data",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
optional_params = {
|
||||
"output_config": {
|
||||
"effort": "low"
|
||||
},
|
||||
"tools": tools,
|
||||
"thinking": {
|
||||
"type": "enabled",
|
||||
"budget_tokens": 1000
|
||||
}
|
||||
}
|
||||
|
||||
result = config.transform_request(
|
||||
model="claude-opus-4-5-20251101",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={}
|
||||
)
|
||||
|
||||
# Verify all features are present
|
||||
assert "output_config" in result
|
||||
assert result["output_config"]["effort"] == "low"
|
||||
assert "tools" in result
|
||||
assert len(result["tools"]) > 0
|
||||
assert "thinking" in result
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue