mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge branch 'main' into litellm_add_anthropic_tool_call_results
This commit is contained in:
Sameer Kankute
GitHub
commit
ec3e30a221
33 changed files with 1549 additions and 241 deletions
29
.github/workflows/ghcr_deploy.yml
vendored
29
.github/workflows/ghcr_deploy.yml
vendored
|
|
@ -5,6 +5,7 @@ on:
|
|||
inputs:
|
||||
tag:
|
||||
description: "The tag version you want to build"
|
||||
required: true
|
||||
release_type:
|
||||
description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
|
||||
type: string
|
||||
|
|
@ -336,9 +337,9 @@ jobs:
|
|||
run: |
|
||||
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
|
||||
if [ -z "${CHART_LIST}" ]; then
|
||||
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
|
||||
echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
|
||||
else
|
||||
# Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
|
||||
# Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
|
||||
VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
|
||||
echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
fi
|
||||
|
|
@ -350,28 +351,42 @@ jobs:
|
|||
id: bump_version
|
||||
uses: christian-draeger/increment-semantic-version@1.1.0
|
||||
with:
|
||||
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
|
||||
current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
|
||||
version-fragment: 'bug'
|
||||
|
||||
# Add suffix for non-stable releases (semantic versioning)
|
||||
- name: Calculate chart version with prerelease suffix
|
||||
- name: Calculate chart and app versions
|
||||
id: chart_version
|
||||
shell: bash
|
||||
run: |
|
||||
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
|
||||
BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
|
||||
RELEASE_TYPE="${{ github.event.inputs.release_type }}"
|
||||
INPUT_TAG="${{ github.event.inputs.tag }}"
|
||||
|
||||
# Chart version (independent Helm chart versioning with release type suffix)
|
||||
if [ "$RELEASE_TYPE" = "stable" ]; then
|
||||
echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
else
|
||||
echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# App version (must match Docker tags)
|
||||
# stable/rc releases: Docker creates main-{tag}, so use the tag
|
||||
# latest/dev releases: Docker only creates main-{release_type}, so use release_type
|
||||
if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
|
||||
APP_VERSION="${INPUT_TAG}"
|
||||
else
|
||||
APP_VERSION="${RELEASE_TYPE}"
|
||||
fi
|
||||
|
||||
echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
- uses: ./.github/actions/helm-oci-chart-releaser
|
||||
with:
|
||||
name: ${{ env.CHART_NAME }}
|
||||
repository: ${{ env.REPO_OWNER }}
|
||||
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
|
||||
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
|
||||
tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
|
||||
app_version: ${{ steps.chart_version.outputs.app_version }}
|
||||
path: deploy/charts/${{ env.CHART_NAME }}
|
||||
registry: ${{ env.REGISTRY }}
|
||||
registry_username: ${{ github.actor }}
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ type: application
|
|||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.10
|
||||
version: 1.0.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: v1.50.2
|
||||
appVersion: v1.80.12
|
||||
|
||||
dependencies:
|
||||
- name: "postgresql"
|
||||
|
|
|
|||
|
|
@ -142,7 +142,47 @@ def completion(
|
|||
- `tool_call_id`: *str (optional)* - Tool call that this message is responding to.
|
||||
|
||||
|
||||
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/8600ec77042dacad324d3879a2bd918fc6a719fa/litellm/types/llms/openai.py#L392)
|
||||
[**See All Message Values**](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L664)
|
||||
|
||||
#### Content Types
|
||||
|
||||
`content` can be a string (text only) or a list of content blocks (multimodal):
|
||||
|
||||
| Type | Description | Docs |
|
||||
|------|-------------|------|
|
||||
| `text` | Text content | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L598) |
|
||||
| `image_url` | Images | [Vision](./vision.md) |
|
||||
| `input_audio` | Audio input | [Audio](./audio.md) |
|
||||
| `video_url` | Video input | [Type Definition](https://github.com/BerriAI/litellm/blob/main/litellm/types/llms/openai.py#L625) |
|
||||
| `file` | Files | [Document Understanding](./document_understanding.md) |
|
||||
| `document` | Documents/PDFs | [Document Understanding](./document_understanding.md) |
|
||||
|
||||
**Examples:**
|
||||
```python
|
||||
# Text
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}]
|
||||
|
||||
# Image
|
||||
messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}]}]
|
||||
|
||||
# Audio
|
||||
messages=[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "<base64>", "format": "wav"}}]}]
|
||||
|
||||
# Video
|
||||
messages=[{"role": "user", "content": [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]}]
|
||||
|
||||
# File
|
||||
messages=[{"role": "user", "content": [{"type": "file", "file": {"file_id": "https://example.com/doc.pdf"}}]}]
|
||||
|
||||
# Document
|
||||
messages=[{"role": "user", "content": [{"type": "document", "source": {"type": "text", "media_type": "application/pdf", "data": "<base64>"}}]}]
|
||||
|
||||
# Combining multiple types (multimodal)
|
||||
messages=[{"role": "user", "content": [
|
||||
{"type": "text", "text": "Generate a product description based on this image"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}
|
||||
]}]
|
||||
```
|
||||
|
||||
## Optional Fields
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
|
|||
|
||||
# High Availability Setup (Resolve DB Deadlocks)
|
||||
|
||||
:::tip Essential for Production
|
||||
|
||||
This configuration is **required** for production deployments handling 1000+ requests per second. Without Redis configured, you may experience PostgreSQL connection exhaustion (`FATAL: sorry, too many clients already`).
|
||||
|
||||
:::
|
||||
|
||||
Resolve any Database Deadlocks you see in high traffic by using this setup
|
||||
|
||||
## What causes the problem?
|
||||
|
|
|
|||
|
|
@ -359,6 +359,26 @@ LiteLLM is compatible with several SDKs - including OpenAI SDK, Anthropic SDK, M
|
|||
### Deploy with Database
|
||||
##### Docker, Kubernetes, Helm Chart
|
||||
|
||||
:::warning High Traffic Deployments (1000+ RPS)
|
||||
|
||||
If you expect high traffic (1000+ requests per second), **Redis is required** to prevent database connection exhaustion and deadlocks.
|
||||
|
||||
Add this to your config:
|
||||
```yaml
|
||||
general_settings:
|
||||
use_redis_transaction_buffer: true
|
||||
|
||||
litellm_settings:
|
||||
cache: true
|
||||
cache_params:
|
||||
type: redis
|
||||
host: your-redis-host
|
||||
```
|
||||
|
||||
See [Resolve DB Deadlocks](/docs/proxy/db_deadlocks) for details.
|
||||
|
||||
:::
|
||||
|
||||
Requirements:
|
||||
- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
|
||||
- Set a `LITELLM_MASTER_KEY`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`)
|
||||
|
|
|
|||
|
|
@ -264,8 +264,15 @@ model_list:
|
|||
model: azure/gpt-4-fallback
|
||||
api_key: os.environ/AZURE_API_KEY_2
|
||||
order: 2 # 👈 Used when order=1 is unavailable
|
||||
|
||||
router_settings:
|
||||
enable_pre_call_checks: true # 👈 Required for 'order' to work
|
||||
```
|
||||
|
||||
:::important
|
||||
The `order` parameter requires `enable_pre_call_checks: true` in `router_settings`.
|
||||
:::
|
||||
|
||||
If `order=1` deployment is unavailable (e.g., rate-limited), the router falls back to `order=2` deployments.
|
||||
|
||||
### When You'll See Load Balancing in Action
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ general_settings:
|
|||
target: string # Target URL for forwarding
|
||||
auth: boolean # Enable LiteLLM authentication (Enterprise)
|
||||
forward_headers: boolean # Forward all incoming headers
|
||||
include_subpath: boolean # If true, forwards requests to sub-paths (default: false)
|
||||
headers: # Custom headers to add
|
||||
Authorization: string # Auth header for target API
|
||||
content-type: string # Request content type
|
||||
|
|
@ -181,6 +182,23 @@ general_settings:
|
|||
- **LANGFUSE_PUBLIC_KEY/SECRET_KEY**: For Langfuse integration
|
||||
- **Custom headers**: Any additional key-value pairs
|
||||
|
||||
### Sub-path Routing
|
||||
|
||||
By default, pass-through endpoints only match the **exact path** specified. To forward requests to sub-paths, set `include_subpath: true`:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
pass_through_endpoints:
|
||||
- path: "/custom-api" # Any path prefix you choose
|
||||
target: "https://api.example.com"
|
||||
include_subpath: true # Forward /custom-api/*, not just /custom-api
|
||||
```
|
||||
|
||||
| Setting | Behavior |
|
||||
|---------|----------|
|
||||
| `include_subpath: false` (default) | Only `/custom-api` is forwarded |
|
||||
| `include_subpath: true` | `/custom-api`, `/custom-api/v1/chat`, `/custom-api/anything` are all forwarded |
|
||||
|
||||
---
|
||||
|
||||
## Advanced: Custom Adapters
|
||||
|
|
|
|||
|
|
@ -861,9 +861,13 @@ model_list = [
|
|||
},
|
||||
]
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
router = Router(model_list=model_list, enable_pre_call_checks=True) # 👈 Required for 'order' to work
|
||||
```
|
||||
|
||||
:::important
|
||||
The `order` parameter requires `enable_pre_call_checks=True` to be set on the Router.
|
||||
:::
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
|
|
@ -880,6 +884,9 @@ model_list:
|
|||
model: azure/gpt-4-fallback
|
||||
api_key: os.environ/AZURE_API_KEY_2
|
||||
order: 2 # 👈 Used when order=1 is unavailable
|
||||
|
||||
router_settings:
|
||||
enable_pre_call_checks: true # 👈 Required for 'order' to work
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
|
|
|||
|
|
@ -37,9 +37,14 @@ class GenerateContentToCompletionHandler:
|
|||
|
||||
completion_kwargs: Dict[str, Any] = dict(completion_request)
|
||||
|
||||
# feed metadata for custom callback
|
||||
if extra_kwargs is not None and "metadata" in extra_kwargs:
|
||||
completion_kwargs["metadata"] = extra_kwargs["metadata"]
|
||||
# Forward extra_kwargs that should be passed to completion call
|
||||
if extra_kwargs is not None:
|
||||
# Forward metadata for custom callback
|
||||
if "metadata" in extra_kwargs:
|
||||
completion_kwargs["metadata"] = extra_kwargs["metadata"]
|
||||
# Forward extra_headers for providers that require custom headers (e.g., github_copilot)
|
||||
if "extra_headers" in extra_kwargs:
|
||||
completion_kwargs["extra_headers"] = extra_kwargs["extra_headers"]
|
||||
|
||||
if stream:
|
||||
completion_kwargs["stream"] = stream
|
||||
|
|
|
|||
|
|
@ -330,6 +330,7 @@ def generate_content(
|
|||
tools=tools,
|
||||
_is_async=_is_async,
|
||||
litellm_params=setup_result.litellm_params,
|
||||
extra_headers=extra_headers,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
|
@ -422,6 +423,7 @@ async def agenerate_content_stream(
|
|||
litellm_params=setup_result.litellm_params,
|
||||
tools=tools,
|
||||
stream=True,
|
||||
extra_headers=extra_headers,
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
|
|
@ -507,6 +509,7 @@ def generate_content_stream(
|
|||
_is_async=_is_async,
|
||||
litellm_params=setup_result.litellm_params,
|
||||
stream=True,
|
||||
extra_headers=extra_headers,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -95,7 +95,9 @@ def handle_messages_with_content_list_to_str_conversion(
|
|||
return messages
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[str] = ["user"]) -> AllMessageValues:
|
||||
def strip_name_from_message(
|
||||
message: AllMessageValues, allowed_name_roles: List[str] = ["user"]
|
||||
) -> AllMessageValues:
|
||||
"""
|
||||
Removes 'name' from message
|
||||
"""
|
||||
|
|
@ -104,6 +106,7 @@ def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[
|
|||
msg_copy.pop("name", None) # type: ignore
|
||||
return msg_copy
|
||||
|
||||
|
||||
def strip_name_from_messages(
|
||||
messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"]
|
||||
) -> List[AllMessageValues]:
|
||||
|
|
@ -444,7 +447,7 @@ def update_responses_input_with_model_file_ids(
|
|||
"""
|
||||
Updates responses API input with provider-specific file IDs.
|
||||
File IDs are always inside the content array, not as direct input_file items.
|
||||
|
||||
|
||||
For managed files (unified file IDs), decodes the base64-encoded unified file ID
|
||||
and extracts the llm_output_file_id directly.
|
||||
"""
|
||||
|
|
@ -452,25 +455,28 @@ def update_responses_input_with_model_file_ids(
|
|||
_is_base64_encoded_unified_file_id,
|
||||
convert_b64_uid_to_unified_uid,
|
||||
)
|
||||
|
||||
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
|
||||
|
||||
if not isinstance(input, list):
|
||||
return input
|
||||
|
||||
|
||||
updated_input = []
|
||||
for item in input:
|
||||
if not isinstance(item, dict):
|
||||
updated_input.append(item)
|
||||
continue
|
||||
|
||||
|
||||
updated_item = item.copy()
|
||||
content = item.get("content")
|
||||
if isinstance(content, list):
|
||||
updated_content = []
|
||||
for content_item in content:
|
||||
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
|
||||
if (
|
||||
isinstance(content_item, dict)
|
||||
and content_item.get("type") == "input_file"
|
||||
):
|
||||
file_id = content_item.get("file_id")
|
||||
if file_id:
|
||||
# Check if this is a managed file ID (base64-encoded unified file ID)
|
||||
|
|
@ -478,7 +484,9 @@ def update_responses_input_with_model_file_ids(
|
|||
if is_unified_file_id:
|
||||
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
|
||||
if "llm_output_file_id," in unified_file_id:
|
||||
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
|
||||
provider_file_id = unified_file_id.split(
|
||||
"llm_output_file_id,"
|
||||
)[1].split(";")[0]
|
||||
else:
|
||||
# Fallback: keep original if we can't extract
|
||||
provider_file_id = file_id
|
||||
|
|
@ -492,9 +500,9 @@ def update_responses_input_with_model_file_ids(
|
|||
else:
|
||||
updated_content.append(content_item)
|
||||
updated_item["content"] = updated_content
|
||||
|
||||
|
||||
updated_input.append(updated_item)
|
||||
|
||||
|
||||
return updated_input
|
||||
|
||||
|
||||
|
|
@ -697,9 +705,9 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]:
|
|||
video/flv
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
url = url.lower()
|
||||
|
||||
|
||||
# Parse URL to extract path without query parameters
|
||||
# This handles URLs like: https://example.com/image.jpg?signature=...
|
||||
parsed = urlparse(url)
|
||||
|
|
@ -744,28 +752,28 @@ def infer_content_type_from_url_and_content(
|
|||
) -> str:
|
||||
"""
|
||||
Infer content type from URL extension and binary content when content-type header is missing or generic.
|
||||
|
||||
|
||||
This helper implements a fallback strategy for determining MIME types when HTTP headers
|
||||
are missing or provide generic values (like binary/octet-stream). It's commonly used
|
||||
when processing images and documents from various sources (S3, URLs, etc.).
|
||||
|
||||
|
||||
Fallback Strategy:
|
||||
1. If current_content_type is valid (not None and not generic octet-stream), return it
|
||||
2. Try to infer from URL extension (handles query parameters)
|
||||
3. Try to detect from binary content signature (magic bytes)
|
||||
4. Raise ValueError if all methods fail
|
||||
|
||||
|
||||
Args:
|
||||
url: The URL of the content (used to extract file extension)
|
||||
content: The binary content (first ~100 bytes are sufficient for detection)
|
||||
current_content_type: The current content-type from headers (may be None or generic)
|
||||
|
||||
|
||||
Returns:
|
||||
str: The inferred MIME type (e.g., "image/png", "application/pdf")
|
||||
|
||||
|
||||
Raises:
|
||||
ValueError: If content type cannot be determined by any method
|
||||
|
||||
|
||||
Example:
|
||||
>>> content_type = infer_content_type_from_url_and_content(
|
||||
... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123",
|
||||
|
|
@ -776,14 +784,14 @@ def infer_content_type_from_url_and_content(
|
|||
"image/png"
|
||||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if current_content_type and current_content_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return current_content_type
|
||||
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
extension_to_mime = {
|
||||
|
|
@ -804,14 +812,14 @@ def infer_content_type_from_url_and_content(
|
|||
"txt": "text/plain",
|
||||
"md": "text/markdown",
|
||||
}
|
||||
|
||||
|
||||
# Try to infer from URL extension
|
||||
if url:
|
||||
extension = url.split(".")[-1].lower().split("?")[0] # Remove query params
|
||||
inferred_type = extension_to_mime.get(extension)
|
||||
if inferred_type:
|
||||
return inferred_type
|
||||
|
||||
|
||||
# Try to detect from binary content signature (magic bytes)
|
||||
if content:
|
||||
detected_type = get_image_type(content[:100])
|
||||
|
|
@ -825,7 +833,7 @@ def infer_content_type_from_url_and_content(
|
|||
}
|
||||
if detected_type in type_to_mime:
|
||||
return type_to_mime[detected_type]
|
||||
|
||||
|
||||
# If all fallbacks failed, raise error
|
||||
raise ValueError(
|
||||
f"Unable to determine content type from URL: {url}. "
|
||||
|
|
@ -1085,7 +1093,9 @@ def _parse_content_for_reasoning(
|
|||
return None, message_text
|
||||
|
||||
reasoning_match = re.match(
|
||||
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)", message_text, re.DOTALL
|
||||
r"<(?:think|thinking|budget:thinking)>(.*?)</(?:think|thinking|budget:thinking)>(.*)",
|
||||
message_text,
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
if reasoning_match:
|
||||
|
|
@ -1135,3 +1145,47 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
|
|||
elif isinstance(image_url, dict) and "url" in image_url:
|
||||
images.append(_extract_base64_data(image_url["url"]))
|
||||
return images
|
||||
|
||||
|
||||
def parse_tool_call_arguments(
|
||||
arguments: Optional[str],
|
||||
tool_name: Optional[str] = None,
|
||||
context: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse tool call arguments from a JSON string.
|
||||
|
||||
This function handles malformed JSON gracefully by raising a ValueError
|
||||
with context about what failed and what the problematic input was.
|
||||
|
||||
Args:
|
||||
arguments: The JSON string containing tool arguments, or None.
|
||||
tool_name: Optional name of the tool (for error messages).
|
||||
context: Optional context string (e.g., "Anthropic Messages API").
|
||||
|
||||
Returns:
|
||||
Parsed arguments as a dictionary. Returns empty dict if arguments is None or empty.
|
||||
|
||||
Raises:
|
||||
ValueError: If the arguments string is not valid JSON.
|
||||
"""
|
||||
import json
|
||||
|
||||
if not arguments:
|
||||
return {}
|
||||
|
||||
try:
|
||||
return json.loads(arguments)
|
||||
except json.JSONDecodeError as e:
|
||||
error_parts = ["Failed to parse tool call arguments"]
|
||||
|
||||
if tool_name:
|
||||
error_parts.append(f"for tool '{tool_name}'")
|
||||
if context:
|
||||
error_parts.append(f"({context})")
|
||||
|
||||
error_message = (
|
||||
" ".join(error_parts) + f". Error: {str(e)}. Arguments: {arguments}"
|
||||
)
|
||||
|
||||
raise ValueError(error_message) from e
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from .common_utils import (
|
|||
convert_content_list_to_str,
|
||||
infer_content_type_from_url_and_content,
|
||||
is_non_content_values_set,
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
from .image_handling import convert_url_to_base64
|
||||
|
||||
|
|
@ -911,13 +912,13 @@ def convert_to_anthropic_image_obj(
|
|||
|
||||
|
||||
def create_anthropic_image_param(
|
||||
image_url_input: Union[str, dict],
|
||||
image_url_input: Union[str, dict],
|
||||
format: Optional[str] = None,
|
||||
is_bedrock_invoke: bool = False
|
||||
is_bedrock_invoke: bool = False,
|
||||
) -> AnthropicMessagesImageParam:
|
||||
"""
|
||||
Create an AnthropicMessagesImageParam from an image URL input.
|
||||
|
||||
|
||||
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
|
||||
"""
|
||||
# Extract URL and format from input
|
||||
|
|
@ -927,7 +928,7 @@ def create_anthropic_image_param(
|
|||
image_url = image_url_input.get("url", "")
|
||||
if format is None:
|
||||
format = image_url_input.get("format")
|
||||
|
||||
|
||||
# Check if the image URL is an HTTP/HTTPS URL
|
||||
if image_url.startswith("http://") or image_url.startswith("https://"):
|
||||
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
|
||||
|
|
@ -1031,9 +1032,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
|
|||
tool_function = get_attribute_or_key(tool, "function")
|
||||
tool_name = get_attribute_or_key(tool_function, "name")
|
||||
tool_arguments = get_attribute_or_key(tool_function, "arguments")
|
||||
parsed_args = parse_tool_call_arguments(
|
||||
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
|
||||
)
|
||||
parameters = "".join(
|
||||
f"<{param}>{val}</{param}>\n"
|
||||
for param, val in json.loads(tool_arguments).items()
|
||||
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
|
||||
)
|
||||
invokes += (
|
||||
"<invoke>\n"
|
||||
|
|
@ -1071,8 +1074,14 @@ def anthropic_messages_pt_xml(messages: list):
|
|||
if isinstance(messages[msg_i]["content"], list):
|
||||
for m in messages[msg_i]["content"]:
|
||||
if m.get("type", "") == "image_url":
|
||||
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
|
||||
image_param = create_anthropic_image_param(m["image_url"], format=format)
|
||||
format = (
|
||||
m["image_url"].get("format")
|
||||
if isinstance(m["image_url"], dict)
|
||||
else None
|
||||
)
|
||||
image_param = create_anthropic_image_param(
|
||||
m["image_url"], format=format
|
||||
)
|
||||
# Convert to dict format for XML version
|
||||
source = image_param["source"]
|
||||
if isinstance(source, dict) and source.get("type") == "url":
|
||||
|
|
@ -1381,10 +1390,10 @@ def convert_to_gemini_tool_call_invoke(
|
|||
if tool_calls is not None:
|
||||
for idx, tool in enumerate(tool_calls):
|
||||
if "function" in tool:
|
||||
gemini_function_call: Optional[
|
||||
VertexFunctionCall
|
||||
] = _gemini_tool_call_invoke_helper(
|
||||
function_call_params=tool["function"]
|
||||
gemini_function_call: Optional[VertexFunctionCall] = (
|
||||
_gemini_tool_call_invoke_helper(
|
||||
function_call_params=tool["function"]
|
||||
)
|
||||
)
|
||||
if gemini_function_call is not None:
|
||||
part_dict: VertexPartType = {
|
||||
|
|
@ -1484,10 +1493,10 @@ def convert_to_gemini_tool_call_result(
|
|||
}
|
||||
"""
|
||||
from litellm.types.llms.vertex_ai import BlobType
|
||||
|
||||
|
||||
content_str: str = ""
|
||||
inline_data: Optional[BlobType] = None
|
||||
|
||||
|
||||
if "content" in message:
|
||||
if isinstance(message["content"], str):
|
||||
content_str = message["content"]
|
||||
|
|
@ -1500,15 +1509,21 @@ def convert_to_gemini_tool_call_result(
|
|||
elif content_type in ("input_image", "image_url"):
|
||||
# Extract image for inline_data (for Computer Use screenshots and tool results)
|
||||
image_url_data = content.get("image_url", "")
|
||||
image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data
|
||||
|
||||
image_url = (
|
||||
image_url_data.get("url", "")
|
||||
if isinstance(image_url_data, dict)
|
||||
else image_url_data
|
||||
)
|
||||
|
||||
if image_url:
|
||||
# Convert image to base64 blob format for Gemini
|
||||
try:
|
||||
image_obj = convert_to_anthropic_image_obj(image_url, format=None)
|
||||
image_obj = convert_to_anthropic_image_obj(
|
||||
image_url, format=None
|
||||
)
|
||||
inline_data = BlobType(
|
||||
data=image_obj["data"],
|
||||
mime_type=image_obj["media_type"]
|
||||
mime_type=image_obj["media_type"],
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1541,6 +1556,7 @@ def convert_to_gemini_tool_call_result(
|
|||
response_data: dict
|
||||
try:
|
||||
import json
|
||||
|
||||
if content_str.strip().startswith("{") or content_str.strip().startswith("["):
|
||||
# Try to parse as JSON (for Computer Use structured responses)
|
||||
parsed = json.loads(content_str)
|
||||
|
|
@ -1553,7 +1569,7 @@ def convert_to_gemini_tool_call_result(
|
|||
except (json.JSONDecodeError, ValueError):
|
||||
# Not valid JSON, wrap in content field
|
||||
response_data = {"content": content_str}
|
||||
|
||||
|
||||
# We can't determine from openai message format whether it's a successful or
|
||||
# error call result so default to the successful result template
|
||||
_function_response = VertexFunctionResponse(
|
||||
|
|
@ -1562,7 +1578,7 @@ def convert_to_gemini_tool_call_result(
|
|||
|
||||
# Create part with function_response, and optionally inline_data for images (Computer Use)
|
||||
_part: VertexPartType = {"function_response": _function_response}
|
||||
|
||||
|
||||
# For Computer Use, if we have an image, we need separate parts:
|
||||
# - One part with function_response
|
||||
# - One part with inline_data
|
||||
|
|
@ -1570,19 +1586,19 @@ def convert_to_gemini_tool_call_result(
|
|||
if inline_data:
|
||||
image_part: VertexPartType = {"inline_data": inline_data}
|
||||
return [_part, image_part]
|
||||
|
||||
|
||||
return _part
|
||||
|
||||
|
||||
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
|
||||
"""
|
||||
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
|
||||
|
||||
|
||||
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
|
||||
This function replaces any invalid characters with underscores.
|
||||
"""
|
||||
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
|
||||
sanitized = re.sub(r'[^a-zA-Z0-9_-]', '_', tool_use_id)
|
||||
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
|
||||
# Ensure it's not empty (fallback to a default if needed)
|
||||
if not sanitized:
|
||||
sanitized = "tool_use_id"
|
||||
|
|
@ -1644,8 +1660,14 @@ def convert_to_anthropic_tool_result(
|
|||
)
|
||||
)
|
||||
elif content["type"] == "image_url":
|
||||
format = content["image_url"].get("format") if isinstance(content["image_url"], dict) else None
|
||||
_anthropic_image_param = create_anthropic_image_param(content["image_url"], format=format)
|
||||
format = (
|
||||
content["image_url"].get("format")
|
||||
if isinstance(content["image_url"], dict)
|
||||
else None
|
||||
)
|
||||
_anthropic_image_param = create_anthropic_image_param(
|
||||
content["image_url"], format=format
|
||||
)
|
||||
_anthropic_image_param = add_cache_control_to_content(
|
||||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
|
|
@ -1665,7 +1687,9 @@ def convert_to_anthropic_tool_result(
|
|||
# We can't determine from openai message format whether it's a successful or
|
||||
# error call result so default to the successful result template
|
||||
anthropic_tool_result = AnthropicMessagesToolResultParam(
|
||||
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
|
||||
type="tool_result",
|
||||
tool_use_id=sanitized_tool_use_id,
|
||||
content=anthropic_content,
|
||||
)
|
||||
|
||||
if message["role"] == "function":
|
||||
|
|
@ -1674,7 +1698,9 @@ def convert_to_anthropic_tool_result(
|
|||
# Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
|
||||
sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
|
||||
anthropic_tool_result = AnthropicMessagesToolResultParam(
|
||||
type="tool_result", tool_use_id=sanitized_tool_use_id, content=anthropic_content
|
||||
type="tool_result",
|
||||
tool_use_id=sanitized_tool_use_id,
|
||||
content=anthropic_content,
|
||||
)
|
||||
|
||||
if anthropic_tool_result is None:
|
||||
|
|
@ -1690,12 +1716,17 @@ def convert_function_to_anthropic_tool_invoke(
|
|||
try:
|
||||
_name = get_attribute_or_key(function_call, "name") or ""
|
||||
_arguments = get_attribute_or_key(function_call, "arguments")
|
||||
|
||||
tool_input = parse_tool_call_arguments(
|
||||
_arguments, tool_name=_name, context="Anthropic function to tool invoke"
|
||||
)
|
||||
|
||||
anthropic_tool_invoke = [
|
||||
AnthropicMessagesToolUseParam(
|
||||
type="tool_use",
|
||||
id=str(uuid.uuid4()),
|
||||
name=_name,
|
||||
input=json.loads(_arguments) if _arguments else {},
|
||||
input=tool_input,
|
||||
)
|
||||
]
|
||||
return anthropic_tool_invoke
|
||||
|
|
@ -1749,7 +1780,9 @@ def convert_to_anthropic_tool_invoke(
|
|||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
"""
|
||||
anthropic_tool_invoke: List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]] = []
|
||||
anthropic_tool_invoke: List[
|
||||
Union[AnthropicMessagesToolUseParam, Dict[str, Any]]
|
||||
] = []
|
||||
|
||||
for tool in tool_calls:
|
||||
if not get_attribute_or_key(tool, "type") == "function":
|
||||
|
|
@ -1760,10 +1793,10 @@ def convert_to_anthropic_tool_invoke(
|
|||
str,
|
||||
get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
|
||||
)
|
||||
tool_input = json.loads(
|
||||
get_attribute_or_key(
|
||||
get_attribute_or_key(tool, "function"), "arguments"
|
||||
)
|
||||
tool_input = parse_tool_call_arguments(
|
||||
get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"),
|
||||
tool_name=tool_name,
|
||||
context="Anthropic tool invoke",
|
||||
)
|
||||
|
||||
# Check if this is a server-side tool (web_search, tool_search, etc.)
|
||||
|
|
@ -2015,11 +2048,17 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
for m in user_message_types_block["content"]:
|
||||
if m.get("type", "") == "image_url":
|
||||
m = cast(ChatCompletionImageObject, m)
|
||||
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
|
||||
format = (
|
||||
m["image_url"].get("format")
|
||||
if isinstance(m["image_url"], dict)
|
||||
else None
|
||||
)
|
||||
# Convert ChatCompletionImageUrlObject to dict if needed
|
||||
image_url_value = m["image_url"]
|
||||
if isinstance(image_url_value, str):
|
||||
image_url_input: Union[str, dict[str, Any]] = image_url_value
|
||||
image_url_input: Union[str, dict[str, Any]] = (
|
||||
image_url_value
|
||||
)
|
||||
else:
|
||||
# ChatCompletionImageUrlObject or dict case - convert to dict
|
||||
image_url_input = {
|
||||
|
|
@ -2029,20 +2068,26 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
# Bedrock invoke models have format: invoke/...
|
||||
# Vertex AI Anthropic also doesn't support URL sources for images
|
||||
is_bedrock_invoke = model.lower().startswith("invoke/")
|
||||
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
|
||||
is_vertex_ai = (
|
||||
llm_provider.startswith("vertex_ai")
|
||||
if llm_provider
|
||||
else False
|
||||
)
|
||||
force_base64 = is_bedrock_invoke or is_vertex_ai
|
||||
_anthropic_content_element = create_anthropic_image_param(
|
||||
image_url_input, format=format, is_bedrock_invoke=force_base64
|
||||
)
|
||||
image_url_input,
|
||||
format=format,
|
||||
is_bedrock_invoke=force_base64,
|
||||
)
|
||||
_content_element = add_cache_control_to_content(
|
||||
anthropic_content_element=_anthropic_content_element,
|
||||
original_content_element=dict(m),
|
||||
)
|
||||
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_content_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_content_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
user_content.append(_anthropic_content_element)
|
||||
elif m.get("type", "") == "text":
|
||||
m = cast(ChatCompletionTextObject, m)
|
||||
|
|
@ -2080,9 +2125,9 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
)
|
||||
|
||||
if "cache_control" in _content_element:
|
||||
_anthropic_content_text_element[
|
||||
"cache_control"
|
||||
] = _content_element["cache_control"]
|
||||
_anthropic_content_text_element["cache_control"] = (
|
||||
_content_element["cache_control"]
|
||||
)
|
||||
|
||||
user_content.append(_anthropic_content_text_element)
|
||||
|
||||
|
|
@ -2178,18 +2223,27 @@ def anthropic_messages_pt( # noqa: PLR0915
|
|||
): # support assistant tool invoke conversion
|
||||
# Get web_search_results from provider_specific_fields for server_tool_use reconstruction
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields_raw = assistant_content_block.get(
|
||||
"provider_specific_fields"
|
||||
)
|
||||
_provider_specific_fields: Dict[str, Any] = {}
|
||||
if isinstance(_provider_specific_fields_raw, dict):
|
||||
_provider_specific_fields = cast(Dict[str, Any], _provider_specific_fields_raw)
|
||||
_web_search_results = _provider_specific_fields.get("web_search_results")
|
||||
_provider_specific_fields = cast(
|
||||
Dict[str, Any], _provider_specific_fields_raw
|
||||
)
|
||||
_web_search_results = _provider_specific_fields.get(
|
||||
"web_search_results"
|
||||
)
|
||||
tool_invoke_results = convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_web_search_results,
|
||||
)
|
||||
# AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
|
||||
assistant_content.extend(
|
||||
cast(List[AnthropicMessagesAssistantMessageValues], tool_invoke_results)
|
||||
cast(
|
||||
List[AnthropicMessagesAssistantMessageValues],
|
||||
tool_invoke_results,
|
||||
)
|
||||
)
|
||||
|
||||
assistant_function_call = assistant_content_block.get("function_call")
|
||||
|
|
@ -3252,14 +3306,18 @@ def _convert_to_bedrock_tool_call_result(
|
|||
"""
|
||||
-
|
||||
"""
|
||||
tool_result_content_blocks:List[BedrockToolResultContentBlock] = []
|
||||
tool_result_content_blocks: List[BedrockToolResultContentBlock] = []
|
||||
if isinstance(message["content"], str):
|
||||
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=message["content"]))
|
||||
tool_result_content_blocks.append(
|
||||
BedrockToolResultContentBlock(text=message["content"])
|
||||
)
|
||||
elif isinstance(message["content"], List):
|
||||
content_list = message["content"]
|
||||
for content in content_list:
|
||||
if content["type"] == "text":
|
||||
tool_result_content_blocks.append(BedrockToolResultContentBlock(text=content["text"]))
|
||||
tool_result_content_blocks.append(
|
||||
BedrockToolResultContentBlock(text=content["text"])
|
||||
)
|
||||
elif content["type"] == "image_url":
|
||||
format: Optional[str] = None
|
||||
if isinstance(content["image_url"], dict):
|
||||
|
|
@ -3267,12 +3325,14 @@ def _convert_to_bedrock_tool_call_result(
|
|||
format = content["image_url"].get("format")
|
||||
else:
|
||||
image_url = content["image_url"]
|
||||
_block:BedrockContentBlock = BedrockImageProcessor.process_image_sync(
|
||||
_block: BedrockContentBlock = BedrockImageProcessor.process_image_sync(
|
||||
image_url=image_url,
|
||||
format=format,
|
||||
)
|
||||
if "image" in _block:
|
||||
tool_result_content_blocks.append(BedrockToolResultContentBlock(image=_block["image"]))
|
||||
tool_result_content_blocks.append(
|
||||
BedrockToolResultContentBlock(image=_block["image"])
|
||||
)
|
||||
|
||||
message.get("name", "")
|
||||
id = str(message.get("tool_call_id", str(uuid.uuid4())))
|
||||
|
|
|
|||
|
|
@ -719,6 +719,7 @@ class ModelResponseIterator:
|
|||
content_block_start=content_block_start,
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
)
|
||||
|
||||
elif content_block_start["content_block"]["type"].endswith("_tool_result"):
|
||||
# Handle all tool result types (web_search, bash_code_execution, text_editor, etc.)
|
||||
content_type = content_block_start["content_block"]["type"]
|
||||
|
|
@ -734,6 +735,16 @@ class ModelResponseIterator:
|
|||
provider_specific_fields["web_search_results"] = (
|
||||
self.web_search_results
|
||||
)
|
||||
elif content_type == "web_fetch_tool_result":
|
||||
# Capture web_fetch_tool_result for multi-turn reconstruction
|
||||
# The full content comes in content_block_start, not in deltas
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18137
|
||||
self.web_search_results.append(
|
||||
content_block_start["content_block"]
|
||||
)
|
||||
provider_specific_fields["web_search_results"] = (
|
||||
self.web_search_results
|
||||
)
|
||||
elif content_type != "tool_search_tool_result":
|
||||
# Handle other tool results (code execution, etc.)
|
||||
# Skip tool_search_tool_result as it's internal metadata
|
||||
|
|
@ -741,6 +752,7 @@ class ModelResponseIterator:
|
|||
self.tool_results = []
|
||||
self.tool_results.append(content_block_start["content_block"])
|
||||
provider_specific_fields["tool_results"] = self.tool_results
|
||||
|
||||
elif type_chunk == "content_block_stop":
|
||||
ContentBlockStop(**chunk) # type: ignore
|
||||
# check if tool call content block - only for tool_use and server_tool_use blocks
|
||||
|
|
|
|||
|
|
@ -1154,6 +1154,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
index=idx,
|
||||
)
|
||||
tool_calls.append(tool_call)
|
||||
|
||||
## TOOL RESULTS - handle all tool result types (code execution, etc.)
|
||||
elif content["type"].endswith("_tool_result"):
|
||||
# Skip tool_search_tool_result as it's internal metadata
|
||||
|
|
@ -1164,11 +1165,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if web_search_results is None:
|
||||
web_search_results = []
|
||||
web_search_results.append(content)
|
||||
elif content["type"] == "web_fetch_tool_result":
|
||||
if web_search_results is None:
|
||||
web_search_results = []
|
||||
web_search_results.append(content)
|
||||
else:
|
||||
# All other tool results (bash_code_execution_tool_result, text_editor_code_execution_tool_result, etc.)
|
||||
if tool_results is None:
|
||||
tool_results = []
|
||||
tool_results.append(content)
|
||||
|
||||
elif content.get("thinking", None) is not None:
|
||||
if thinking_blocks is None:
|
||||
thinking_blocks = []
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ from typing import (
|
|||
|
||||
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicToolsValues,
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
|
|
@ -425,15 +429,15 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
) -> Optional[str]:
|
||||
"""
|
||||
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
|
||||
|
||||
|
||||
Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int}
|
||||
OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default'
|
||||
"""
|
||||
if not isinstance(thinking, dict):
|
||||
return None
|
||||
|
||||
|
||||
thinking_type = thinking.get("type", "disabled")
|
||||
|
||||
|
||||
if thinking_type == "disabled":
|
||||
return None
|
||||
elif thinking_type == "enabled":
|
||||
|
|
@ -446,7 +450,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return "low"
|
||||
else:
|
||||
return "minimal"
|
||||
|
||||
|
||||
return None
|
||||
|
||||
def translate_anthropic_tool_choice_to_openai(
|
||||
|
|
@ -676,10 +680,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
type="tool_use",
|
||||
id=tool_call.id,
|
||||
name=tool_call.function.name or "",
|
||||
input=(
|
||||
json.loads(tool_call.function.arguments)
|
||||
if tool_call.function.arguments
|
||||
else {}
|
||||
input=parse_tool_call_arguments(
|
||||
tool_call.function.arguments,
|
||||
tool_name=tool_call.function.name,
|
||||
context="Anthropic pass-through adapter",
|
||||
),
|
||||
)
|
||||
# Add provider_specific_fields if signature is present
|
||||
|
|
|
|||
|
|
@ -74,6 +74,41 @@ class BaseAWSLLM:
|
|||
"aws_external_id",
|
||||
]
|
||||
|
||||
def _get_ssl_verify(self):
|
||||
"""
|
||||
Get SSL verification setting for boto3 clients.
|
||||
|
||||
This ensures that custom CA certificates are properly used for all AWS API calls,
|
||||
including STS and Bedrock services.
|
||||
|
||||
Returns:
|
||||
Union[bool, str]: SSL verification setting - False to disable, True to enable,
|
||||
or a string path to a CA bundle file
|
||||
"""
|
||||
import litellm
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
# Check environment variable first (highest priority)
|
||||
ssl_verify = os.getenv("SSL_VERIFY", litellm.ssl_verify)
|
||||
|
||||
# Convert string "False"/"True" to boolean
|
||||
if isinstance(ssl_verify, str):
|
||||
# Check if it's a file path
|
||||
if os.path.exists(ssl_verify):
|
||||
return ssl_verify
|
||||
# Otherwise try to convert to boolean
|
||||
ssl_verify_bool = str_to_bool(ssl_verify)
|
||||
if ssl_verify_bool is not None:
|
||||
ssl_verify = ssl_verify_bool
|
||||
|
||||
# Check SSL_CERT_FILE environment variable for custom CA bundle
|
||||
if ssl_verify is True or ssl_verify == "True":
|
||||
ssl_cert_file = os.getenv("SSL_CERT_FILE")
|
||||
if ssl_cert_file and os.path.exists(ssl_cert_file):
|
||||
return ssl_cert_file
|
||||
|
||||
return ssl_verify
|
||||
|
||||
def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
|
||||
"""
|
||||
Generate a unique cache key based on the credential arguments.
|
||||
|
|
@ -569,6 +604,7 @@ class BaseAWSLLM:
|
|||
"sts",
|
||||
region_name=aws_region_name,
|
||||
endpoint_url=sts_endpoint,
|
||||
verify=self._get_ssl_verify(),
|
||||
)
|
||||
|
||||
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
|
||||
|
|
@ -625,7 +661,7 @@ class BaseAWSLLM:
|
|||
|
||||
# Create an STS client without credentials
|
||||
with tracer.trace("boto3.client(sts) for manual IRSA"):
|
||||
sts_client = boto3.client("sts", region_name=region)
|
||||
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
|
||||
|
||||
# Manually assume the IRSA role with the session name
|
||||
verbose_logger.debug(
|
||||
|
|
@ -648,6 +684,7 @@ class BaseAWSLLM:
|
|||
aws_access_key_id=irsa_creds["AccessKeyId"],
|
||||
aws_secret_access_key=irsa_creds["SecretAccessKey"],
|
||||
aws_session_token=irsa_creds["SessionToken"],
|
||||
verify=self._get_ssl_verify(),
|
||||
)
|
||||
|
||||
# Get current caller identity for debugging
|
||||
|
|
@ -686,7 +723,7 @@ class BaseAWSLLM:
|
|||
|
||||
verbose_logger.debug("Same account role assumption, using automatic IRSA")
|
||||
with tracer.trace("boto3.client(sts) with automatic IRSA"):
|
||||
sts_client = boto3.client("sts", region_name=region)
|
||||
sts_client = boto3.client("sts", region_name=region, verify=self._get_ssl_verify())
|
||||
|
||||
# Get current caller identity for debugging
|
||||
try:
|
||||
|
|
@ -809,7 +846,7 @@ class BaseAWSLLM:
|
|||
# This allows the web identity token to work automatically
|
||||
if aws_access_key_id is None and aws_secret_access_key is None:
|
||||
with tracer.trace("boto3.client(sts)"):
|
||||
sts_client = boto3.client("sts")
|
||||
sts_client = boto3.client("sts", verify=self._get_ssl_verify())
|
||||
else:
|
||||
with tracer.trace("boto3.client(sts)"):
|
||||
sts_client = boto3.client(
|
||||
|
|
@ -817,6 +854,7 @@ class BaseAWSLLM:
|
|||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_session_token=aws_session_token,
|
||||
verify=self._get_ssl_verify(),
|
||||
)
|
||||
|
||||
assume_role_params = {
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ def init_bedrock_client(
|
|||
status_code=401,
|
||||
)
|
||||
|
||||
sts_client = boto3.client("sts")
|
||||
sts_client = boto3.client("sts", verify=ssl_verify)
|
||||
|
||||
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
|
||||
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ class BedrockFilesHandler(BaseAWSLLM):
|
|||
aws_secret_access_key=credentials.secret_key,
|
||||
aws_session_token=credentials.token,
|
||||
region_name=aws_region_name,
|
||||
verify=self._get_ssl_verify(),
|
||||
)
|
||||
|
||||
# Download file from S3
|
||||
|
|
|
|||
|
|
@ -24,6 +24,37 @@ class BedrockPassthroughConfig(
|
|||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
return "stream" in endpoint
|
||||
|
||||
def _encode_model_id_for_endpoint(self, model_id: str) -> str:
|
||||
"""
|
||||
Encode model_id (especially ARNs) for use in Bedrock endpoints.
|
||||
|
||||
ARNs contain special characters like colons and slashes that need to be
|
||||
properly URL-encoded when used in HTTP request paths. For example:
|
||||
arn:aws:bedrock:us-east-1:123:application-inference-profile/abc123
|
||||
becomes:
|
||||
arn:aws:bedrock:us-east-1:123:application-inference-profile%2Fabc123
|
||||
|
||||
Args:
|
||||
model_id: The model ID or ARN to encode
|
||||
|
||||
Returns:
|
||||
The encoded model_id suitable for use in endpoint URLs
|
||||
"""
|
||||
from litellm.passthrough.utils import CommonUtils
|
||||
import re
|
||||
|
||||
# Create a temporary endpoint with the model_id to check if encoding is needed
|
||||
temp_endpoint = f"/model/{model_id}/converse"
|
||||
encoded_temp_endpoint = CommonUtils.encode_bedrock_runtime_modelid_arn(temp_endpoint)
|
||||
|
||||
# Extract the encoded model_id from the temporary endpoint
|
||||
encoded_model_id_match = re.search(r'/model/([^/]+)/', encoded_temp_endpoint)
|
||||
if encoded_model_id_match:
|
||||
return encoded_model_id_match.group(1)
|
||||
else:
|
||||
# Fallback to original model_id if extraction fails
|
||||
return model_id
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
|
|
@ -53,9 +84,13 @@ class BedrockPassthroughConfig(
|
|||
# If model_id is provided (e.g., Application Inference Profile ARN), use it in the endpoint
|
||||
# instead of the translated model name
|
||||
if model_id is not None:
|
||||
# Replace the model name in the endpoint with the model_id
|
||||
import re
|
||||
endpoint = re.sub(r'model/[^/]+/', f'model/{model_id}/', endpoint)
|
||||
|
||||
# Encode the model_id if it's an ARN to properly handle special characters
|
||||
encoded_model_id = self._encode_model_id_for_endpoint(model_id)
|
||||
|
||||
# Replace the model name in the endpoint with the encoded model_id
|
||||
endpoint = re.sub(r'model/[^/]+/', f'model/{encoded_model_id}/', endpoint)
|
||||
return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url
|
||||
|
||||
def sign_request(
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
|
|||
"stop",
|
||||
"logprobs",
|
||||
"frequency_penalty",
|
||||
"presence_penalty",
|
||||
"modalities",
|
||||
"parallel_tool_calls",
|
||||
"web_search_options",
|
||||
|
|
|
|||
|
|
@ -1124,8 +1124,11 @@ def adapt_messages_to_generic_oci_standard_content_message(
|
|||
|
||||
elif type == "image_url":
|
||||
image_url = content_item.get("image_url")
|
||||
# Handle both OpenAI format (object with url) and string format
|
||||
if isinstance(image_url, dict):
|
||||
image_url = image_url.get("url")
|
||||
if not isinstance(image_url, str):
|
||||
raise Exception("Prop `image_url` is not a string")
|
||||
raise Exception("Prop `image_url` must be a string or an object with a `url` property")
|
||||
new_content.append(OCIImageContentPart(imageUrl=image_url))
|
||||
|
||||
return OCIMessage(
|
||||
|
|
|
|||
|
|
@ -310,9 +310,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"""
|
||||
return Tools(googleSearch={})
|
||||
|
||||
def _transform_computer_use_config(
|
||||
self, computer_use_config: dict
|
||||
) -> dict:
|
||||
def _transform_computer_use_config(self, computer_use_config: dict) -> dict:
|
||||
"""
|
||||
Transform Computer Use configuration to Gemini API format.
|
||||
|
||||
|
|
@ -323,7 +321,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
Transformed computer use configuration for Gemini API
|
||||
"""
|
||||
transformed_config = {}
|
||||
|
||||
|
||||
# Transform environment values if needed
|
||||
if "environment" in computer_use_config:
|
||||
env_value = computer_use_config["environment"]
|
||||
|
|
@ -339,13 +337,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
f"Invalid environment value for computer_use: {env_value}. "
|
||||
f"Supported: 'browser', 'unspecified', 'ENVIRONMENT_BROWSER', 'ENVIRONMENT_UNSPECIFIED'"
|
||||
)
|
||||
|
||||
|
||||
# Transform excluded_predefined_functions to camelCase
|
||||
if "excluded_predefined_functions" in computer_use_config:
|
||||
transformed_config["excludedPredefinedFunctions"] = computer_use_config["excluded_predefined_functions"]
|
||||
transformed_config["excludedPredefinedFunctions"] = computer_use_config[
|
||||
"excluded_predefined_functions"
|
||||
]
|
||||
elif "excludedPredefinedFunctions" in computer_use_config:
|
||||
transformed_config["excludedPredefinedFunctions"] = computer_use_config["excludedPredefinedFunctions"]
|
||||
|
||||
transformed_config["excludedPredefinedFunctions"] = computer_use_config[
|
||||
"excludedPredefinedFunctions"
|
||||
]
|
||||
|
||||
return transformed_config
|
||||
|
||||
def _extract_google_maps_retrieval_config(
|
||||
|
|
@ -446,9 +448,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
value = _remove_strict_from_schema(value)
|
||||
|
||||
for tool in value:
|
||||
openai_function_object: Optional[
|
||||
ChatCompletionToolParamFunctionChunk
|
||||
] = None
|
||||
openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = (
|
||||
None
|
||||
)
|
||||
if "function" in tool: # tools list
|
||||
_openai_function_object = ChatCompletionToolParamFunctionChunk( # type: ignore
|
||||
**tool["function"]
|
||||
|
|
@ -553,7 +555,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request."
|
||||
)
|
||||
|
||||
# Build list of Tool objects - each Tool should contain exactly one type
|
||||
# Build list of Tool objects - each Tool should contain exactly one type
|
||||
# per Vertex AI API spec: "A Tool object should contain exactly one type of Tool"
|
||||
_tools_list: List[Tools] = []
|
||||
|
||||
|
|
@ -570,11 +572,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
_tools_list.append(search_tool)
|
||||
if googleSearchRetrieval is not None:
|
||||
retrieval_tool = Tools()
|
||||
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
|
||||
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = (
|
||||
googleSearchRetrieval
|
||||
)
|
||||
_tools_list.append(retrieval_tool)
|
||||
if enterpriseWebSearch is not None:
|
||||
enterprise_tool = Tools()
|
||||
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
|
||||
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = (
|
||||
enterpriseWebSearch
|
||||
)
|
||||
_tools_list.append(enterprise_tool)
|
||||
if code_execution is not None:
|
||||
code_tool = Tools()
|
||||
|
|
@ -593,7 +599,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
computer_tool[VertexToolName.COMPUTER_USE.value] = computerUse
|
||||
_tools_list.append(computer_tool)
|
||||
|
||||
|
||||
# Add retrieval config to toolConfig if googleMaps has location data
|
||||
if google_maps_retrieval_config is not None:
|
||||
if "toolConfig" not in optional_params:
|
||||
|
|
@ -710,8 +715,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
GeminiThinkingConfig with thinkingLevel and includeThoughts
|
||||
"""
|
||||
# Check if this is gemini-3-flash which supports MINIMAL thinking level
|
||||
is_gemini3flash= model and (
|
||||
"gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
|
||||
is_gemini3flash = model and (
|
||||
"gemini-3-flash-preview" in model.lower()
|
||||
or "gemini-3-flash" in model.lower()
|
||||
)
|
||||
if reasoning_effort == "minimal":
|
||||
if is_gemini3flash:
|
||||
|
|
@ -799,7 +805,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
thinking_budget = thinking_param.get("budget_tokens")
|
||||
|
||||
params: GeminiThinkingConfig = {}
|
||||
|
||||
|
||||
# For Gemini 3+ models, use thinkingLevel instead of thinkingBudget
|
||||
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
if thinking_enabled:
|
||||
|
|
@ -808,11 +814,21 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
else:
|
||||
params["includeThoughts"] = True
|
||||
if thinking_budget >= 10000:
|
||||
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
|
||||
params["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
|
||||
is_gemini3flash = (
|
||||
"gemini-3-flash-preview" in model.lower()
|
||||
or "gemini-3-flash" in model.lower()
|
||||
)
|
||||
params["thinkingLevel"] = (
|
||||
"minimal" if is_gemini3flash else "low"
|
||||
)
|
||||
else:
|
||||
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
|
||||
params["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
|
||||
is_gemini3flash = (
|
||||
"gemini-3-flash-preview" in model.lower()
|
||||
or "gemini-3-flash" in model.lower()
|
||||
)
|
||||
params["thinkingLevel"] = (
|
||||
"minimal" if is_gemini3flash else "low"
|
||||
)
|
||||
else:
|
||||
# Thinking disabled
|
||||
params["includeThoughts"] = False
|
||||
|
|
@ -824,7 +840,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
params["includeThoughts"] = True
|
||||
if thinking_budget is not None and isinstance(thinking_budget, int):
|
||||
params["thinkingBudget"] = thinking_budget
|
||||
|
||||
|
||||
return params
|
||||
|
||||
def map_response_modalities(self, value: list) -> list:
|
||||
|
|
@ -980,16 +996,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
param_description="thinking_budget",
|
||||
)
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
optional_params[
|
||||
"thinkingConfig"
|
||||
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
|
||||
value, model
|
||||
optional_params["thinkingConfig"] = (
|
||||
VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
|
||||
value, model
|
||||
)
|
||||
)
|
||||
else:
|
||||
optional_params[
|
||||
"thinkingConfig"
|
||||
] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
|
||||
value, model
|
||||
optional_params["thinkingConfig"] = (
|
||||
VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
|
||||
value, model
|
||||
)
|
||||
)
|
||||
elif param == "thinking":
|
||||
# Validate no conflict with thinking_level
|
||||
|
|
@ -998,11 +1014,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
param_name="thinking",
|
||||
param_description="thinking_budget",
|
||||
)
|
||||
optional_params[
|
||||
"thinkingConfig"
|
||||
] = VertexGeminiConfig._map_thinking_param(
|
||||
cast(AnthropicThinkingParam, value),
|
||||
model=model,
|
||||
optional_params["thinkingConfig"] = (
|
||||
VertexGeminiConfig._map_thinking_param(
|
||||
cast(AnthropicThinkingParam, value),
|
||||
model=model,
|
||||
)
|
||||
)
|
||||
elif param == "modalities" and isinstance(value, list):
|
||||
response_modalities = self.map_response_modalities(value)
|
||||
|
|
@ -1036,8 +1052,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
):
|
||||
# For gemini-3-flash-preview, default to "minimal" to match Gemini 2.5 Flash behavior
|
||||
# For other Gemini 3 models, default to "low"
|
||||
is_gemini3flash = "gemini-3-flash-preview" in model.lower() or "gemini-3-flash" in model.lower()
|
||||
thinking_config["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
|
||||
is_gemini3flash = (
|
||||
"gemini-3-flash-preview" in model.lower()
|
||||
or "gemini-3-flash" in model.lower()
|
||||
)
|
||||
thinking_config["thinkingLevel"] = (
|
||||
"minimal" if is_gemini3flash else "low"
|
||||
)
|
||||
optional_params["thinkingConfig"] = thinking_config
|
||||
|
||||
return optional_params
|
||||
|
|
@ -1226,7 +1247,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
block: ChatCompletionThinkingBlock = {
|
||||
"type": "thinking",
|
||||
"thinking": thinking_text,
|
||||
}
|
||||
}
|
||||
signature = part.get("thoughtSignature")
|
||||
if signature is not None:
|
||||
block["signature"] = signature
|
||||
|
|
@ -1360,10 +1381,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
_tool_response_chunk["provider_specific_fields"] = { # type: ignore
|
||||
"thought_signature": thought_signature
|
||||
}
|
||||
_tool_response_chunk[
|
||||
"id"
|
||||
] = _encode_tool_call_id_with_signature(
|
||||
_tool_response_chunk["id"] or "", thought_signature
|
||||
_tool_response_chunk["id"] = (
|
||||
_encode_tool_call_id_with_signature(
|
||||
_tool_response_chunk["id"] or "", thought_signature
|
||||
)
|
||||
)
|
||||
_tools.append(_tool_response_chunk)
|
||||
cumulative_tool_call_idx += 1
|
||||
|
|
@ -1551,13 +1572,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif modality == "IMAGE":
|
||||
response_tokens_details.image_tokens = token_count
|
||||
|
||||
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
|
||||
# candidatesTokenCount includes all modalities, so: text = total - (image + audio)
|
||||
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
|
||||
# candidatesTokenCount includes all modalities, so: text = total - (image + audio)
|
||||
candidates_token_count = usage_metadata.get("candidatesTokenCount", 0)
|
||||
if candidates_token_count > 0:
|
||||
if response_tokens_details is None:
|
||||
response_tokens_details = CompletionTokensDetailsWrapper()
|
||||
if response_tokens_details.text_tokens is None:
|
||||
candidates_token_count = usage_metadata.get("candidatesTokenCount", 0)
|
||||
image_tokens = response_tokens_details.image_tokens or 0
|
||||
audio_tokens_candidate = response_tokens_details.audio_tokens or 0
|
||||
calculated_text_tokens = candidates_token_count - image_tokens - audio_tokens_candidate
|
||||
calculated_text_tokens = (
|
||||
candidates_token_count - image_tokens - audio_tokens_candidate
|
||||
)
|
||||
response_tokens_details.text_tokens = calculated_text_tokens
|
||||
#########################################################
|
||||
|
||||
|
|
@ -2076,28 +2102,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
## ADD METADATA TO RESPONSE ##
|
||||
|
||||
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
|
||||
model_response._hidden_params[
|
||||
"vertex_ai_grounding_metadata"
|
||||
] = grounding_metadata
|
||||
model_response._hidden_params["vertex_ai_grounding_metadata"] = (
|
||||
grounding_metadata
|
||||
)
|
||||
|
||||
setattr(
|
||||
model_response, "vertex_ai_url_context_metadata", url_context_metadata
|
||||
)
|
||||
|
||||
model_response._hidden_params[
|
||||
"vertex_ai_url_context_metadata"
|
||||
] = url_context_metadata
|
||||
model_response._hidden_params["vertex_ai_url_context_metadata"] = (
|
||||
url_context_metadata
|
||||
)
|
||||
|
||||
setattr(model_response, "vertex_ai_safety_results", safety_ratings)
|
||||
model_response._hidden_params[
|
||||
"vertex_ai_safety_results"
|
||||
] = safety_ratings # older approach - maintaining to prevent regressions
|
||||
model_response._hidden_params["vertex_ai_safety_results"] = (
|
||||
safety_ratings # older approach - maintaining to prevent regressions
|
||||
)
|
||||
|
||||
## ADD CITATION METADATA ##
|
||||
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
|
||||
model_response._hidden_params[
|
||||
"vertex_ai_citation_metadata"
|
||||
] = citation_metadata # older approach - maintaining to prevent regressions
|
||||
model_response._hidden_params["vertex_ai_citation_metadata"] = (
|
||||
citation_metadata # older approach - maintaining to prevent regressions
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
raise VertexAIError(
|
||||
|
|
|
|||
|
|
@ -23340,13 +23340,13 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-oss-20b": {
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"input_cost_per_token": 2e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-20b",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
|
|
|
|||
|
|
@ -23340,13 +23340,13 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-oss-20b": {
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"input_cost_per_token": 2e-08,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-20b",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
|
|
|
|||
|
|
@ -197,3 +197,90 @@ class TestOCIGetCompleteUrl:
|
|||
|
||||
assert "eu-frankfurt-1" in url
|
||||
assert "inference.generativeai" in url
|
||||
|
||||
|
||||
class TestOCIImageUrlTransformation:
|
||||
"""Tests for OCI image_url format handling in multimodal messages.
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/18270
|
||||
"""
|
||||
|
||||
def test_image_url_as_string(self):
|
||||
"""Test that image_url as a plain string works."""
|
||||
from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": "https://example.com/image.png"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = adapt_messages_to_generic_oci_standard(messages)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].role == "USER"
|
||||
assert len(result[0].content) == 2
|
||||
assert result[0].content[1].imageUrl == "https://example.com/image.png"
|
||||
|
||||
def test_image_url_as_openai_object(self):
|
||||
"""Test that image_url as OpenAI-style object {"url": "..."} works."""
|
||||
from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = adapt_messages_to_generic_oci_standard(messages)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0].role == "USER"
|
||||
assert len(result[0].content) == 2
|
||||
assert result[0].content[1].imageUrl == "https://example.com/image.png"
|
||||
|
||||
def test_image_url_invalid_type_raises_error(self):
|
||||
"""Test that invalid image_url type raises an error."""
|
||||
from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": 12345}, # Invalid type
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
adapt_messages_to_generic_oci_standard(messages)
|
||||
|
||||
assert "image_url" in str(exc_info.value)
|
||||
|
||||
def test_image_url_object_missing_url_raises_error(self):
|
||||
"""Test that object without 'url' property raises an error."""
|
||||
from litellm.llms.oci.chat.transformation import adapt_messages_to_generic_oci_standard
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{"type": "image_url", "image_url": {"detail": "high"}}, # Missing 'url'
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
adapt_messages_to_generic_oci_standard(messages)
|
||||
|
||||
assert "image_url" in str(exc_info.value)
|
||||
|
|
|
|||
|
|
@ -7,11 +7,10 @@ import pytest
|
|||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
from typing import Union, List
|
||||
from typing import List
|
||||
|
||||
# from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory
|
||||
import litellm
|
||||
from litellm import completion
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
_bedrock_tools_pt,
|
||||
anthropic_messages_pt,
|
||||
|
|
@ -31,7 +30,7 @@ from litellm.llms.vertex_ai.gemini.transformation import (
|
|||
_gemini_convert_messages_with_history,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def test_llama_3_prompt():
|
||||
|
|
@ -129,10 +128,6 @@ def test_anthropic_pt_formatting():
|
|||
|
||||
|
||||
def test_anthropic_messages_nested_pt():
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
AnthropicMessagesUserMessageParam,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"content": [{"text": "here is a task", "type": "text"}], "role": "user"},
|
||||
|
|
@ -214,7 +209,7 @@ def test_create_anthropic_image_param_with_http_url():
|
|||
image_param = create_anthropic_image_param(
|
||||
"https://example.com/image.jpg", format=None
|
||||
)
|
||||
|
||||
|
||||
assert image_param["type"] == "image"
|
||||
assert image_param["source"]["type"] == "url"
|
||||
assert image_param["source"]["url"] == "https://example.com/image.jpg"
|
||||
|
|
@ -225,7 +220,7 @@ def test_create_anthropic_image_param_with_https_url():
|
|||
image_param = create_anthropic_image_param(
|
||||
"https://example.com/image.png", format=None
|
||||
)
|
||||
|
||||
|
||||
assert image_param["type"] == "image"
|
||||
assert image_param["source"]["type"] == "url"
|
||||
assert image_param["source"]["url"] == "https://example.com/image.png"
|
||||
|
|
@ -236,7 +231,7 @@ def test_create_anthropic_image_param_with_dict_input():
|
|||
image_param = create_anthropic_image_param(
|
||||
{"url": "https://example.com/image.jpg", "format": "image/jpeg"}, format=None
|
||||
)
|
||||
|
||||
|
||||
assert image_param["type"] == "image"
|
||||
assert image_param["source"]["type"] == "url"
|
||||
assert image_param["source"]["url"] == "https://example.com/image.jpg"
|
||||
|
|
@ -247,7 +242,7 @@ def test_create_anthropic_image_param_with_base64_data_uri():
|
|||
image_param = create_anthropic_image_param(
|
||||
"data:image/jpeg;base64,/9j/4AAQSkZJRg==", format=None
|
||||
)
|
||||
|
||||
|
||||
assert image_param["type"] == "image"
|
||||
assert image_param["source"]["type"] == "base64"
|
||||
assert image_param["source"]["media_type"] == "image/jpeg"
|
||||
|
|
@ -259,7 +254,7 @@ def test_create_anthropic_image_param_with_format_override():
|
|||
image_param = create_anthropic_image_param(
|
||||
"data:image/jpeg;base64,1234", format="image/png"
|
||||
)
|
||||
|
||||
|
||||
assert image_param["type"] == "image"
|
||||
assert image_param["source"]["type"] == "base64"
|
||||
assert image_param["source"]["media_type"] == "image/png"
|
||||
|
|
@ -279,19 +274,19 @@ def test_anthropic_messages_pt_with_url_image():
|
|||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
result = anthropic_messages_pt(
|
||||
messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic"
|
||||
)
|
||||
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["role"] == "user"
|
||||
assert isinstance(result[0]["content"], list)
|
||||
assert len(result[0]["content"]) == 2
|
||||
|
||||
|
||||
# Check text content
|
||||
assert result[0]["content"][0]["type"] == "text"
|
||||
|
||||
|
||||
# Check image content - should be URL reference, not base64
|
||||
assert result[0]["content"][1]["type"] == "image"
|
||||
assert result[0]["content"][1]["source"]["type"] == "url"
|
||||
|
|
@ -312,16 +307,16 @@ def test_anthropic_messages_pt_with_base64_image():
|
|||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
result = anthropic_messages_pt(
|
||||
messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic"
|
||||
)
|
||||
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["role"] == "user"
|
||||
assert isinstance(result[0]["content"], list)
|
||||
assert len(result[0]["content"]) == 2
|
||||
|
||||
|
||||
# Check image content - should be base64, not URL
|
||||
assert result[0]["content"][1]["type"] == "image"
|
||||
assert result[0]["content"][1]["source"]["type"] == "base64"
|
||||
|
|
@ -568,7 +563,9 @@ def test_vertex_only_image_user_message():
|
|||
},
|
||||
]
|
||||
|
||||
response = _gemini_convert_messages_with_history(messages=messages, model="gemini-1.5-pro")
|
||||
response = _gemini_convert_messages_with_history(
|
||||
messages=messages, model="gemini-1.5-pro"
|
||||
)
|
||||
|
||||
expected_response = [
|
||||
{
|
||||
|
|
@ -962,8 +959,8 @@ def test_convert_to_anthropic_tool_invoke_regular_tool():
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"location": "San Francisco"}'
|
||||
}
|
||||
"arguments": '{"location": "San Francisco"}',
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -979,7 +976,7 @@ def test_convert_to_anthropic_tool_invoke_regular_tool():
|
|||
def test_convert_to_anthropic_tool_invoke_server_tool():
|
||||
"""
|
||||
Test that server_tool_use (srvtoolu_) is reconstructed as server_tool_use.
|
||||
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
"""
|
||||
tool_calls = [
|
||||
|
|
@ -988,8 +985,8 @@ def test_convert_to_anthropic_tool_invoke_server_tool():
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"arguments": '{"query": "elephant weight"}'
|
||||
}
|
||||
"arguments": '{"query": "elephant weight"}',
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -1005,7 +1002,7 @@ def test_convert_to_anthropic_tool_invoke_server_tool():
|
|||
def test_convert_to_anthropic_tool_invoke_with_web_search_results():
|
||||
"""
|
||||
Test that web_search_tool_result is included after server_tool_use.
|
||||
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
"""
|
||||
tool_calls = [
|
||||
|
|
@ -1014,8 +1011,8 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results():
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"arguments": '{"query": "elephant weight"}'
|
||||
}
|
||||
"arguments": '{"query": "elephant weight"}',
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -1028,13 +1025,15 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results():
|
|||
"type": "web_search_result",
|
||||
"url": "https://example.com",
|
||||
"title": "Elephant Facts",
|
||||
"snippet": "Elephants weigh 5000 kg"
|
||||
"snippet": "Elephants weigh 5000 kg",
|
||||
}
|
||||
]
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
result = convert_to_anthropic_tool_invoke(tool_calls, web_search_results=web_search_results)
|
||||
result = convert_to_anthropic_tool_invoke(
|
||||
tool_calls, web_search_results=web_search_results
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
# First: server_tool_use
|
||||
|
|
@ -1048,7 +1047,7 @@ def test_convert_to_anthropic_tool_invoke_with_web_search_results():
|
|||
def test_convert_to_anthropic_tool_invoke_mixed_tools():
|
||||
"""
|
||||
Test that mixed server and regular tools are reconstructed correctly.
|
||||
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
"""
|
||||
tool_calls = [
|
||||
|
|
@ -1057,28 +1056,27 @@ def test_convert_to_anthropic_tool_invoke_mixed_tools():
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"arguments": '{"query": "elephant weight"}'
|
||||
}
|
||||
"arguments": '{"query": "elephant weight"}',
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "toolu_01XYZ789",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "add_numbers",
|
||||
"arguments": '{"a": 5000, "b": 100}'
|
||||
}
|
||||
}
|
||||
"function": {"name": "add_numbers", "arguments": '{"a": 5000, "b": 100}'},
|
||||
},
|
||||
]
|
||||
|
||||
web_search_results = [
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_01ABC123",
|
||||
"content": [{"url": "https://example.com", "title": "Test"}]
|
||||
"content": [{"url": "https://example.com", "title": "Test"}],
|
||||
}
|
||||
]
|
||||
|
||||
result = convert_to_anthropic_tool_invoke(tool_calls, web_search_results=web_search_results)
|
||||
result = convert_to_anthropic_tool_invoke(
|
||||
tool_calls, web_search_results=web_search_results
|
||||
)
|
||||
|
||||
assert len(result) == 3
|
||||
# First: server_tool_use
|
||||
|
|
@ -1094,7 +1092,7 @@ def test_convert_to_anthropic_tool_invoke_mixed_tools():
|
|||
def test_anthropic_messages_pt_with_server_tool_use():
|
||||
"""
|
||||
Test that anthropic_messages_pt correctly reconstructs server_tool_use from provider_specific_fields.
|
||||
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
"""
|
||||
messages = [
|
||||
|
|
@ -1108,36 +1106,40 @@ def test_anthropic_messages_pt_with_server_tool_use():
|
|||
"type": "function",
|
||||
"function": {
|
||||
"name": "web_search",
|
||||
"arguments": '{"query": "elephant weight"}'
|
||||
}
|
||||
"arguments": '{"query": "elephant weight"}',
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "toolu_01XYZ789",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "add_numbers",
|
||||
"arguments": '{"a": 5000, "b": 100}'
|
||||
}
|
||||
}
|
||||
"arguments": '{"a": 5000, "b": 100}',
|
||||
},
|
||||
},
|
||||
],
|
||||
"provider_specific_fields": {
|
||||
"web_search_results": [
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_01ABC123",
|
||||
"content": [{"url": "https://example.com", "title": "Test", "snippet": "5000 kg"}]
|
||||
"content": [
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"title": "Test",
|
||||
"snippet": "5000 kg",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "toolu_01XYZ789",
|
||||
"content": "5100"
|
||||
}
|
||||
{"role": "tool", "tool_call_id": "toolu_01XYZ789", "content": "5100"},
|
||||
]
|
||||
|
||||
result = anthropic_messages_pt(messages, model="claude-sonnet-4-5", llm_provider="anthropic")
|
||||
result = anthropic_messages_pt(
|
||||
messages, model="claude-sonnet-4-5", llm_provider="anthropic"
|
||||
)
|
||||
|
||||
# Find the assistant message
|
||||
assistant_msg = next(m for m in result if m["role"] == "assistant")
|
||||
|
|
@ -1162,3 +1164,73 @@ def test_anthropic_messages_pt_with_server_tool_use():
|
|||
# Verify regular tool_use
|
||||
tool_use = next(c for c in content if c.get("type") == "tool_use")
|
||||
assert tool_use["id"] == "toolu_01XYZ789"
|
||||
|
||||
|
||||
# ============ parse_tool_call_arguments Tests ============
|
||||
# Tests for the shared utility that parses tool call JSON arguments
|
||||
|
||||
|
||||
def test_parse_tool_call_arguments_valid_json():
|
||||
"""Test that valid JSON is parsed correctly."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
|
||||
result = parse_tool_call_arguments('{"city": "Paris", "units": "celsius"}')
|
||||
assert result == {"city": "Paris", "units": "celsius"}
|
||||
|
||||
|
||||
def test_parse_tool_call_arguments_empty_input():
|
||||
"""Test that None/empty input returns empty dict."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
|
||||
assert parse_tool_call_arguments(None) == {}
|
||||
assert parse_tool_call_arguments("") == {}
|
||||
|
||||
|
||||
def test_parse_tool_call_arguments_malformed_json():
|
||||
"""Test that malformed JSON raises ValueError with context."""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
parse_tool_call_arguments,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
parse_tool_call_arguments(
|
||||
'{"skill_name": "pptx',
|
||||
tool_name="load_skill",
|
||||
context="Anthropic tool invoke",
|
||||
)
|
||||
|
||||
error_msg = str(exc_info.value)
|
||||
assert "load_skill" in error_msg
|
||||
assert "Anthropic tool invoke" in error_msg
|
||||
assert '{"skill_name": "pptx' in error_msg
|
||||
assert "Unterminated string" in error_msg
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_invoke_malformed_json():
|
||||
"""
|
||||
Test that convert_to_anthropic_tool_invoke raises ValueError with context
|
||||
when tool arguments contain malformed JSON.
|
||||
|
||||
Fixes: https://github.com/BerriAI/litellm/issues/18920
|
||||
"""
|
||||
tool_calls = [
|
||||
{
|
||||
"id": "toolu_01_invalid",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "bad_tool",
|
||||
"arguments": '{"truncated', # Malformed JSON
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
convert_to_anthropic_tool_invoke(tool_calls)
|
||||
|
||||
error_msg = str(exc_info.value)
|
||||
assert "bad_tool" in error_msg
|
||||
assert '{"truncated' in error_msg
|
||||
|
|
|
|||
|
|
@ -1120,12 +1120,13 @@ async def test_google_generate_content_with_openai():
|
|||
|
||||
# Print the response for verification
|
||||
print(f"Response: {response}")
|
||||
#########################################################
|
||||
#########################################################
|
||||
# validate only expected fields were sent to litellm.completion
|
||||
passed_fields = set(call_kwargs.keys())
|
||||
# remove any GenericLiteLLMParams fields
|
||||
passed_fields = passed_fields - set(GenericLiteLLMParams.model_fields.keys())
|
||||
assert passed_fields == set(["model", "messages"]), f"Expected only model and messages to be passed through, got {passed_fields}"
|
||||
# extra_headers is now explicitly passed through for providers that need custom headers
|
||||
assert passed_fields == set(["model", "messages", "extra_headers"]), f"Expected model, messages, and extra_headers to be passed through, got {passed_fields}"
|
||||
@pytest.mark.asyncio
|
||||
async def test_agenerate_content_x_goog_api_key_header():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ def test_stream_transformation_error_handling():
|
|||
def test_non_stream_response_when_stream_requested():
|
||||
"""Test handling of non-stream responses when streaming was requested"""
|
||||
from litellm.types.utils import Choices
|
||||
|
||||
|
||||
# Mock a non-stream response (ModelResponse with valid choices)
|
||||
mock_response = ModelResponse(
|
||||
id="test-123",
|
||||
|
|
@ -270,13 +270,13 @@ def test_non_stream_response_when_stream_requested():
|
|||
model="gpt-3.5-turbo",
|
||||
object="chat.completion"
|
||||
)
|
||||
|
||||
|
||||
# Create an instance of the adapter
|
||||
adapter = GoogleGenAIAdapter()
|
||||
|
||||
|
||||
# Test the adapter's translate_completion_to_generate_content method directly
|
||||
result = adapter.translate_completion_to_generate_content(mock_response)
|
||||
|
||||
|
||||
# Verify the result is a valid Google GenAI format response
|
||||
assert "candidates" in result
|
||||
assert isinstance(result["candidates"], list)
|
||||
|
|
@ -287,4 +287,70 @@ def test_non_stream_response_when_stream_requested():
|
|||
assert isinstance(candidate["content"]["parts"], list)
|
||||
assert len(candidate["content"]["parts"]) > 0
|
||||
assert "text" in candidate["content"]["parts"][0]
|
||||
assert candidate["content"]["parts"][0]["text"] == "Hello, world!"
|
||||
assert candidate["content"]["parts"][0]["text"] == "Hello, world!"
|
||||
|
||||
|
||||
def test_extra_headers_forwarding():
|
||||
"""Test that extra_headers is correctly forwarded to completion call.
|
||||
|
||||
This is important for providers like github_copilot that require custom
|
||||
headers (e.g., Editor-Version) for authentication.
|
||||
"""
|
||||
# Test that extra_headers is included in completion kwargs
|
||||
model = "gpt-3.5-turbo"
|
||||
contents = {"role": "user", "parts": [{"text": "Test"}]}
|
||||
config = {"temperature": 0.7}
|
||||
|
||||
extra_kwargs = {
|
||||
"extra_headers": {
|
||||
"Editor-Version": "vscode/1.95.0",
|
||||
"Editor-Plugin-Version": "copilot-chat/0.22.4",
|
||||
"Custom-Header": "custom-value"
|
||||
},
|
||||
"metadata": {"user_id": "test-user"}
|
||||
}
|
||||
|
||||
completion_kwargs = GenerateContentToCompletionHandler._prepare_completion_kwargs(
|
||||
model=model,
|
||||
contents=contents,
|
||||
config=config,
|
||||
stream=False,
|
||||
extra_kwargs=extra_kwargs
|
||||
)
|
||||
|
||||
# Verify extra_headers is forwarded
|
||||
assert "extra_headers" in completion_kwargs, "extra_headers should be forwarded to completion call"
|
||||
assert completion_kwargs["extra_headers"]["Editor-Version"] == "vscode/1.95.0"
|
||||
assert completion_kwargs["extra_headers"]["Editor-Plugin-Version"] == "copilot-chat/0.22.4"
|
||||
assert completion_kwargs["extra_headers"]["Custom-Header"] == "custom-value"
|
||||
|
||||
# Verify metadata is also forwarded (existing behavior)
|
||||
assert "metadata" in completion_kwargs
|
||||
assert completion_kwargs["metadata"]["user_id"] == "test-user"
|
||||
|
||||
|
||||
def test_extra_headers_not_present():
|
||||
"""Test that missing extra_headers doesn't cause issues."""
|
||||
model = "gpt-3.5-turbo"
|
||||
contents = {"role": "user", "parts": [{"text": "Test"}]}
|
||||
config = {"temperature": 0.7}
|
||||
|
||||
# extra_kwargs without extra_headers
|
||||
extra_kwargs = {
|
||||
"metadata": {"user_id": "test-user"}
|
||||
}
|
||||
|
||||
completion_kwargs = GenerateContentToCompletionHandler._prepare_completion_kwargs(
|
||||
model=model,
|
||||
contents=contents,
|
||||
config=config,
|
||||
stream=False,
|
||||
extra_kwargs=extra_kwargs
|
||||
)
|
||||
|
||||
# Verify extra_headers is not present (no error)
|
||||
assert "extra_headers" not in completion_kwargs
|
||||
|
||||
# Verify metadata is still forwarded
|
||||
assert "metadata" in completion_kwargs
|
||||
assert completion_kwargs["metadata"]["user_id"] == "test-user"
|
||||
|
|
@ -473,7 +473,6 @@ def test_partial_json_chunk_accumulation():
|
|||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
|
||||
# Simulate a complete JSON chunk being split into two parts
|
||||
partial_chunk_1 = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hel'
|
||||
partial_chunk_2 = 'lo"}}'
|
||||
|
||||
|
|
@ -781,6 +780,169 @@ def test_web_search_tool_result_captured_in_provider_specific_fields():
|
|||
), "First result title should match"
|
||||
|
||||
|
||||
def test_web_fetch_tool_result_captured_in_provider_specific_fields():
|
||||
"""
|
||||
Test that web_fetch_tool_result content is captured in provider_specific_fields.
|
||||
|
||||
This tests the fix for https://github.com/BerriAI/litellm/issues/18137
|
||||
where streaming with Anthropic web fetch wasn't capturing web_fetch_tool_result
|
||||
blocks, causing multi-turn conversations to fail.
|
||||
|
||||
The web_fetch_tool_result content comes ALL AT ONCE in content_block_start,
|
||||
not in deltas, so we need to capture it there.
|
||||
"""
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
|
||||
# Simulate the streaming sequence with web_fetch_tool_result
|
||||
chunks = [
|
||||
# 1. message_start
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
# 2. server_tool_use block starts (web_fetch)
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01ABC123",
|
||||
"name": "web_fetch",
|
||||
},
|
||||
},
|
||||
# 3. input_json_delta with the url
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com"}'},
|
||||
},
|
||||
# 4. content_block_stop for server_tool_use
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
# 5. web_fetch_tool_result block starts - THIS IS WHERE THE RESULTS ARE
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01ABC123",
|
||||
"content": {
|
||||
"type": "web_fetch_result",
|
||||
"url": "https://example.com",
|
||||
"retrieved_at": "2025-12-16T19:28:29.758000+00:00",
|
||||
"content": {
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "text",
|
||||
"media_type": "text/plain",
|
||||
"data": "Hello World",
|
||||
},
|
||||
"title": "Example Page",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
# 6. content_block_stop for web_fetch_tool_result
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
]
|
||||
|
||||
web_search_results = None
|
||||
for chunk in chunks:
|
||||
parsed = iterator.chunk_parser(chunk)
|
||||
if (
|
||||
parsed.choices
|
||||
and parsed.choices[0].delta.provider_specific_fields
|
||||
and "web_search_results" in parsed.choices[0].delta.provider_specific_fields
|
||||
):
|
||||
web_search_results = parsed.choices[0].delta.provider_specific_fields[
|
||||
"web_search_results"
|
||||
]
|
||||
|
||||
# Verify web_fetch_tool_result was captured (stored in web_search_results list)
|
||||
assert web_search_results is not None, "web_search_results should be captured"
|
||||
assert len(web_search_results) == 1, "Should have 1 web_fetch_tool_result block"
|
||||
assert (
|
||||
web_search_results[0]["type"] == "web_fetch_tool_result"
|
||||
), "Block type should be web_fetch_tool_result"
|
||||
assert (
|
||||
web_search_results[0]["tool_use_id"] == "srvtoolu_01ABC123"
|
||||
), "tool_use_id should match"
|
||||
assert (
|
||||
web_search_results[0]["content"]["url"] == "https://example.com"
|
||||
), "URL should match"
|
||||
assert (
|
||||
web_search_results[0]["content"]["content"]["title"] == "Example Page"
|
||||
), "Title should match"
|
||||
|
||||
|
||||
def test_web_fetch_tool_result_no_extra_tool_calls():
|
||||
"""
|
||||
Test that web_fetch_tool_result blocks don't emit tool call chunks.
|
||||
|
||||
This tests the fix for https://github.com/BerriAI/litellm/issues/18137
|
||||
where streaming with Anthropic web fetch was causing issues with tool call arguments.
|
||||
|
||||
The issue was that web_fetch_tool_result blocks have input_json_delta events with {}
|
||||
that were incorrectly being converted to tool calls.
|
||||
"""
|
||||
iterator = ModelResponseIterator(
|
||||
streaming_response=MagicMock(), sync_stream=True, json_mode=False
|
||||
)
|
||||
|
||||
# to verify it doesn't emit tool calls
|
||||
chunks = [
|
||||
# 1. web_fetch_tool_result block starts
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 1,
|
||||
"content_block": {
|
||||
"type": "web_fetch_tool_result",
|
||||
"tool_use_id": "srvtoolu_01ABC123",
|
||||
"content": {
|
||||
"type": "web_fetch_result",
|
||||
"url": "https://example.com",
|
||||
"retrieved_at": "2025-12-16T19:28:29.758000+00:00",
|
||||
"content": {
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "text",
|
||||
"media_type": "text/plain",
|
||||
"data": "Hello World",
|
||||
},
|
||||
"title": "Example Page",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
# 2. input_json_delta with {} - this should NOT emit a tool call
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 1,
|
||||
"delta": {"type": "input_json_delta", "partial_json": "{}"},
|
||||
},
|
||||
# 3. content_block_stop for web_fetch_tool_result
|
||||
{"type": "content_block_stop", "index": 1},
|
||||
]
|
||||
|
||||
tool_call_count = 0
|
||||
for chunk in chunks:
|
||||
parsed = iterator.chunk_parser(chunk)
|
||||
if parsed.choices and parsed.choices[0].delta.tool_calls:
|
||||
tool_call_count += 1
|
||||
|
||||
# Should have 0 tool calls - web_fetch_tool_result should not emit tool calls
|
||||
assert (
|
||||
tool_call_count == 0
|
||||
), f"Expected 0 tool calls, got {tool_call_count}. web_fetch_tool_result should not emit tool calls"
|
||||
|
||||
|
||||
def test_container_in_provider_specific_fields_streaming():
|
||||
"""
|
||||
Test that container is captured in provider_specific_fields for streaming responses.
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ def test_bedrock_passthrough_with_application_inference_profile():
|
|||
|
||||
This test verifies the fix for GitHub issue #18761 where Bedrock passthrough
|
||||
was not working with Application Inference Profiles. The model_id (ARN) should
|
||||
replace the translated model name in the endpoint URL.
|
||||
replace the translated model name in the endpoint URL and be properly encoded.
|
||||
"""
|
||||
config = BedrockPassthroughConfig()
|
||||
|
||||
|
|
@ -204,19 +204,21 @@ def test_bedrock_passthrough_with_application_inference_profile():
|
|||
litellm_params={"model_id": model_id, "aws_region_name": "eu-west-1"}
|
||||
)
|
||||
|
||||
# Verify that the URL contains the model_id (ARN) instead of the model name
|
||||
# Verify that the URL contains the encoded model_id (ARN) instead of the model name
|
||||
url_str = str(url)
|
||||
assert model_id in url_str, f"Expected model_id ARN in URL, but got: {url_str}"
|
||||
# The ARN slash should be encoded as %2F
|
||||
assert "application-inference-profile%2F" in url_str, f"Expected encoded ARN in URL, but got: {url_str}"
|
||||
assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}"
|
||||
assert "/invoke" in url_str, "Expected /invoke action in URL"
|
||||
|
||||
# Verify the complete URL structure
|
||||
expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{model_id}/invoke"
|
||||
# Verify the complete URL structure with encoded ARN
|
||||
encoded_model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile%2Fabcdefgh1234"
|
||||
expected_url = f"https://bedrock-runtime.eu-west-1.amazonaws.com/model/{encoded_model_id}/invoke"
|
||||
assert url_str == expected_url, f"Expected {expected_url}, but got: {url_str}"
|
||||
|
||||
|
||||
def test_bedrock_passthrough_with_inference_profile_converse_endpoint():
|
||||
"""Test Application Inference Profile with converse endpoint"""
|
||||
"""Test Application Inference Profile with converse endpoint and proper ARN encoding"""
|
||||
config = BedrockPassthroughConfig()
|
||||
|
||||
model = "anthropic.claude-sonnet-4-20250514-v1:0"
|
||||
|
|
@ -239,7 +241,8 @@ def test_bedrock_passthrough_with_inference_profile_converse_endpoint():
|
|||
)
|
||||
|
||||
url_str = str(url)
|
||||
assert model_id in url_str
|
||||
# The ARN should be encoded with %2F
|
||||
assert "application-inference-profile%2F" in url_str
|
||||
assert "/converse" in url_str
|
||||
assert model not in url_str
|
||||
|
||||
|
|
@ -304,3 +307,125 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn():
|
|||
# Verify that the region from ARN is used in the base URL
|
||||
assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}"
|
||||
|
||||
|
||||
def test_bedrock_passthrough_model_id_arn_encoding():
|
||||
"""
|
||||
Test that model_id ARNs are properly URL-encoded when used in endpoints.
|
||||
|
||||
This is the critical fix for the issue where ARNs with slashes need to be encoded
|
||||
so they're treated as a single path component rather than multiple path segments.
|
||||
|
||||
For example:
|
||||
arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7
|
||||
should become:
|
||||
arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7
|
||||
"""
|
||||
config = BedrockPassthroughConfig()
|
||||
|
||||
model = "bedrock-claude-4-5-sonnet"
|
||||
# ARN with a slash that needs encoding
|
||||
model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile/b943q2qbl3m7"
|
||||
endpoint = f"/model/{model}/converse"
|
||||
|
||||
with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \
|
||||
patch.object(config, 'get_runtime_endpoint', return_value=(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
)):
|
||||
|
||||
url, api_base = config.get_complete_url(
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
request_query_params=None,
|
||||
litellm_params={"model_id": model_id}
|
||||
)
|
||||
|
||||
url_str = str(url)
|
||||
|
||||
# The slash in the ARN after application-inference-profile should be encoded as %2F
|
||||
assert "application-inference-profile%2F" in url_str, \
|
||||
f"Expected encoded ARN with %2F in URL, but got: {url_str}"
|
||||
|
||||
# The unencoded version should NOT be in the URL
|
||||
assert "application-inference-profile/" not in url_str, \
|
||||
f"ARN slash should be encoded, but found unencoded version in: {url_str}"
|
||||
|
||||
# Verify the complete expected URL structure
|
||||
expected_encoded_model_id = "arn:aws:bedrock:us-east-1:590183661440:application-inference-profile%2Fb943q2qbl3m7"
|
||||
expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/converse"
|
||||
assert url_str == expected_url, f"Expected {expected_url}, but got: {url_str}"
|
||||
|
||||
|
||||
def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint():
|
||||
"""
|
||||
Test ARN encoding with /invoke endpoint (not just /converse).
|
||||
"""
|
||||
config = BedrockPassthroughConfig()
|
||||
|
||||
model = "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789"
|
||||
endpoint = f"/model/{model}/invoke"
|
||||
|
||||
with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \
|
||||
patch.object(config, 'get_runtime_endpoint', return_value=(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
)):
|
||||
|
||||
url, api_base = config.get_complete_url(
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
request_query_params=None,
|
||||
litellm_params={"model_id": model_id}
|
||||
)
|
||||
|
||||
url_str = str(url)
|
||||
|
||||
# Verify encoding
|
||||
assert "application-inference-profile%2F" in url_str
|
||||
assert "/invoke" in url_str
|
||||
|
||||
expected_encoded_model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789"
|
||||
expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/invoke"
|
||||
assert url_str == expected_url
|
||||
|
||||
|
||||
def test_bedrock_passthrough_model_id_without_arn():
|
||||
"""
|
||||
Test that non-ARN model_ids (regular model IDs) are not affected by encoding logic.
|
||||
"""
|
||||
config = BedrockPassthroughConfig()
|
||||
|
||||
model = "my-model"
|
||||
# Regular model ID (not an ARN)
|
||||
model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
|
||||
endpoint = f"/model/{model}/converse"
|
||||
|
||||
with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \
|
||||
patch.object(config, 'get_runtime_endpoint', return_value=(
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
"https://bedrock-runtime.us-east-1.amazonaws.com"
|
||||
)):
|
||||
|
||||
url, api_base = config.get_complete_url(
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
model=model,
|
||||
endpoint=endpoint,
|
||||
request_query_params=None,
|
||||
litellm_params={"model_id": model_id}
|
||||
)
|
||||
|
||||
url_str = str(url)
|
||||
|
||||
# Regular model ID should be used as-is (no encoding needed)
|
||||
assert model_id in url_str
|
||||
assert "%2F" not in url_str, "Non-ARN model IDs should not be encoded"
|
||||
|
||||
expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse"
|
||||
assert url_str == expected_url
|
||||
|
||||
|
|
|
|||
|
|
@ -582,7 +582,8 @@ def test_eks_irsa_ambient_credentials_used():
|
|||
)
|
||||
|
||||
# Should create STS client without explicit credentials (using ambient credentials)
|
||||
mock_boto3_client.assert_called_once_with("sts")
|
||||
# Note: verify parameter is passed for SSL verification
|
||||
mock_boto3_client.assert_called_once_with("sts", verify=True)
|
||||
|
||||
# Should call assume_role
|
||||
mock_sts_client.assume_role.assert_called_once_with(
|
||||
|
|
@ -637,11 +638,13 @@ def test_explicit_credentials_used_when_provided():
|
|||
)
|
||||
|
||||
# Should create STS client with explicit credentials
|
||||
# Note: verify parameter is passed for SSL verification
|
||||
mock_boto3_client.assert_called_once_with(
|
||||
"sts",
|
||||
aws_access_key_id="explicit-access-key",
|
||||
aws_secret_access_key="explicit-secret-key",
|
||||
aws_session_token="assumed-session-token",
|
||||
verify=True,
|
||||
)
|
||||
|
||||
# Should call assume_role
|
||||
|
|
@ -701,6 +704,7 @@ def test_partial_credentials_still_use_ambient():
|
|||
aws_access_key_id="AKIAEXAMPLE",
|
||||
aws_secret_access_key=None,
|
||||
aws_session_token=None,
|
||||
verify=True,
|
||||
)
|
||||
|
||||
# Should still call assume_role
|
||||
|
|
@ -748,7 +752,7 @@ def test_cross_account_role_assumption():
|
|||
)
|
||||
|
||||
# Should use ambient credentials
|
||||
mock_boto3_client.assert_called_once_with("sts")
|
||||
mock_boto3_client.assert_called_once_with("sts", verify=True)
|
||||
|
||||
# Should call assume_role with cross-account role
|
||||
mock_sts_client.assume_role.assert_called_once_with(
|
||||
|
|
|
|||
349
tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py
Normal file
349
tests/test_litellm/llms/bedrock/test_bedrock_ssl_verify.py
Normal file
|
|
@ -0,0 +1,349 @@
|
|||
"""
|
||||
Test SSL verification for AWS Bedrock boto3 clients.
|
||||
|
||||
This test ensures that custom CA certificates are properly passed to all boto3 clients
|
||||
(STS and Bedrock services) to support internal certificate authorities.
|
||||
|
||||
Issue: https://github.com/BerriAI/litellm/issues/XXXX
|
||||
User reported that SSL_CERT_FILE environment variable and ssl_verify config were not
|
||||
being applied to boto3 clients, causing "certificate verify failed" errors.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import init_bedrock_client
|
||||
|
||||
|
||||
class TestBedrockSSLVerify:
|
||||
"""Test suite for SSL verification in Bedrock boto3 clients."""
|
||||
|
||||
def test_base_aws_llm_get_ssl_verify_default(self):
|
||||
"""Test that _get_ssl_verify returns default value when no custom config is set."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Clear any environment variables
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
|
||||
# Reset litellm.ssl_verify to default
|
||||
litellm.ssl_verify = True
|
||||
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
assert ssl_verify is True
|
||||
|
||||
def test_base_aws_llm_get_ssl_verify_false(self):
|
||||
"""Test that _get_ssl_verify returns False when SSL verification is disabled."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Set SSL_VERIFY to False via environment
|
||||
os.environ["SSL_VERIFY"] = "False"
|
||||
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
assert ssl_verify is False
|
||||
|
||||
# Clean up
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
|
||||
def test_base_aws_llm_get_ssl_verify_custom_ca_bundle(self):
|
||||
"""Test that _get_ssl_verify returns custom CA bundle path when SSL_CERT_FILE is set."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set SSL_CERT_FILE environment variable
|
||||
os.environ["SSL_CERT_FILE"] = ca_bundle_path
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
litellm.ssl_verify = True
|
||||
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
assert ssl_verify == ca_bundle_path
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
def test_base_aws_llm_get_ssl_verify_litellm_config(self):
|
||||
"""Test that _get_ssl_verify uses litellm.ssl_verify when set."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Clear environment variables
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set litellm.ssl_verify to custom CA bundle
|
||||
litellm.ssl_verify = ca_bundle_path
|
||||
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
# When ssl_verify is a path, it should be returned directly
|
||||
assert ssl_verify == ca_bundle_path
|
||||
finally:
|
||||
# Clean up
|
||||
litellm.ssl_verify = True
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
@patch("boto3.client")
|
||||
def test_init_bedrock_client_passes_ssl_verify_to_sts(self, mock_boto3_client):
|
||||
"""Test that init_bedrock_client passes ssl_verify to STS client."""
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set SSL_CERT_FILE environment variable
|
||||
os.environ["SSL_CERT_FILE"] = ca_bundle_path
|
||||
litellm.ssl_verify = True
|
||||
|
||||
# Mock the STS client and Bedrock client
|
||||
mock_sts_client = MagicMock()
|
||||
mock_sts_response = {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "test_access_key",
|
||||
"SecretAccessKey": "test_secret_key",
|
||||
"SessionToken": "test_session_token",
|
||||
}
|
||||
}
|
||||
mock_sts_client.assume_role.return_value = mock_sts_response
|
||||
|
||||
mock_bedrock_client = MagicMock()
|
||||
|
||||
# Configure mock to return different clients based on service name
|
||||
def side_effect(service_name=None, **kwargs):
|
||||
if service_name == "sts":
|
||||
return mock_sts_client
|
||||
elif service_name == "bedrock-runtime":
|
||||
return mock_bedrock_client
|
||||
return MagicMock()
|
||||
|
||||
mock_boto3_client.side_effect = side_effect
|
||||
|
||||
# Call init_bedrock_client with role assumption
|
||||
client = init_bedrock_client(
|
||||
aws_region_name="us-west-2",
|
||||
aws_access_key_id="test_key",
|
||||
aws_secret_access_key="test_secret",
|
||||
aws_role_name="arn:aws:iam::123456789012:role/test-role",
|
||||
aws_session_name="test-session",
|
||||
)
|
||||
|
||||
# Verify that boto3.client was called with verify parameter for STS
|
||||
sts_calls = [
|
||||
call for call in mock_boto3_client.call_args_list
|
||||
if (len(call[0]) > 0 and call[0][0] == "sts") or
|
||||
("service_name" not in call[1]) # STS calls don't use service_name kwarg
|
||||
]
|
||||
|
||||
assert len(sts_calls) > 0, "STS client should have been created"
|
||||
|
||||
# Check that verify parameter was passed to STS client
|
||||
sts_call = sts_calls[0]
|
||||
assert "verify" in sts_call[1], "verify parameter should be passed to STS client"
|
||||
assert sts_call[1]["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {sts_call[1]['verify']}"
|
||||
|
||||
# Verify that boto3.client was called with verify parameter for Bedrock
|
||||
bedrock_calls = [
|
||||
call for call in mock_boto3_client.call_args_list
|
||||
if "service_name" in call[1] and call[1]["service_name"] == "bedrock-runtime"
|
||||
]
|
||||
|
||||
assert len(bedrock_calls) > 0, "Bedrock client should have been created"
|
||||
|
||||
bedrock_call = bedrock_calls[0]
|
||||
assert "verify" in bedrock_call[1], "verify parameter should be passed to Bedrock client"
|
||||
assert bedrock_call[1]["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {bedrock_call[1]['verify']}"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
@patch("boto3.client")
|
||||
def test_base_aws_llm_auth_with_role_passes_ssl_verify(self, mock_boto3_client):
|
||||
"""Test that _auth_with_aws_role passes ssl_verify to STS client."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set SSL_CERT_FILE environment variable
|
||||
os.environ["SSL_CERT_FILE"] = ca_bundle_path
|
||||
litellm.ssl_verify = True
|
||||
|
||||
# Mock the STS client
|
||||
mock_sts_client = MagicMock()
|
||||
mock_sts_response = {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "test_access_key",
|
||||
"SecretAccessKey": "test_secret_key",
|
||||
"SessionToken": "test_session_token",
|
||||
"Expiration": "2025-01-10T00:00:00Z",
|
||||
}
|
||||
}
|
||||
|
||||
# Convert Expiration to datetime
|
||||
from datetime import datetime, timezone
|
||||
mock_sts_response["Credentials"]["Expiration"] = datetime.now(timezone.utc)
|
||||
|
||||
mock_sts_client.assume_role.return_value = mock_sts_response
|
||||
mock_boto3_client.return_value = mock_sts_client
|
||||
|
||||
# Call _auth_with_aws_role
|
||||
credentials, ttl = base_aws._auth_with_aws_role(
|
||||
aws_access_key_id="test_key",
|
||||
aws_secret_access_key="test_secret",
|
||||
aws_session_token=None,
|
||||
aws_role_name="arn:aws:iam::123456789012:role/test-role",
|
||||
aws_session_name="test-session",
|
||||
)
|
||||
|
||||
# Verify that boto3.client was called with verify parameter
|
||||
assert mock_boto3_client.called, "boto3.client should have been called"
|
||||
|
||||
call_kwargs = mock_boto3_client.call_args[1]
|
||||
assert "verify" in call_kwargs, "verify parameter should be passed to STS client"
|
||||
assert call_kwargs["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {call_kwargs['verify']}"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
@patch("litellm.llms.bedrock.base_aws_llm.get_secret")
|
||||
@patch("boto3.client")
|
||||
def test_base_aws_llm_auth_with_web_identity_passes_ssl_verify(self, mock_boto3_client, mock_get_secret):
|
||||
"""Test that _auth_with_web_identity_token passes ssl_verify to STS client."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set SSL_CERT_FILE environment variable
|
||||
os.environ["SSL_CERT_FILE"] = ca_bundle_path
|
||||
litellm.ssl_verify = True
|
||||
|
||||
# Mock get_secret to return the token
|
||||
mock_get_secret.return_value = "mocked_oidc_token"
|
||||
|
||||
# Mock the STS client
|
||||
mock_sts_client = MagicMock()
|
||||
mock_sts_response = {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "test_access_key",
|
||||
"SecretAccessKey": "test_secret_key",
|
||||
"SessionToken": "test_session_token",
|
||||
},
|
||||
"PackedPolicySize": 100,
|
||||
}
|
||||
|
||||
mock_sts_client.assume_role_with_web_identity.return_value = mock_sts_response
|
||||
|
||||
# Mock boto3.Session
|
||||
mock_session = MagicMock()
|
||||
mock_credentials = MagicMock()
|
||||
mock_session.get_credentials.return_value = mock_credentials
|
||||
|
||||
mock_boto3_client.return_value = mock_sts_client
|
||||
|
||||
with patch("boto3.Session", return_value=mock_session):
|
||||
# Call _auth_with_web_identity_token
|
||||
credentials, ttl = base_aws._auth_with_web_identity_token(
|
||||
aws_web_identity_token="test_token",
|
||||
aws_role_name="arn:aws:iam::123456789012:role/test-role",
|
||||
aws_session_name="test-session",
|
||||
aws_region_name="us-west-2",
|
||||
aws_sts_endpoint=None,
|
||||
)
|
||||
|
||||
# Verify that boto3.client was called with verify parameter
|
||||
assert mock_boto3_client.called, "boto3.client should have been called"
|
||||
|
||||
call_kwargs = mock_boto3_client.call_args[1]
|
||||
assert "verify" in call_kwargs, "verify parameter should be passed to STS client"
|
||||
assert call_kwargs["verify"] == ca_bundle_path, f"verify should be set to CA bundle path, got {call_kwargs['verify']}"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
def test_ssl_verify_priority_env_over_litellm_config(self):
|
||||
"""Test that SSL_VERIFY environment variable takes priority over litellm.ssl_verify."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Set litellm.ssl_verify to True
|
||||
litellm.ssl_verify = True
|
||||
|
||||
# Set SSL_VERIFY environment variable to False
|
||||
os.environ["SSL_VERIFY"] = "False"
|
||||
|
||||
try:
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
assert ssl_verify is False, "Environment variable should take priority"
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
litellm.ssl_verify = True
|
||||
|
||||
def test_ssl_cert_file_priority_over_default(self):
|
||||
"""Test that SSL_CERT_FILE takes priority when ssl_verify is True."""
|
||||
base_aws = BaseAWSLLM()
|
||||
|
||||
# Create a temporary CA bundle file
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as f:
|
||||
f.write("-----BEGIN CERTIFICATE-----\n")
|
||||
f.write("FAKE CERTIFICATE FOR TESTING\n")
|
||||
f.write("-----END CERTIFICATE-----\n")
|
||||
ca_bundle_path = f.name
|
||||
|
||||
try:
|
||||
# Set SSL_CERT_FILE environment variable
|
||||
os.environ["SSL_CERT_FILE"] = ca_bundle_path
|
||||
os.environ.pop("SSL_VERIFY", None)
|
||||
litellm.ssl_verify = True
|
||||
|
||||
ssl_verify = base_aws._get_ssl_verify()
|
||||
assert ssl_verify == ca_bundle_path, "SSL_CERT_FILE should be used when ssl_verify is True"
|
||||
finally:
|
||||
# Clean up
|
||||
os.environ.pop("SSL_CERT_FILE", None)
|
||||
os.unlink(ca_bundle_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run tests
|
||||
pytest.main([__file__, "-v", "-s"])
|
||||
|
|
@ -14,6 +14,7 @@ from litellm.llms.vertex_ai.common_utils import VertexAIError
|
|||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
|
||||
from litellm.types.llms.vertex_ai import UsageMetadata
|
||||
from litellm.types.utils import ChoiceLogprobs, Usage
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
|
@ -2315,6 +2316,17 @@ def test_partial_json_chunk_on_first_chunk():
|
|||
assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode"
|
||||
|
||||
|
||||
|
||||
def test_google_ai_studio_presence_penalty_supported():
|
||||
"""
|
||||
Test that presence_penalty is supported for Google AI Studio Gemini.
|
||||
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/14753
|
||||
"""
|
||||
config = GoogleAIStudioGeminiConfig()
|
||||
supported_params = config.get_supported_openai_params(model="gemini-2.0-flash")
|
||||
|
||||
assert "presence_penalty" in supported_params
|
||||
# ==================== Tool Type Separation Tests ====================
|
||||
# These tests verify that each Tool object contains exactly one type per Vertex AI API spec
|
||||
# Ref: https://cloud.google.com/vertex-ai/generative-ai/docs/reference/rest/v1beta1/Tool
|
||||
|
|
@ -2505,3 +2517,72 @@ def test_vertex_ai_multiple_function_declarations_grouped():
|
|||
func_names = [f["name"] for f in tools[0]["function_declarations"]]
|
||||
assert "func1" in func_names
|
||||
assert "func2" in func_names
|
||||
|
||||
|
||||
def test_gemini_3_flash_preview_token_usage_fallback():
|
||||
"""Test fallback logic when candidatesTokensDetails is missing (e.g. Gemini 3 Flash Preview)."""
|
||||
v = VertexGeminiConfig()
|
||||
|
||||
usage_metadata_dict = {
|
||||
"promptTokenCount": 2145,
|
||||
"candidatesTokenCount": 509,
|
||||
"totalTokenCount": 2654,
|
||||
# candidatesTokensDetails intentionally omitted
|
||||
}
|
||||
|
||||
completion_response = {"usageMetadata": usage_metadata_dict}
|
||||
result = v._calculate_usage(completion_response=completion_response)
|
||||
|
||||
assert result.completion_tokens == 509
|
||||
assert result.prompt_tokens == 2145
|
||||
assert result.total_tokens == 2654
|
||||
|
||||
# Text tokens should be derived from candidatesTokenCount
|
||||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.text_tokens == 509
|
||||
assert result.completion_tokens_details.image_tokens is None
|
||||
assert result.completion_tokens_details.audio_tokens is None
|
||||
|
||||
|
||||
def test_gemini_no_reasoning_fallback():
|
||||
"""Test fallback when reasoning_effort is absent and details are missing."""
|
||||
v = VertexGeminiConfig()
|
||||
|
||||
usage_metadata_dict = {
|
||||
"promptTokenCount": 100,
|
||||
"candidatesTokenCount": 264,
|
||||
"totalTokenCount": 364,
|
||||
}
|
||||
|
||||
completion_response = {"usageMetadata": usage_metadata_dict}
|
||||
result = v._calculate_usage(completion_response=completion_response)
|
||||
|
||||
assert result.completion_tokens == 264
|
||||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.text_tokens == 264
|
||||
assert (
|
||||
result.completion_tokens_details.reasoning_tokens is None
|
||||
or result.completion_tokens_details.reasoning_tokens == 0
|
||||
)
|
||||
|
||||
|
||||
def test_gemini_token_usage_standard_response():
|
||||
"""Verify that standard responses with details are computed correctly and not overwritten."""
|
||||
v = VertexGeminiConfig()
|
||||
|
||||
usage_metadata_dict = {
|
||||
"promptTokenCount": 100,
|
||||
"candidatesTokenCount": 50,
|
||||
"totalTokenCount": 150,
|
||||
"candidatesTokensDetails": [
|
||||
{"modality": "TEXT", "tokenCount": 40},
|
||||
{"modality": "IMAGE", "tokenCount": 10},
|
||||
],
|
||||
}
|
||||
|
||||
completion_response = {"usageMetadata": usage_metadata_dict}
|
||||
result = v._calculate_usage(completion_response=completion_response)
|
||||
|
||||
assert result.completion_tokens == 50
|
||||
assert result.completion_tokens_details.text_tokens == 40
|
||||
assert result.completion_tokens_details.image_tokens == 10
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue