scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(vertex_ai): convert image URLs to base64 in tool messages for Anthropic (#19896)

Browse source 
* fix(vertex_ai): convert image URLs to base64 in tool messages for Anthropic

Fixes #19891

Vertex AI Anthropic models don't support URL sources for images. LiteLLM
already converted image URLs to base64 for user messages, but not for tool
messages (role='tool'). This caused errors when using ToolOutputImage with
image_url in tool outputs.

Changes:
- Add force_base64 parameter to convert_to_anthropic_tool_result()
- Pass force_base64 to create_anthropic_image_param() for tool message images
- Calculate force_base64 in anthropic_messages_pt() based on llm_provider
- Add unit tests for tool message image handling

* chore: remove extra comment from test file header
This commit is contained in:
Cesar Garcia 2026-01-29 00:42:51 -03:00 • committed by GitHub
parent a785eecf7a
commit 8a26033a4b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 203 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1632,6 +1632,7 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
def convert_to_anthropic_tool_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
force_base64: bool = False,
) -> AnthropicMessagesToolResultParam:
"""
OpenAI message with a tool result looks like:
@ -1694,7 +1695,7 @@ def convert_to_anthropic_tool_result(
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format
content["image_url"], format=format, is_bedrock_invoke=force_base64
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
@ -2056,6 +2057,12 @@ def anthropic_messages_pt( # noqa: PLR0915
else:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE_TYPED)
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
msg_i = 0
while msg_i < len(messages):
user_content: List[AnthropicMessagesUserMessageValues] = []
@ -2165,7 +2172,9 @@ def anthropic_messages_pt( # noqa: PLR0915
):
# OpenAI's tool message content will always be a string
user_content.append(
convert_to_anthropic_tool_result(user_message_types_block)
convert_to_anthropic_tool_result(
user_message_types_block, force_base64=force_base64
)
)
msg_i += 1

192
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py
View file

@ -17,6 +17,7 @@ sys.path.insert(
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
convert_to_anthropic_tool_result,
create_anthropic_image_param,
)
@ -177,3 +178,194 @@ class TestCreateAnthropicImageParam:
mock_convert_url.assert_not_called()
assert result["source"]["type"] == "url"
assert result["source"]["url"] == "https://example.com/image.jpg"
class TestToolMessageImageURLHandling:
"""
Test that tool messages with image_url are converted to base64 for Vertex AI.
Issue: https://github.com/BerriAI/litellm/issues/19891
"""
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_convert_to_anthropic_tool_result_with_force_base64(
self, mock_convert_url: MagicMock
):
"""
Test that convert_to_anthropic_tool_result converts image URLs to base64
when force_base64=True.
"""
mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg=="
tool_message = {
"role": "tool",
"tool_call_id": "call_123",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/tool_result.jpg"},
}
],
}
result = convert_to_anthropic_tool_result(tool_message, force_base64=True)
mock_convert_url.assert_called_once_with(url="https://example.com/tool_result.jpg")
assert result["type"] == "tool_result"
assert result["tool_use_id"] == "call_123"
# Check the image content is base64
content = result["content"]
assert len(content) == 1
assert content[0]["type"] == "image"
assert content[0]["source"]["type"] == "base64"
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_convert_to_anthropic_tool_result_without_force_base64(
self, mock_convert_url: MagicMock
):
"""
Test that convert_to_anthropic_tool_result uses URL type when force_base64=False.
"""
tool_message = {
"role": "tool",
"tool_call_id": "call_456",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.jpg"},
}
],
}
result = convert_to_anthropic_tool_result(tool_message, force_base64=False)
mock_convert_url.assert_not_called()
assert result["type"] == "tool_result"
# Check the image content uses URL type
content = result["content"]
assert len(content) == 1
assert content[0]["type"] == "image"
assert content[0]["source"]["type"] == "url"
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_vertex_ai_tool_message_converts_image_to_base64(
self, mock_convert_url: MagicMock
):
"""
Test full conversation with tool result containing image for Vertex AI.
The image URL should be converted to base64.
"""
mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg=="
messages = [
{
"role": "user",
"content": "Get me an image and describe it",
},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_789",
"type": "function",
"function": {
"name": "get_image",
"arguments": "{}",
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_789",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/result.jpg"},
}
],
},
]
result = anthropic_messages_pt(
messages=messages,
model="claude-sonnet-4",
llm_provider="vertex_ai",
)
# Verify convert_url_to_base64 was called for the tool result image
mock_convert_url.assert_called_once_with(url="https://example.com/result.jpg")
# Find the tool_result in the converted messages
for msg in result:
if msg.get("role") == "user":
for content_item in msg.get("content", []):
if isinstance(content_item, dict) and content_item.get("type") == "tool_result":
tool_content = content_item.get("content", [])
for item in tool_content:
if isinstance(item, dict) and item.get("type") == "image":
assert item["source"]["type"] == "base64"
return
pytest.fail("Could not find image in tool result")
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_regular_anthropic_tool_message_uses_url(
self, mock_convert_url: MagicMock
):
"""
Test that regular Anthropic API uses URL type for tool result images.
"""
messages = [
{
"role": "user",
"content": "Get me an image",
},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc",
"type": "function",
"function": {
"name": "get_image",
"arguments": "{}",
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_abc",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.jpg"},
}
],
},
]
result = anthropic_messages_pt(
messages=messages,
model="claude-sonnet-4",
llm_provider="anthropic",
)
# convert_url_to_base64 should NOT be called for regular Anthropic
mock_convert_url.assert_not_called()
# Find the tool_result and verify URL type
for msg in result:
if msg.get("role") == "user":
for content_item in msg.get("content", []):
if isinstance(content_item, dict) and content_item.get("type") == "tool_result":
tool_content = content_item.get("content", [])
for item in tool_content:
if isinstance(item, dict) and item.get("type") == "image":
assert item["source"]["type"] == "url"
return
pytest.fail("Could not find image in tool result")