From 8a26033a4ba25a136c353f4c4da75d7dafb6c41f Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 29 Jan 2026 00:42:51 -0300 Subject: [PATCH] fix(vertex_ai): convert image URLs to base64 in tool messages for Anthropic (#19896) * fix(vertex_ai): convert image URLs to base64 in tool messages for Anthropic Fixes #19891 Vertex AI Anthropic models don't support URL sources for images. LiteLLM already converted image URLs to base64 for user messages, but not for tool messages (role='tool'). This caused errors when using ToolOutputImage with image_url in tool outputs. Changes: - Add force_base64 parameter to convert_to_anthropic_tool_result() - Pass force_base64 to create_anthropic_image_param() for tool message images - Calculate force_base64 in anthropic_messages_pt() based on llm_provider - Add unit tests for tool message image handling * chore: remove extra comment from test file header --- .../prompt_templates/factory.py | 13 +- ..._vertex_ai_anthropic_image_url_handling.py | 192 ++++++++++++++++++ 2 files changed, 203 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 03488ad0183..98ee5e4fa86 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1632,6 +1632,7 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: def convert_to_anthropic_tool_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], + force_base64: bool = False, ) -> AnthropicMessagesToolResultParam: """ OpenAI message with a tool result looks like: @@ -1694,7 +1695,7 @@ def convert_to_anthropic_tool_result( else None ) _anthropic_image_param = create_anthropic_image_param( - content["image_url"], format=format + content["image_url"], format=format, is_bedrock_invoke=force_base64 ) _anthropic_image_param = add_cache_control_to_content( anthropic_content_element=_anthropic_image_param, @@ -2056,6 +2057,12 @@ def anthropic_messages_pt( # noqa: PLR0915 else: messages.append(DEFAULT_USER_CONTINUE_MESSAGE_TYPED) + # Bedrock invoke models have format: invoke/... + # Vertex AI Anthropic also doesn't support URL sources for images + is_bedrock_invoke = model.lower().startswith("invoke/") + is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False + force_base64 = is_bedrock_invoke or is_vertex_ai + msg_i = 0 while msg_i < len(messages): user_content: List[AnthropicMessagesUserMessageValues] = [] @@ -2165,7 +2172,9 @@ def anthropic_messages_pt( # noqa: PLR0915 ): # OpenAI's tool message content will always be a string user_content.append( - convert_to_anthropic_tool_result(user_message_types_block) + convert_to_anthropic_tool_result( + user_message_types_block, force_base64=force_base64 + ) ) msg_i += 1 diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index fca784342d7..3f014d65d4d 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -17,6 +17,7 @@ sys.path.insert( from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, + convert_to_anthropic_tool_result, create_anthropic_image_param, ) @@ -177,3 +178,194 @@ class TestCreateAnthropicImageParam: mock_convert_url.assert_not_called() assert result["source"]["type"] == "url" assert result["source"]["url"] == "https://example.com/image.jpg" + + +class TestToolMessageImageURLHandling: + """ + Test that tool messages with image_url are converted to base64 for Vertex AI. + + Issue: https://github.com/BerriAI/litellm/issues/19891 + """ + + @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") + def test_convert_to_anthropic_tool_result_with_force_base64( + self, mock_convert_url: MagicMock + ): + """ + Test that convert_to_anthropic_tool_result converts image URLs to base64 + when force_base64=True. + """ + mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg==" + + tool_message = { + "role": "tool", + "tool_call_id": "call_123", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/tool_result.jpg"}, + } + ], + } + + result = convert_to_anthropic_tool_result(tool_message, force_base64=True) + + mock_convert_url.assert_called_once_with(url="https://example.com/tool_result.jpg") + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "call_123" + + # Check the image content is base64 + content = result["content"] + assert len(content) == 1 + assert content[0]["type"] == "image" + assert content[0]["source"]["type"] == "base64" + + @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") + def test_convert_to_anthropic_tool_result_without_force_base64( + self, mock_convert_url: MagicMock + ): + """ + Test that convert_to_anthropic_tool_result uses URL type when force_base64=False. + """ + tool_message = { + "role": "tool", + "tool_call_id": "call_456", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + } + ], + } + + result = convert_to_anthropic_tool_result(tool_message, force_base64=False) + + mock_convert_url.assert_not_called() + assert result["type"] == "tool_result" + + # Check the image content uses URL type + content = result["content"] + assert len(content) == 1 + assert content[0]["type"] == "image" + assert content[0]["source"]["type"] == "url" + + @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") + def test_vertex_ai_tool_message_converts_image_to_base64( + self, mock_convert_url: MagicMock + ): + """ + Test full conversation with tool result containing image for Vertex AI. + The image URL should be converted to base64. + """ + mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg==" + + messages = [ + { + "role": "user", + "content": "Get me an image and describe it", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_789", + "type": "function", + "function": { + "name": "get_image", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_789", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/result.jpg"}, + } + ], + }, + ] + + result = anthropic_messages_pt( + messages=messages, + model="claude-sonnet-4", + llm_provider="vertex_ai", + ) + + # Verify convert_url_to_base64 was called for the tool result image + mock_convert_url.assert_called_once_with(url="https://example.com/result.jpg") + + # Find the tool_result in the converted messages + for msg in result: + if msg.get("role") == "user": + for content_item in msg.get("content", []): + if isinstance(content_item, dict) and content_item.get("type") == "tool_result": + tool_content = content_item.get("content", []) + for item in tool_content: + if isinstance(item, dict) and item.get("type") == "image": + assert item["source"]["type"] == "base64" + return + pytest.fail("Could not find image in tool result") + + @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") + def test_regular_anthropic_tool_message_uses_url( + self, mock_convert_url: MagicMock + ): + """ + Test that regular Anthropic API uses URL type for tool result images. + """ + messages = [ + { + "role": "user", + "content": "Get me an image", + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc", + "type": "function", + "function": { + "name": "get_image", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_abc", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + } + ], + }, + ] + + result = anthropic_messages_pt( + messages=messages, + model="claude-sonnet-4", + llm_provider="anthropic", + ) + + # convert_url_to_base64 should NOT be called for regular Anthropic + mock_convert_url.assert_not_called() + + # Find the tool_result and verify URL type + for msg in result: + if msg.get("role") == "user": + for content_item in msg.get("content", []): + if isinstance(content_item, dict) and content_item.get("type") == "tool_result": + tool_content = content_item.get("content", []) + for item in tool_content: + if isinstance(item, dict) and item.get("type") == "image": + assert item["source"]["type"] == "url" + return + pytest.fail("Could not find image in tool result")