diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 6fc319c26ae..7efa0eaba0f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -843,7 +843,7 @@ def create_anthropic_image_param( if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 # as these providers don't support URL sources for images - if is_bedrock_invoke or image_url.startswith("http://"): + if is_bedrock_invoke: base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) return AnthropicMessagesImageParam( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index b7c2ce3c568..d198f6f9e6f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -29,7 +29,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, async_inline_remote_media, - inline_remote_image_urls, ) from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -1866,8 +1865,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): break return headers - def inlines_remote_media(self, media: RemoteMedia) -> bool: - return inline_remote_image_urls(media) and media.url.startswith("http://") + def inlines_remote_media(self, _media: RemoteMedia) -> bool: + return False async def async_transform_request( self, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 7b03736920b..75b4f584ef4 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -225,6 +225,41 @@ def test_create_anthropic_image_param_with_https_url(): assert image_param["source"]["url"] == "https://example.com/image.png" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_forwards_url( + mock_convert_url: MagicMock, +): + """Plain http:// image URLs are forwarded as URL sources, like https://. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None + ) + + mock_convert_url.assert_not_called() + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "url" + assert image_param["source"]["url"] == "http://example.com/image.png" + + +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock( + mock_convert_url: MagicMock, +): + """Bedrock invoke keeps converting URLs to base64 (provider has no URL source).""" + mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg==" + + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None, is_bedrock_invoke=True + ) + + mock_convert_url.assert_called_once_with(url="http://example.com/image.png") + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "base64" + + def test_create_anthropic_image_param_with_dict_input(): """Test that dict input with URL is handled correctly.""" image_param = create_anthropic_image_param( @@ -292,6 +327,37 @@ def test_anthropic_messages_pt_with_url_image(): assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock): + """Plain http:// image URLs are forwarded as URL sources for regular Anthropic. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "http://example.com/image.jpg", + }, + ], + } + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" + ) + + mock_convert_url.assert_not_called() + assert len(result) == 1 + image_content = result[0]["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "url" + assert image_content["source"]["url"] == "http://example.com/image.jpg" + + def test_anthropic_messages_pt_with_base64_image(): """Test that anthropic_messages_pt correctly handles data URIs as base64.""" messages = [ diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 7167a67d80d..e3d0f6c1b10 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -17,6 +17,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) +from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -6519,3 +6520,17 @@ def test_eager_input_streaming_reaches_anthropic_request_tools(): assert result["tools"][0]["eager_input_streaming"] is True assert result["tools"][0]["name"] == "write_file" + + +@pytest.mark.parametrize( + "url", ["http://example.com/image.png", "https://example.com/image.png"] +) +def test_inlines_remote_media_never_inlines_image_urls(url: str): + """Regular Anthropic forwards remote image URLs as url sources, so the async + request path must not download and inline them. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + media = RemoteMedia(url=url, fields={}, part_type="image_url") + + assert AnthropicConfig().inlines_remote_media(media) is False