fix(anthropic): forward http:// image URLs as url sources instead of downloading

OpenAI to Anthropic conversion downloaded image_url parts with http:// URLs
and sent them as base64, while https:// URLs were forwarded as url sources.
Both schemes are now forwarded as url sources. Bedrock invoke, Vertex AI and
Snowflake still convert to base64 since those providers have no URL source.

Fixes #43098

Signed-off-by: Junyi Yao <j.yao@wustl.edu>

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -843,7 +843,7 @@ def create_anthropic_image_param(
     if image_url.startswith("http://") or image_url.startswith("https://"):
         # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
         # as these providers don't support URL sources for images
-        if is_bedrock_invoke or image_url.startswith("http://"):
+        if is_bedrock_invoke:
             base64_url: Final = convert_url_to_base64(url=image_url)
             image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
             return AnthropicMessagesImageParam(

litellm/llms/anthropic/chat/transformation.py

@@ -29,7 +29,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     RemoteMedia,
     async_inline_remote_media,
-    inline_remote_image_urls,
 )
 from litellm.llms.base_llm.base_utils import type_to_response_format_param
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -1866,8 +1865,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 break
         return headers
 
-    def inlines_remote_media(self, media: RemoteMedia) -> bool:
-        return inline_remote_image_urls(media) and media.url.startswith("http://")
+    def inlines_remote_media(self, _media: RemoteMedia) -> bool:
+        return False
 
     async def async_transform_request(
         self,

tests/llm_translation/test_prompt_factory.py

@@ -225,6 +225,41 @@ def test_create_anthropic_image_param_with_https_url():
     assert image_param["source"]["url"] == "https://example.com/image.png"
 
 
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_create_anthropic_image_param_with_http_url_forwards_url(
+    mock_convert_url: MagicMock,
+):
+    """Plain http:// image URLs are forwarded as URL sources, like https://.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098: the
+    proxy used to download http:// URLs and send them as base64 without notice.
+    """
+    image_param = create_anthropic_image_param(
+        "http://example.com/image.png", format=None
+    )
+
+    mock_convert_url.assert_not_called()
+    assert image_param["type"] == "image"
+    assert image_param["source"]["type"] == "url"
+    assert image_param["source"]["url"] == "http://example.com/image.png"
+
+
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock(
+    mock_convert_url: MagicMock,
+):
+    """Bedrock invoke keeps converting URLs to base64 (provider has no URL source)."""
+    mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg=="
+
+    image_param = create_anthropic_image_param(
+        "http://example.com/image.png", format=None, is_bedrock_invoke=True
+    )
+
+    mock_convert_url.assert_called_once_with(url="http://example.com/image.png")
+    assert image_param["type"] == "image"
+    assert image_param["source"]["type"] == "base64"
+
+
 def test_create_anthropic_image_param_with_dict_input():
     """Test that dict input with URL is handled correctly."""
     image_param = create_anthropic_image_param(
@@ -292,6 +327,37 @@ def test_anthropic_messages_pt_with_url_image():
     assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg"
 
 
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock):
+    """Plain http:// image URLs are forwarded as URL sources for regular Anthropic.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": "http://example.com/image.jpg",
+                },
+            ],
+        }
+    ]
+
+    result = anthropic_messages_pt(
+        messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic"
+    )
+
+    mock_convert_url.assert_not_called()
+    assert len(result) == 1
+    image_content = result[0]["content"][1]
+    assert image_content["type"] == "image"
+    assert image_content["source"]["type"] == "url"
+    assert image_content["source"]["url"] == "http://example.com/image.jpg"
+
+
 def test_anthropic_messages_pt_with_base64_image():
     """Test that anthropic_messages_pt correctly handles data URIs as base64."""
     messages = [

tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -17,6 +17,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
     RESPONSE_FORMAT_TOOL_NAME,
 )
+from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
     AnthropicMessagesConfig,
@@ -6519,3 +6520,17 @@ def test_eager_input_streaming_reaches_anthropic_request_tools():
 
     assert result["tools"][0]["eager_input_streaming"] is True
     assert result["tools"][0]["name"] == "write_file"
+
+
+@pytest.mark.parametrize(
+    "url", ["http://example.com/image.png", "https://example.com/image.png"]
+)
+def test_inlines_remote_media_never_inlines_image_urls(url: str):
+    """Regular Anthropic forwards remote image URLs as url sources, so the async
+    request path must not download and inline them.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098.
+    """
+    media = RemoteMedia(url=url, fields={}, part_type="image_url")
+
+    assert AnthropicConfig().inlines_remote_media(media) is False