Merge 53276be166 into a66adb4ff8

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -847,7 +847,7 @@ def create_anthropic_image_param(
     if image_url.startswith("http://") or image_url.startswith("https://"):
         # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
         # as these providers don't support URL sources for images
-        if is_bedrock_invoke or image_url.startswith("http://"):
+        if is_bedrock_invoke:
             base64_url: Final = convert_url_to_base64(url=image_url)
             image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
             return AnthropicMessagesImageParam(

litellm/llms/anthropic/chat/transformation.py

@@ -29,7 +29,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     RemoteMedia,
     async_inline_remote_media,
-    inline_remote_image_urls,
 )
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
     place_mid_conversation_system,
@@ -1876,8 +1875,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 break
         return headers
 
-    def inlines_remote_media(self, media: RemoteMedia) -> bool:
-        return inline_remote_image_urls(media) and media.url.startswith("http://")
+    def inlines_remote_media(self, _media: RemoteMedia) -> bool:
+        return False
 
     async def async_transform_request(
         self,

tests/llm_translation/test_prompt_factory.py

@@ -225,6 +225,41 @@ def test_create_anthropic_image_param_with_https_url():
     assert image_param["source"]["url"] == "https://example.com/image.png"
 
 
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_create_anthropic_image_param_with_http_url_forwards_url(
+    mock_convert_url: MagicMock,
+):
+    """Plain http:// image URLs are forwarded as URL sources, like https://.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098: the
+    proxy used to download http:// URLs and send them as base64 without notice.
+    """
+    image_param = create_anthropic_image_param(
+        "http://example.com/image.png", format=None
+    )
+
+    mock_convert_url.assert_not_called()
+    assert image_param["type"] == "image"
+    assert image_param["source"]["type"] == "url"
+    assert image_param["source"]["url"] == "http://example.com/image.png"
+
+
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock(
+    mock_convert_url: MagicMock,
+):
+    """Bedrock invoke keeps converting URLs to base64 (provider has no URL source)."""
+    mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg=="
+
+    image_param = create_anthropic_image_param(
+        "http://example.com/image.png", format=None, is_bedrock_invoke=True
+    )
+
+    mock_convert_url.assert_called_once_with(url="http://example.com/image.png")
+    assert image_param["type"] == "image"
+    assert image_param["source"]["type"] == "base64"
+
+
 def test_create_anthropic_image_param_with_dict_input():
     """Test that dict input with URL is handled correctly."""
     image_param = create_anthropic_image_param(
@@ -292,6 +327,37 @@ def test_anthropic_messages_pt_with_url_image():
     assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg"
 
 
+@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock):
+    """Plain http:// image URLs are forwarded as URL sources for regular Anthropic.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": "http://example.com/image.jpg",
+                },
+            ],
+        }
+    ]
+
+    result = anthropic_messages_pt(
+        messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic"
+    )
+
+    mock_convert_url.assert_not_called()
+    assert len(result) == 1
+    image_content = result[0]["content"][1]
+    assert image_content["type"] == "image"
+    assert image_content["source"]["type"] == "url"
+    assert image_content["source"]["url"] == "http://example.com/image.jpg"
+
+
 def test_anthropic_messages_pt_with_base64_image():
     """Test that anthropic_messages_pt correctly handles data URIs as base64."""
     messages = [

tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py

@@ -86,7 +86,12 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
     assert "default_api_key_tpm_limit" not in request_body
 
 
-async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch):
+async def test_anthropic_async_completion_forwards_http_images_as_url_sources(async_only_image_fetch):
+    """http:// image URLs are forwarded as URL sources, never downloaded.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098: the
+    proxy used to download http:// URLs and send them as base64 without notice.
+    """
     http_image_url = f"http://img.example/{uuid.uuid4()}.png"
     https_image_url = f"https://img.example/{uuid.uuid4()}.png"
     captured = {}
@@ -126,10 +131,10 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop
     )
 
     assert response.choices[0].message.content == "Green"
-    assert async_only_image_fetch.fetched == [http_image_url]
+    assert async_only_image_fetch.fetched == []
     sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"]
     assert sources == [
-        {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png},
+        {"type": "url", "url": http_image_url},
         {"type": "url", "url": https_image_url},
     ]
 

tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -17,7 +17,10 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
     RESPONSE_FORMAT_TOOL_NAME,
 )
-from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    encrypted_reasoning_signature,
+)
+from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.pass_through.messages.transformation import (
     AnthropicMessagesConfig,
@@ -6379,6 +6382,20 @@ def test_eager_input_streaming_reaches_anthropic_request_tools():
     assert result["tools"][0]["name"] == "write_file"
 
 
+@pytest.mark.parametrize(
+    "url", ["http://example.com/image.png", "https://example.com/image.png"]
+)
+def test_inlines_remote_media_never_inlines_image_urls(url: str):
+    """Regular Anthropic forwards remote image URLs as url sources, so the async
+    request path must not download and inline them.
+
+    Regression test for https://github.com/BerriAI/litellm/issues/43098.
+    """
+    media = RemoteMedia(url=url, fields={}, part_type="image_url")
+
+    assert AnthropicConfig().inlines_remote_media(media) is False
+
+
 # ---------------------------------------------------------------------------
 # Mid-conversation ``role: "system"`` on the chat completions path.
 #