diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..a72a22dd9a0 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -930,7 +930,7 @@ def create_anthropic_image_param( if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 # as these providers don't support URL sources for images - if is_bedrock_invoke or image_url.startswith("http://"): + if is_bedrock_invoke: base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) return AnthropicMessagesImageParam( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..a89ba3f87fd 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -41,7 +41,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, async_inline_remote_media, - inline_remote_image_urls, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( place_mid_conversation_system, @@ -2232,8 +2231,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): break return headers - def inlines_remote_media(self, media: RemoteMedia) -> bool: - return inline_remote_image_urls(media) and media.url.startswith("http://") + def inlines_remote_media(self, _media: RemoteMedia) -> bool: + return False async def async_transform_request( self, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 53be3608979..83ad4535745 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -6258,6 +6258,42 @@ def test_create_anthropic_image_param_with_https_url(): assert image_param["source"]["url"] == "https://example.com/image.png" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_forwards_url( + mock_convert_url: MagicMock, +): + """Plain http:// image URLs are forwarded as URL sources, like https://. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None + ) + + mock_convert_url.assert_not_called() + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "url" + assert image_param["source"]["url"] == "http://example.com/image.png" + + + +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock( + mock_convert_url: MagicMock, +): + """Bedrock invoke keeps converting URLs to base64 (provider has no URL source).""" + mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg==" + + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None, is_bedrock_invoke=True + ) + + mock_convert_url.assert_called_once_with(url="http://example.com/image.png") + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "base64" + + def test_create_anthropic_image_param_with_dict_input(): """Test that dict input with URL is handled correctly.""" image_param = create_anthropic_image_param( @@ -6317,6 +6353,37 @@ def test_anthropic_messages_pt_with_url_image(): assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock): + """Plain http:// image URLs are forwarded as URL sources for regular Anthropic. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "http://example.com/image.jpg", + }, + ], + } + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" + ) + + mock_convert_url.assert_not_called() + assert len(result) == 1 + image_content = result[0]["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "url" + assert image_content["source"]["url"] == "http://example.com/image.jpg" + + def test_anthropic_messages_pt_with_base64_image(): """Test that anthropic_messages_pt correctly handles data URIs as base64.""" messages = [ diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 355d407001c..391b967118d 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -250,7 +250,12 @@ def test_anthropic_tool_call_error_is_mapped_from_the_provider_response(): assert len(route.calls) == 1 -async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): +async def test_anthropic_async_completion_forwards_http_images_as_url_sources(async_only_image_fetch): + """http:// image URLs are forwarded as URL sources, never downloaded. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" captured = {} @@ -290,10 +295,10 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop ) assert response.choices[0].message.content == "Green" - assert async_only_image_fetch.fetched == [http_image_url] + assert async_only_image_fetch.fetched == [] sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"] assert sources == [ - {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}, + {"type": "url", "url": http_image_url}, {"type": "url", "url": https_image_url}, ] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 4d5d6cc17a9..c23675ef583 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature +from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers @@ -9807,3 +9808,17 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +@pytest.mark.parametrize( + "url", ["http://example.com/image.png", "https://example.com/image.png"] +) +def test_inlines_remote_media_never_inlines_image_urls(url: str): + """Regular Anthropic forwards remote image URLs as url sources, so the async + request path must not download and inline them. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + media = RemoteMedia(url=url, fields={}, part_type="image_url") + + assert AnthropicConfig().inlines_remote_media(media) is False