From 13f9b8255209c9abb7d0bcd06ee2d26438ae97ed Mon Sep 17 00:00:00 2001 From: Junyi Yao Date: Thu, 24 Sep 2026 18:51:31 -0700 Subject: [PATCH 1/2] fix(anthropic): forward http:// image URLs as url sources instead of downloading OpenAI to Anthropic conversion downloaded image_url parts with http:// URLs and sent them as base64, while https:// URLs were forwarded as url sources. Both schemes are now forwarded as url sources. Bedrock invoke, Vertex AI and Snowflake still convert to base64 since those providers have no URL source. Fixes #43098 Signed-off-by: Junyi Yao --- .../prompt_templates/factory.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 5 +- ...llm_core_utils_prompt_templates_factory.py | 67 +++++++++++++++++++ .../test_anthropic_chat_transformation.py | 15 +++++ 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3d865bd1819..abad49cad6c 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -926,7 +926,7 @@ def create_anthropic_image_param( if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 # as these providers don't support URL sources for images - if is_bedrock_invoke or image_url.startswith("http://"): + if is_bedrock_invoke: base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) return AnthropicMessagesImageParam( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9a63afd5cc0..e653e284146 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -41,7 +41,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, async_inline_remote_media, - inline_remote_image_urls, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( place_mid_conversation_system, @@ -2231,8 +2230,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): break return headers - def inlines_remote_media(self, media: RemoteMedia) -> bool: - return inline_remote_image_urls(media) and media.url.startswith("http://") + def inlines_remote_media(self, _media: RemoteMedia) -> bool: + return False async def async_transform_request( self, diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index fad1c6178dc..8682c3716b5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -6158,6 +6158,42 @@ def test_create_anthropic_image_param_with_https_url(): assert image_param["source"]["url"] == "https://example.com/image.png" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_forwards_url( + mock_convert_url: MagicMock, +): + """Plain http:// image URLs are forwarded as URL sources, like https://. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None + ) + + mock_convert_url.assert_not_called() + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "url" + assert image_param["source"]["url"] == "http://example.com/image.png" + + + +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock( + mock_convert_url: MagicMock, +): + """Bedrock invoke keeps converting URLs to base64 (provider has no URL source).""" + mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg==" + + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None, is_bedrock_invoke=True + ) + + mock_convert_url.assert_called_once_with(url="http://example.com/image.png") + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "base64" + + def test_create_anthropic_image_param_with_dict_input(): """Test that dict input with URL is handled correctly.""" image_param = create_anthropic_image_param( @@ -6217,6 +6253,37 @@ def test_anthropic_messages_pt_with_url_image(): assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock): + """Plain http:// image URLs are forwarded as URL sources for regular Anthropic. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "http://example.com/image.jpg", + }, + ], + } + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" + ) + + mock_convert_url.assert_not_called() + assert len(result) == 1 + image_content = result[0]["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "url" + assert image_content["source"]["url"] == "http://example.com/image.jpg" + + def test_anthropic_messages_pt_with_base64_image(): """Test that anthropic_messages_pt correctly handles data URIs as base64.""" messages = [ diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 32f18b952ac..8089ed3f8b9 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature +from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers @@ -9816,3 +9817,17 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +@pytest.mark.parametrize( + "url", ["http://example.com/image.png", "https://example.com/image.png"] +) +def test_inlines_remote_media_never_inlines_image_urls(url: str): + """Regular Anthropic forwards remote image URLs as url sources, so the async + request path must not download and inline them. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + media = RemoteMedia(url=url, fields={}, part_type="image_url") + + assert AnthropicConfig().inlines_remote_media(media) is False From 43b39ff72ee0af0dc5a0485a9edfb4811227b4a8 Mon Sep 17 00:00:00 2001 From: Junyi Yao Date: Thu, 24 Sep 2026 20:53:54 -0700 Subject: [PATCH 2/2] test: update anthropic http image test for url-forwarding behavior The async completion test asserted the old behavior where http:// image URLs were downloaded and inlined as base64. Since http:// URLs are now forwarded as url sources like https://, the test asserts nothing is downloaded and both images arrive as url sources. Signed-off-by: Junyi Yao --- .../anthropic/chat/test_anthropic_chat_handler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 1f4a5aaa256..d1edba6633c 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -207,7 +207,12 @@ def test_anthropic_tool_call_error_is_mapped_from_the_provider_response(): assert len(route.calls) == 1 -async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): +async def test_anthropic_async_completion_forwards_http_images_as_url_sources(async_only_image_fetch): + """http:// image URLs are forwarded as URL sources, never downloaded. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" captured = {} @@ -247,10 +252,10 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop ) assert response.choices[0].message.content == "Green" - assert async_only_image_fetch.fetched == [http_image_url] + assert async_only_image_fetch.fetched == [] sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"] assert sources == [ - {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}, + {"type": "url", "url": http_image_url}, {"type": "url", "url": https_image_url}, ]