From 26684a6643132070ac9291987b1301f7dccf50b6 Mon Sep 17 00:00:00 2001 From: Junyi Yao Date: Thu, 24 Sep 2026 18:51:31 -0700 Subject: [PATCH 1/2] fix(anthropic): forward http:// image URLs as url sources instead of downloading OpenAI to Anthropic conversion downloaded image_url parts with http:// URLs and sent them as base64, while https:// URLs were forwarded as url sources. Both schemes are now forwarded as url sources. Bedrock invoke, Vertex AI and Snowflake still convert to base64 since those providers have no URL source. Fixes #43098 Signed-off-by: Junyi Yao --- .../prompt_templates/factory.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 5 +- tests/llm_translation/test_prompt_factory.py | 66 +++++++++++++++++++ .../test_anthropic_chat_transformation.py | 15 +++++ 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 6fc319c26ae..7efa0eaba0f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -843,7 +843,7 @@ def create_anthropic_image_param( if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 # as these providers don't support URL sources for images - if is_bedrock_invoke or image_url.startswith("http://"): + if is_bedrock_invoke: base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) return AnthropicMessagesImageParam( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index b7c2ce3c568..d198f6f9e6f 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -29,7 +29,6 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, async_inline_remote_media, - inline_remote_image_urls, ) from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -1866,8 +1865,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): break return headers - def inlines_remote_media(self, media: RemoteMedia) -> bool: - return inline_remote_image_urls(media) and media.url.startswith("http://") + def inlines_remote_media(self, _media: RemoteMedia) -> bool: + return False async def async_transform_request( self, diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 7b03736920b..75b4f584ef4 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -225,6 +225,41 @@ def test_create_anthropic_image_param_with_https_url(): assert image_param["source"]["url"] == "https://example.com/image.png" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_forwards_url( + mock_convert_url: MagicMock, +): + """Plain http:// image URLs are forwarded as URL sources, like https://. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None + ) + + mock_convert_url.assert_not_called() + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "url" + assert image_param["source"]["url"] == "http://example.com/image.png" + + +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_create_anthropic_image_param_with_http_url_still_base64_for_bedrock( + mock_convert_url: MagicMock, +): + """Bedrock invoke keeps converting URLs to base64 (provider has no URL source).""" + mock_convert_url.return_value = "data:image/png;base64,/9j/4AAQSkZJRg==" + + image_param = create_anthropic_image_param( + "http://example.com/image.png", format=None, is_bedrock_invoke=True + ) + + mock_convert_url.assert_called_once_with(url="http://example.com/image.png") + assert image_param["type"] == "image" + assert image_param["source"]["type"] == "base64" + + def test_create_anthropic_image_param_with_dict_input(): """Test that dict input with URL is handled correctly.""" image_param = create_anthropic_image_param( @@ -292,6 +327,37 @@ def test_anthropic_messages_pt_with_url_image(): assert result[0]["content"][1]["source"]["url"] == "https://example.com/image.jpg" +@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") +def test_anthropic_messages_pt_with_http_url_image(mock_convert_url: MagicMock): + """Plain http:// image URLs are forwarded as URL sources for regular Anthropic. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": "http://example.com/image.jpg", + }, + ], + } + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-3-5-sonnet", llm_provider="anthropic" + ) + + mock_convert_url.assert_not_called() + assert len(result) == 1 + image_content = result[0]["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "url" + assert image_content["source"]["url"] == "http://example.com/image.jpg" + + def test_anthropic_messages_pt_with_base64_image(): """Test that anthropic_messages_pt correctly handles data URIs as base64.""" messages = [ diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 7167a67d80d..e3d0f6c1b10 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -17,6 +17,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) +from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -6519,3 +6520,17 @@ def test_eager_input_streaming_reaches_anthropic_request_tools(): assert result["tools"][0]["eager_input_streaming"] is True assert result["tools"][0]["name"] == "write_file" + + +@pytest.mark.parametrize( + "url", ["http://example.com/image.png", "https://example.com/image.png"] +) +def test_inlines_remote_media_never_inlines_image_urls(url: str): + """Regular Anthropic forwards remote image URLs as url sources, so the async + request path must not download and inline them. + + Regression test for https://github.com/BerriAI/litellm/issues/43098. + """ + media = RemoteMedia(url=url, fields={}, part_type="image_url") + + assert AnthropicConfig().inlines_remote_media(media) is False From fe050fb0a91364db214fa34fabc1059a9348cb74 Mon Sep 17 00:00:00 2001 From: Junyi Yao Date: Thu, 24 Sep 2026 20:53:54 -0700 Subject: [PATCH 2/2] test: update anthropic http image test for url-forwarding behavior The async completion test asserted the old behavior where http:// image URLs were downloaded and inlined as base64. Since http:// URLs are now forwarded as url sources like https://, the test asserts nothing is downloaded and both images arrive as url sources. Signed-off-by: Junyi Yao --- .../anthropic/chat/test_anthropic_chat_handler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 1e0d2e55373..9a6535dc1dc 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -87,7 +87,12 @@ def test_anthropic_completion_does_not_send_deployment_default_limits(): assert "default_api_key_tpm_limit" not in request_body -async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): +async def test_anthropic_async_completion_forwards_http_images_as_url_sources(async_only_image_fetch): + """http:// image URLs are forwarded as URL sources, never downloaded. + + Regression test for https://github.com/BerriAI/litellm/issues/43098: the + proxy used to download http:// URLs and send them as base64 without notice. + """ http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" captured = {} @@ -127,10 +132,10 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop ) assert response.choices[0].message.content == "Green" - assert async_only_image_fetch.fetched == [http_image_url] + assert async_only_image_fetch.fetched == [] sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"] assert sources == [ - {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}, + {"type": "url", "url": http_image_url}, {"type": "url", "url": https_image_url}, ]