From 9ae9bd43f81904bf2978a46351ae3aee58040ca7 Mon Sep 17 00:00:00 2001 From: Jerry-Xin Date: Sat, 7 Mar 2026 17:48:39 +0800 Subject: [PATCH] fix(vertex_ai): use async image URL conversion to avoid blocking event loop - Add async_transform_anthropic_messages_request method to base class - Add _convert_image_urls_to_base64_async method for non-blocking conversion - Update HTTP handler to use async transformation - Improve error message for URLs without file extensions --- .../prompt_templates/image_handling.py | 7 +- .../anthropic_messages/transformation.py | 22 ++ litellm/llms/custom_httpx/llm_http_handler.py | 4 +- .../transformation.py | 104 +++++++++ .../litellm_core_utils/test_image_handling.py | 30 +++ ..._vertex_ai_anthropic_image_url_handling.py | 216 ++++++++++++++++++ 6 files changed, 379 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index c7169bf17e1..47a6db99403 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -36,11 +36,14 @@ def _infer_media_type_from_url(url: str) -> str: """ # Strip query parameters for signed URLs (e.g., ?x-oss-signature=...) url_without_query = url.split("?")[0] - extension = url_without_query.split(".")[-1].lower() + # Only take the last path segment to avoid matching dots in the path + last_segment = url_without_query.rstrip("/").split("/")[-1] + # Check if the segment contains a dot (has an extension) + extension = last_segment.rsplit(".", 1)[-1].lower() if "." in last_segment else "" media_type = EXTENSION_TO_MEDIA_TYPE.get(extension) if media_type is None: raise Exception( - f"Error: Unsupported image format. Extension={extension}. " + f"Error: Unsupported image format. Could not infer media type from URL '{url}'. " f"Supported types = {list(SUPPORTED_IMAGE_TYPES)}" ) return media_type diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index fdad1633e8f..29d1b352b5e 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -74,6 +74,28 @@ class BaseAnthropicMessagesConfig(ABC): ) -> Dict: pass + async def async_transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Async version of transform_anthropic_messages_request. + + Override this method to perform async operations (e.g., fetching image URLs) + without blocking the event loop. Default implementation calls the sync version. + """ + return self.transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + @abstractmethod def transform_anthropic_messages_response( self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1cef3e9ce15..f4f62be5ce0 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1905,8 +1905,8 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) - # Prepare request body - request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( + # Prepare request body (use async version to avoid blocking event loop) + request_body = await anthropic_messages_provider_config.async_transform_anthropic_messages_request( model=model, messages=messages, anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 652905649d4..34167a3b80f 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -3,6 +3,9 @@ from typing import Any, Dict, List, Optional, Tuple from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, +) from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -195,6 +198,107 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert return converted_messages + @staticmethod + async def _convert_image_urls_to_base64_async(messages: List[Dict]) -> List[Dict]: + """ + Async version: Convert image URL sources to base64 format for Vertex AI. + + Vertex AI Anthropic does not support URL sources for images. + This method converts: + {"type": "image", "source": {"type": "url", "url": "https://..."}} + to: + {"type": "image", "source": {"type": "base64", "media_type": "...", "data": "..."}} + """ + converted_messages = [] + for message in messages: + if not isinstance(message, dict): + converted_messages.append(message) + continue + + content = message.get("content") + if not isinstance(content, list): + converted_messages.append(message) + continue + + new_content = [] + for block in content: + if not isinstance(block, dict): + new_content.append(block) + continue + + # Check if this is an image block with URL source + if block.get("type") == "image": + source = block.get("source", {}) + if isinstance(source, dict) and source.get("type") == "url": + url = source.get("url") + if url: + # Convert URL to base64 using async utility + data_uri = await async_convert_url_to_base64(url) + # Parse the data URI to extract media_type and data + # Format: "data:image/jpeg;base64,/9j/..." + media_type_part, base64_data = data_uri.split( + "data:" + )[1].split(";base64,") + # Preserve all original block fields (e.g., cache_control) + # while replacing the source + new_block = { + **block, + "source": { + "type": "base64", + "media_type": media_type_part, + "data": base64_data, + }, + } + new_content.append(new_block) + continue + + new_content.append(block) + + new_message = {**message, "content": new_content} + converted_messages.append(new_message) + + return converted_messages + + async def async_transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Async version of transform_anthropic_messages_request. + Uses async image URL to base64 conversion to avoid blocking the event loop. + """ + # Convert image URLs to base64 for Vertex AI using async method + # Vertex AI Anthropic does not support URL sources for images + converted_messages = await self._convert_image_urls_to_base64_async(messages) + + anthropic_messages_request = super().transform_anthropic_messages_request( + model=model, + messages=converted_messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" + + anthropic_messages_request.pop( + "model", None + ) # do not pass model in request body to vertex ai + + anthropic_messages_request.pop( + "output_format", None + ) # do not pass output_format in request body to vertex ai - vertex ai does not support output_format as yet + + anthropic_messages_request.pop( + "output_config", None + ) # do not pass output_config in request body to vertex ai - vertex ai does not support output_config + + return anthropic_messages_request + def transform_anthropic_messages_request( self, model: str, diff --git a/tests/test_litellm/litellm_core_utils/test_image_handling.py b/tests/test_litellm/litellm_core_utils/test_image_handling.py index cf5ab5b69c5..961f660cb1c 100644 --- a/tests/test_litellm/litellm_core_utils/test_image_handling.py +++ b/tests/test_litellm/litellm_core_utils/test_image_handling.py @@ -286,6 +286,36 @@ class TestInferMediaTypeFromUrl: result = _infer_media_type_from_url("https://example.com/IMAGE.PNG") assert result == "image/png" + def test_url_without_extension_raises_with_clear_message(self): + """Test that URL without extension raises an exception with a clear message.""" + url = "https://cdn.example.com/images/abc123" + with pytest.raises(Exception) as excinfo: + _infer_media_type_from_url(url) + error_msg = str(excinfo.value) + assert "Unsupported image format" in error_msg + # Should show the full URL for clarity, not just a confusing "extension" + assert url in error_msg + assert "Supported types" in error_msg + + def test_url_with_trailing_slash_no_extension(self): + """Test URL with trailing slash and no extension.""" + url = "https://cdn.example.com/images/abc123/" + with pytest.raises(Exception) as excinfo: + _infer_media_type_from_url(url) + error_msg = str(excinfo.value) + assert "Unsupported image format" in error_msg + assert url in error_msg + + def test_url_with_dots_in_path_but_no_image_extension(self): + """Test URL with dots in path segments but no valid image extension.""" + url = "https://api.example.com/v1.0/images/get" + with pytest.raises(Exception) as excinfo: + _infer_media_type_from_url(url) + error_msg = str(excinfo.value) + assert "Unsupported image format" in error_msg + # Should not confuse "0" from "v1.0" as the extension + assert url in error_msg + class TestGetValidMediaType: """Tests for _get_valid_media_type function.""" diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index 1fbfec7eed0..753f7155759 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -625,3 +625,219 @@ class TestVertexAIAnthropicPassThroughImageURLHandling: assert image2["source"]["type"] == "base64" assert image2["source"]["media_type"] == "image/png" assert image2["source"]["data"] == "iVBORw0image2" + + +class TestVertexAIAnthropicPassThroughImageURLHandlingAsync: + """ + Test the async version of image URL to base64 conversion for /v1/messages endpoint. + + Issue: https://github.com/BerriAI/litellm/issues/23026 + The sync version blocks the event loop. The async version uses async_convert_url_to_base64. + """ + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.async_convert_url_to_base64" + ) + def test_async_convert_image_urls_to_base64(self, mock_async_convert: MagicMock): + """ + Test that the async version correctly converts image URLs to base64. + """ + import asyncio + + async def async_mock_return(url): + return "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + + mock_async_convert.side_effect = async_mock_return + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.jpg", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = asyncio.run(config._convert_image_urls_to_base64_async(messages)) + + # Verify async_convert_url_to_base64 was called + mock_async_convert.assert_called_once_with("https://example.com/image.jpg") + + # Check the result has base64 source type + user_message = converted[0] + assert user_message["role"] == "user" + image_content = user_message["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "base64" + assert image_content["source"]["media_type"] == "image/jpeg" + assert image_content["source"]["data"] == "/9j/4AAQSkZJRgABAQAAAQ==" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.async_convert_url_to_base64" + ) + def test_async_preserves_base64_images(self, mock_async_convert: MagicMock): + """ + Test that async version preserves images already in base64 format. + """ + import asyncio + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = asyncio.run(config._convert_image_urls_to_base64_async(messages)) + + # async_convert_url_to_base64 should NOT be called for base64 images + mock_async_convert.assert_not_called() + + # Check the image is unchanged + image_content = converted[0]["content"][1] + assert image_content["source"]["type"] == "base64" + assert image_content["source"]["media_type"] == "image/png" + assert image_content["source"]["data"] == "iVBORw0KGgo=" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.async_convert_url_to_base64" + ) + def test_async_preserves_cache_control(self, mock_async_convert: MagicMock): + """ + Test that async version preserves cache_control when converting image URLs. + """ + import asyncio + + async def async_mock_return(url): + return "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + + mock_async_convert.side_effect = async_mock_return + + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.jpg", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = asyncio.run(config._convert_image_urls_to_base64_async(messages)) + + # Check cache_control is preserved + image_content = converted[0]["content"][0] + assert image_content["source"]["type"] == "base64" + assert image_content["cache_control"] == {"type": "ephemeral"} + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.async_convert_url_to_base64" + ) + def test_async_converts_multiple_images(self, mock_async_convert: MagicMock): + """ + Test that async version converts multiple image URLs in a message. + """ + import asyncio + + call_count = [0] + async def async_mock_return(url): + results = [ + "data:image/jpeg;base64,/9j/image1", + "data:image/png;base64,iVBORw0image2", + ] + result = results[call_count[0]] + call_count[0] += 1 + return result + + mock_async_convert.side_effect = async_mock_return + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these images"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image1.jpg", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image2.png", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = asyncio.run(config._convert_image_urls_to_base64_async(messages)) + + # Verify both URLs were converted + assert mock_async_convert.call_count == 2 + + # Check both images are converted to base64 + image1 = converted[0]["content"][1] + assert image1["source"]["type"] == "base64" + assert image1["source"]["media_type"] == "image/jpeg" + assert image1["source"]["data"] == "/9j/image1" + + image2 = converted[0]["content"][2] + assert image2["source"]["type"] == "base64" + assert image2["source"]["media_type"] == "image/png" + assert image2["source"]["data"] == "iVBORw0image2" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.async_convert_url_to_base64" + ) + def test_async_handles_string_content(self, mock_async_convert: MagicMock): + """ + Test that async version handles messages with string content correctly. + """ + import asyncio + + messages = [ + { + "role": "user", + "content": "Hello, how are you?", + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = asyncio.run(config._convert_image_urls_to_base64_async(messages)) + + # async_convert_url_to_base64 should NOT be called for string content + mock_async_convert.assert_not_called() + + # Check the message is unchanged + assert converted[0]["content"] == "Hello, how are you?"