diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 6bede1a2352..b15000e4db8 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -1,5 +1,8 @@ from typing import Any, Dict, List, Optional, Tuple +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + convert_url_to_base64, +) from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -33,10 +36,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert """ vertex_ai_project = VertexBase.safe_get_vertex_ai_project(litellm_params) vertex_ai_location = VertexBase.safe_get_vertex_ai_location(litellm_params) - + project_id: Optional[str] = None if "Authorization" not in headers: - vertex_credentials = VertexBase.safe_get_vertex_ai_credentials(litellm_params) + vertex_credentials = VertexBase.safe_get_vertex_ai_credentials( + litellm_params + ) access_token, project_id = self._ensure_access_token( credentials=vertex_credentials, @@ -62,11 +67,11 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert ) headers["content-type"] = "application/json" - + # Add beta headers for Vertex AI tools = optional_params.get("tools", []) beta_values: set[str] = set() - + # Get existing beta headers if any existing_beta = headers.get("anthropic-beta") if existing_beta: @@ -79,36 +84,42 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert edits = context_management_param.get("edits", []) has_compact = False has_other = False - + for edit in edits: edit_type = edit.get("type", "") if edit_type == "compact_20260112": has_compact = True else: has_other = True - + # Add compact header if any compact edits exist if has_compact: beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - + # Add context management header if any other edits exist if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.add( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + ) # Check for web search tool for tool in tools: - if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value): - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value) + if isinstance(tool, dict) and tool.get("type", "").startswith( + ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value + ): + beta_values.add( + ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value + ) break - + # Check for tool search tools - Vertex AI uses different beta header anthropic_model_info = AnthropicModelInfo() if anthropic_model_info.is_tool_search_used(tools): beta_values.add(get_tool_search_beta_header("vertex_ai")) - + if beta_values: headers["anthropic-beta"] = ",".join(beta_values) - + return headers, api_base def get_complete_url( @@ -126,6 +137,72 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert ) return api_base # no transformation is needed - handled in validate_environment + @staticmethod + def _convert_image_urls_to_base64(messages: List[Dict]) -> List[Dict]: + """ + Convert image URL sources to base64 format for Vertex AI. + + Vertex AI Anthropic does not support URL sources for images. + This method converts: + {"type": "image", "source": {"type": "url", "url": "https://..."}} + to: + {"type": "image", "source": {"type": "base64", "media_type": "...", "data": "..."}} + """ + converted_messages = [] + for message in messages: + if not isinstance(message, dict): + converted_messages.append(message) + continue + + content = message.get("content") + if not isinstance(content, list): + converted_messages.append(message) + continue + + new_content = [] + for block in content: + if not isinstance(block, dict): + new_content.append(block) + continue + + # Check if this is an image block with URL source + if block.get("type") == "image": + source = block.get("source", {}) + if isinstance(source, dict) and source.get("type") == "url": + url = source.get("url") + if url: + # Convert URL to base64 + base64_data_url = convert_url_to_base64(url=url) + # Parse the data URL: data:image/jpeg;base64, + if base64_data_url.startswith("data:"): + # Extract media type and data + parts = base64_data_url.split(";base64,", 1) + if len(parts) == 2: + media_type = parts[0].replace("data:", "") + data = parts[1] + new_block = { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": data, + }, + } + # Preserve cache_control if present + if "cache_control" in block: + new_block["cache_control"] = block[ + "cache_control" + ] + new_content.append(new_block) + continue + + new_content.append(block) + + new_message = {**message, "content": new_content} + converted_messages.append(new_message) + + return converted_messages + def transform_anthropic_messages_request( self, model: str, @@ -134,9 +211,13 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert litellm_params: GenericLiteLLMParams, headers: dict, ) -> Dict: + # Convert image URLs to base64 for Vertex AI + # Vertex AI Anthropic does not support URL sources for images + converted_messages = self._convert_image_urls_to_base64(messages) + anthropic_messages_request = super().transform_anthropic_messages_request( model=model, - messages=messages, + messages=converted_messages, anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, litellm_params=litellm_params, headers=headers, diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py index 3f014d65d4d..e9a1f106586 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py @@ -4,7 +4,11 @@ Tests for Vertex AI Anthropic image URL handling. Issue: https://github.com/BerriAI/litellm/issues/18430 Vertex AI Anthropic models don't support URL sources for images. LiteLLM should convert image URLs to base64 when using Vertex AI Anthropic. + +Issue: https://github.com/BerriAI/litellm/issues/23016 +/v1/messages endpoint with Vertex AI Anthropic should also convert image URLs to base64. """ + import os import sys from unittest.mock import patch, MagicMock @@ -20,6 +24,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_tool_result, create_anthropic_image_param, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + VertexAIPartnerModelsAnthropicMessagesConfig, +) class TestVertexAIAnthropicImageURLHandling: @@ -35,7 +42,9 @@ class TestVertexAIAnthropicImageURLHandling: For regular Anthropic, HTTPS URLs are passed through as URL type. For Vertex AI Anthropic, HTTPS URLs should be converted to base64. """ - mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + mock_convert_url.return_value = ( + "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + ) messages = [ { @@ -108,9 +117,7 @@ class TestVertexAIAnthropicImageURLHandling: assert image_content["source"]["url"] == "https://example.com/image.jpg" @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") - def test_vertex_ai_beta_also_converts_to_base64( - self, mock_convert_url: MagicMock - ): + def test_vertex_ai_beta_also_converts_to_base64(self, mock_convert_url: MagicMock): """ Test that vertex_ai_beta provider also converts image URLs to base64. """ @@ -210,7 +217,9 @@ class TestToolMessageImageURLHandling: result = convert_to_anthropic_tool_result(tool_message, force_base64=True) - mock_convert_url.assert_called_once_with(url="https://example.com/tool_result.jpg") + mock_convert_url.assert_called_once_with( + url="https://example.com/tool_result.jpg" + ) assert result["type"] == "tool_result" assert result["tool_use_id"] == "call_123" @@ -303,7 +312,10 @@ class TestToolMessageImageURLHandling: for msg in result: if msg.get("role") == "user": for content_item in msg.get("content", []): - if isinstance(content_item, dict) and content_item.get("type") == "tool_result": + if ( + isinstance(content_item, dict) + and content_item.get("type") == "tool_result" + ): tool_content = content_item.get("content", []) for item in tool_content: if isinstance(item, dict) and item.get("type") == "image": @@ -312,9 +324,7 @@ class TestToolMessageImageURLHandling: pytest.fail("Could not find image in tool result") @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64") - def test_regular_anthropic_tool_message_uses_url( - self, mock_convert_url: MagicMock - ): + def test_regular_anthropic_tool_message_uses_url(self, mock_convert_url: MagicMock): """ Test that regular Anthropic API uses URL type for tool result images. """ @@ -362,10 +372,250 @@ class TestToolMessageImageURLHandling: for msg in result: if msg.get("role") == "user": for content_item in msg.get("content", []): - if isinstance(content_item, dict) and content_item.get("type") == "tool_result": + if ( + isinstance(content_item, dict) + and content_item.get("type") == "tool_result" + ): tool_content = content_item.get("content", []) for item in tool_content: if isinstance(item, dict) and item.get("type") == "image": assert item["source"]["type"] == "url" return pytest.fail("Could not find image in tool result") + + +class TestVertexAIAnthropicPassThroughImageURLHandling: + """ + Test that /v1/messages endpoint (pass-through) converts image URLs to base64 for Vertex AI. + + Issue: https://github.com/BerriAI/litellm/issues/23016 + """ + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_converts_image_url_to_base64( + self, mock_convert_url: MagicMock + ): + """ + Test that the /v1/messages endpoint converts image URLs to base64 for Vertex AI. + + When using Anthropic native format with URL source type, + Vertex AI should convert it to base64. + """ + mock_convert_url.return_value = ( + "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.jpg", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # Verify convert_url_to_base64 was called + mock_convert_url.assert_called_once_with(url="https://example.com/image.jpg") + + # Check the result has base64 source type + user_message = converted[0] + assert user_message["role"] == "user" + image_content = user_message["content"][1] + assert image_content["type"] == "image" + assert image_content["source"]["type"] == "base64" + assert image_content["source"]["media_type"] == "image/jpeg" + assert image_content["source"]["data"] == "/9j/4AAQSkZJRgABAQAAAQ==" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_preserves_base64_images( + self, mock_convert_url: MagicMock + ): + """ + Test that images already in base64 format are not modified. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # convert_url_to_base64 should NOT be called for base64 images + mock_convert_url.assert_not_called() + + # Check the image is unchanged + image_content = converted[0]["content"][1] + assert image_content["source"]["type"] == "base64" + assert image_content["source"]["media_type"] == "image/png" + assert image_content["source"]["data"] == "iVBORw0KGgo=" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_preserves_cache_control( + self, mock_convert_url: MagicMock + ): + """ + Test that cache_control is preserved when converting image URLs. + """ + mock_convert_url.return_value = ( + "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ==" + ) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.jpg", + }, + "cache_control": {"type": "ephemeral"}, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # Check cache_control is preserved + image_content = converted[0]["content"][0] + assert image_content["source"]["type"] == "base64" + assert image_content["cache_control"] == {"type": "ephemeral"} + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_handles_text_only_messages( + self, mock_convert_url: MagicMock + ): + """ + Test that text-only messages are handled correctly. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello, how are you?"}, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # convert_url_to_base64 should NOT be called for text-only messages + mock_convert_url.assert_not_called() + + # Check the message is unchanged + assert converted[0]["content"][0]["type"] == "text" + assert converted[0]["content"][0]["text"] == "Hello, how are you?" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_handles_string_content( + self, mock_convert_url: MagicMock + ): + """ + Test that messages with string content are handled correctly. + """ + messages = [ + { + "role": "user", + "content": "Hello, how are you?", + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # convert_url_to_base64 should NOT be called for string content + mock_convert_url.assert_not_called() + + # Check the message is unchanged + assert converted[0]["content"] == "Hello, how are you?" + + @patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64" + ) + def test_vertex_ai_messages_converts_multiple_images( + self, mock_convert_url: MagicMock + ): + """ + Test that multiple image URLs in a message are all converted. + """ + mock_convert_url.side_effect = [ + "data:image/jpeg;base64,/9j/image1", + "data:image/png;base64,iVBORw0image2", + ] + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these images"}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image1.jpg", + }, + }, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image2.png", + }, + }, + ], + } + ] + + config = VertexAIPartnerModelsAnthropicMessagesConfig() + converted = config._convert_image_urls_to_base64(messages) + + # Verify both URLs were converted + assert mock_convert_url.call_count == 2 + + # Check both images are converted to base64 + image1 = converted[0]["content"][1] + assert image1["source"]["type"] == "base64" + assert image1["source"]["media_type"] == "image/jpeg" + assert image1["source"]["data"] == "/9j/image1" + + image2 = converted[0]["content"][2] + assert image2["source"]["type"] == "base64" + assert image2["source"]["media_type"] == "image/png" + assert image2["source"]["data"] == "iVBORw0image2"