scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

fix(vertex_ai): convert image URLs to base64 for /v1/messages endpoint

Browse source 
Fixes #23016

The /v1/messages endpoint with Vertex AI Anthropic was failing with
'URL sources are not supported' error because the pass-through path
didn't convert image URLs to base64.

This PR adds the same URL-to-base64 conversion logic that was added
in PR #18497 for /v1/chat/completions to the Vertex AI pass-through
transformation layer.

Changes:
- Add _convert_image_urls_to_base64 method to convert Anthropic native
  image format URLs to base64
- Call conversion in transform_anthropic_messages_request before
  forwarding to Vertex AI
- Add 6 new test cases for the pass-through image handling
This commit is contained in:
Jerry-Xin 2026-03-07 15:00:02 +08:00
parent b314e8d20a
commit 80f3e7ac97
2 changed files with 355 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -1,5 +1,8 @@
from typing import Any, Dict, List, Optional, Tuple
from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -33,10 +36,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
"""
vertex_ai_project = VertexBase.safe_get_vertex_ai_project(litellm_params)
vertex_ai_location = VertexBase.safe_get_vertex_ai_location(litellm_params)
project_id: Optional[str] = None
if "Authorization" not in headers:
vertex_credentials = VertexBase.safe_get_vertex_ai_credentials(litellm_params)
vertex_credentials = VertexBase.safe_get_vertex_ai_credentials(
litellm_params
)
access_token, project_id = self._ensure_access_token(
credentials=vertex_credentials,
@ -62,11 +67,11 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
)
headers["content-type"] = "application/json"
# Add beta headers for Vertex AI
tools = optional_params.get("tools", [])
beta_values: set[str] = set()
# Get existing beta headers if any
existing_beta = headers.get("anthropic-beta")
if existing_beta:
@ -79,36 +84,42 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
edits = context_management_param.get("edits", [])
has_compact = False
has_other = False
for edit in edits:
edit_type = edit.get("type", "")
if edit_type == "compact_20260112":
has_compact = True
else:
has_other = True
# Add compact header if any compact edits exist
if has_compact:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value)
# Add context management header if any other edits exist
if has_other:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value)
beta_values.add(
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value
)
# Check for web search tool
for tool in tools:
if isinstance(tool, dict) and tool.get("type", "").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value)
if isinstance(tool, dict) and tool.get("type", "").startswith(
ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value
):
beta_values.add(
ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value
)
break
# Check for tool search tools - Vertex AI uses different beta header
anthropic_model_info = AnthropicModelInfo()
if anthropic_model_info.is_tool_search_used(tools):
beta_values.add(get_tool_search_beta_header("vertex_ai"))
if beta_values:
headers["anthropic-beta"] = ",".join(beta_values)
return headers, api_base
def get_complete_url(
@ -126,6 +137,72 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
)
return api_base # no transformation is needed - handled in validate_environment
@staticmethod
def _convert_image_urls_to_base64(messages: List[Dict]) -> List[Dict]:
"""
Convert image URL sources to base64 format for Vertex AI.
Vertex AI Anthropic does not support URL sources for images.
This method converts:
{"type": "image", "source": {"type": "url", "url": "https://..."}}
to:
{"type": "image", "source": {"type": "base64", "media_type": "...", "data": "..."}}
"""
converted_messages = []
for message in messages:
if not isinstance(message, dict):
converted_messages.append(message)
continue
content = message.get("content")
if not isinstance(content, list):
converted_messages.append(message)
continue
new_content = []
for block in content:
if not isinstance(block, dict):
new_content.append(block)
continue
# Check if this is an image block with URL source
if block.get("type") == "image":
source = block.get("source", {})
if isinstance(source, dict) and source.get("type") == "url":
url = source.get("url")
if url:
# Convert URL to base64
base64_data_url = convert_url_to_base64(url=url)
# Parse the data URL: data:image/jpeg;base64,<data>
if base64_data_url.startswith("data:"):
# Extract media type and data
parts = base64_data_url.split(";base64,", 1)
if len(parts) == 2:
media_type = parts[0].replace("data:", "")
data = parts[1]
new_block = {
"type": "image",
"source": {
"type": "base64",
"media_type": media_type,
"data": data,
},
}
# Preserve cache_control if present
if "cache_control" in block:
new_block["cache_control"] = block[
"cache_control"
]
new_content.append(new_block)
continue
new_content.append(block)
new_message = {**message, "content": new_content}
converted_messages.append(new_message)
return converted_messages
def transform_anthropic_messages_request(
self,
model: str,
@ -134,9 +211,13 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Dict:
# Convert image URLs to base64 for Vertex AI
# Vertex AI Anthropic does not support URL sources for images
converted_messages = self._convert_image_urls_to_base64(messages)
anthropic_messages_request = super().transform_anthropic_messages_request(
model=model,
messages=messages,
messages=converted_messages,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
litellm_params=litellm_params,
headers=headers,

270
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py
View file

@ -4,7 +4,11 @@ Tests for Vertex AI Anthropic image URL handling.
Issue: https://github.com/BerriAI/litellm/issues/18430
Vertex AI Anthropic models don't support URL sources for images.
LiteLLM should convert image URLs to base64 when using Vertex AI Anthropic.
Issue: https://github.com/BerriAI/litellm/issues/23016
/v1/messages endpoint with Vertex AI Anthropic should also convert image URLs to base64.
"""
import os
import sys
from unittest.mock import patch, MagicMock
@ -20,6 +24,9 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
create_anthropic_image_param,
)
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
class TestVertexAIAnthropicImageURLHandling:
@ -35,7 +42,9 @@ class TestVertexAIAnthropicImageURLHandling:
For regular Anthropic, HTTPS URLs are passed through as URL type.
For Vertex AI Anthropic, HTTPS URLs should be converted to base64.
"""
mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ=="
mock_convert_url.return_value = (
"data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ=="
)
messages = [
{
@ -108,9 +117,7 @@ class TestVertexAIAnthropicImageURLHandling:
assert image_content["source"]["url"] == "https://example.com/image.jpg"
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_vertex_ai_beta_also_converts_to_base64(
self, mock_convert_url: MagicMock
):
def test_vertex_ai_beta_also_converts_to_base64(self, mock_convert_url: MagicMock):
"""
Test that vertex_ai_beta provider also converts image URLs to base64.
"""
@ -210,7 +217,9 @@ class TestToolMessageImageURLHandling:
result = convert_to_anthropic_tool_result(tool_message, force_base64=True)
mock_convert_url.assert_called_once_with(url="https://example.com/tool_result.jpg")
mock_convert_url.assert_called_once_with(
url="https://example.com/tool_result.jpg"
)
assert result["type"] == "tool_result"
assert result["tool_use_id"] == "call_123"
@ -303,7 +312,10 @@ class TestToolMessageImageURLHandling:
for msg in result:
if msg.get("role") == "user":
for content_item in msg.get("content", []):
if isinstance(content_item, dict) and content_item.get("type") == "tool_result":
if (
isinstance(content_item, dict)
and content_item.get("type") == "tool_result"
):
tool_content = content_item.get("content", [])
for item in tool_content:
if isinstance(item, dict) and item.get("type") == "image":
@ -312,9 +324,7 @@ class TestToolMessageImageURLHandling:
pytest.fail("Could not find image in tool result")
@patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
def test_regular_anthropic_tool_message_uses_url(
self, mock_convert_url: MagicMock
):
def test_regular_anthropic_tool_message_uses_url(self, mock_convert_url: MagicMock):
"""
Test that regular Anthropic API uses URL type for tool result images.
"""
@ -362,10 +372,250 @@ class TestToolMessageImageURLHandling:
for msg in result:
if msg.get("role") == "user":
for content_item in msg.get("content", []):
if isinstance(content_item, dict) and content_item.get("type") == "tool_result":
if (
isinstance(content_item, dict)
and content_item.get("type") == "tool_result"
):
tool_content = content_item.get("content", [])
for item in tool_content:
if isinstance(item, dict) and item.get("type") == "image":
assert item["source"]["type"] == "url"
return
pytest.fail("Could not find image in tool result")
class TestVertexAIAnthropicPassThroughImageURLHandling:
"""
Test that /v1/messages endpoint (pass-through) converts image URLs to base64 for Vertex AI.
Issue: https://github.com/BerriAI/litellm/issues/23016
"""
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_converts_image_url_to_base64(
self, mock_convert_url: MagicMock
):
"""
Test that the /v1/messages endpoint converts image URLs to base64 for Vertex AI.
When using Anthropic native format with URL source type,
Vertex AI should convert it to base64.
"""
mock_convert_url.return_value = (
"data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ=="
)
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this image"},
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image.jpg",
},
},
],
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# Verify convert_url_to_base64 was called
mock_convert_url.assert_called_once_with(url="https://example.com/image.jpg")
# Check the result has base64 source type
user_message = converted[0]
assert user_message["role"] == "user"
image_content = user_message["content"][1]
assert image_content["type"] == "image"
assert image_content["source"]["type"] == "base64"
assert image_content["source"]["media_type"] == "image/jpeg"
assert image_content["source"]["data"] == "/9j/4AAQSkZJRgABAQAAAQ=="
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_preserves_base64_images(
self, mock_convert_url: MagicMock
):
"""
Test that images already in base64 format are not modified.
"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this image"},
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "iVBORw0KGgo=",
},
},
],
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# convert_url_to_base64 should NOT be called for base64 images
mock_convert_url.assert_not_called()
# Check the image is unchanged
image_content = converted[0]["content"][1]
assert image_content["source"]["type"] == "base64"
assert image_content["source"]["media_type"] == "image/png"
assert image_content["source"]["data"] == "iVBORw0KGgo="
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_preserves_cache_control(
self, mock_convert_url: MagicMock
):
"""
Test that cache_control is preserved when converting image URLs.
"""
mock_convert_url.return_value = (
"data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ=="
)
messages = [
{
"role": "user",
"content": [
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image.jpg",
},
"cache_control": {"type": "ephemeral"},
},
],
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# Check cache_control is preserved
image_content = converted[0]["content"][0]
assert image_content["source"]["type"] == "base64"
assert image_content["cache_control"] == {"type": "ephemeral"}
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_handles_text_only_messages(
self, mock_convert_url: MagicMock
):
"""
Test that text-only messages are handled correctly.
"""
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello, how are you?"},
],
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# convert_url_to_base64 should NOT be called for text-only messages
mock_convert_url.assert_not_called()
# Check the message is unchanged
assert converted[0]["content"][0]["type"] == "text"
assert converted[0]["content"][0]["text"] == "Hello, how are you?"
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_handles_string_content(
self, mock_convert_url: MagicMock
):
"""
Test that messages with string content are handled correctly.
"""
messages = [
{
"role": "user",
"content": "Hello, how are you?",
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# convert_url_to_base64 should NOT be called for string content
mock_convert_url.assert_not_called()
# Check the message is unchanged
assert converted[0]["content"] == "Hello, how are you?"
@patch(
"litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation.convert_url_to_base64"
)
def test_vertex_ai_messages_converts_multiple_images(
self, mock_convert_url: MagicMock
):
"""
Test that multiple image URLs in a message are all converted.
"""
mock_convert_url.side_effect = [
"data:image/jpeg;base64,/9j/image1",
"data:image/png;base64,iVBORw0image2",
]
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "Compare these images"},
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image1.jpg",
},
},
{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image2.png",
},
},
],
}
]
config = VertexAIPartnerModelsAnthropicMessagesConfig()
converted = config._convert_image_urls_to_base64(messages)
# Verify both URLs were converted
assert mock_convert_url.call_count == 2
# Check both images are converted to base64
image1 = converted[0]["content"][1]
assert image1["source"]["type"] == "base64"
assert image1["source"]["media_type"] == "image/jpeg"
assert image1["source"]["data"] == "/9j/image1"
image2 = converted[0]["content"][2]
assert image2["source"]["type"] == "base64"
assert image2["source"]["media_type"] == "image/png"
assert image2["source"]["data"] == "iVBORw0image2"