From 69d765d32c48a0d1ba46bf18cb76cbfbb6fe101d Mon Sep 17 00:00:00 2001 From: Chau Vu / CPF-FAMILY Date: Fri, 30 Oct 2026 19:42:37 +0700 Subject: [PATCH] fix(anthropic): convert document and PDF URLs to Anthropic document blocks - Ensure HTTPS URLs with format='application/pdf', ending in .pdf, or document MIME types are converted to base64 Anthropic document blocks instead of invalid image URL blocks - Route text/plain and other document media types to AnthropicMessagesDocumentParam instead of AnthropicMessagesImageParam - Add unit tests covering PDF URL conversion and plain text data URI handling --- .../prompt_templates/factory.py | 29 +++-- ...llm_core_utils_prompt_templates_factory.py | 102 ++++++++++++++++++ 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index de896cde4dc..71c130adb44 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -908,6 +908,9 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> ) +_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"} + + def create_anthropic_image_param( image_url_input: str | dict, format: str | None = None, @@ -930,9 +933,18 @@ def create_anthropic_image_param( if image_url.startswith("http://") or image_url.startswith("https://"): # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64 # as these providers don't support URL sources for images - if is_bedrock_invoke or image_url.startswith("http://"): + clean_url: Final = image_url.split("#")[0].split("?")[0].lower() + is_document: Final = ( + (format in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES) + or (format is not None and select_anthropic_content_block_type_for_file(format) == "document") + or clean_url.endswith(".pdf") + ) + if is_bedrock_invoke or image_url.startswith("http://") or is_document: + inferred_format = format + if inferred_format is None and clean_url.endswith(".pdf"): + inferred_format = "application/pdf" base64_url: Final = convert_url_to_base64(url=image_url) - image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) + image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=inferred_format) return _anthropic_content_element_factory(image_chunk) else: # HTTPS URL - pass directly for regular Anthropic @@ -1069,7 +1081,7 @@ def anthropic_messages_pt_xml(messages: list): base64_source = cast(AnthropicContentParamSource, source) user_content.append( { - "type": "image", + "type": image_param.get("type", "image"), "source": { "type": "base64", "media_type": base64_source["media_type"], @@ -1788,9 +1800,6 @@ def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str: return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}" -_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"} - - def _is_anthropic_document_data_uri(url: str) -> bool: # Anthropic's base64 document source accepts only application/pdf and # text/plain (see select_anthropic_content_block_type_for_file). Routing @@ -2184,13 +2193,17 @@ def add_cache_control_to_content( def _anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: - if image_chunk["media_type"] == "application/pdf": + media_type = image_chunk.get("media_type") + if ( + media_type in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + or (isinstance(media_type, str) and select_anthropic_content_block_type_for_file(media_type) == "document") + ): _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = ( AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type=media_type, data=image_chunk["data"], ), ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 778c9d68be5..f25ba6df081 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1320,6 +1320,108 @@ def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None: } +@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai")) +def test_anthropic_messages_pt_converts_plain_text_data_uri_to_document(llm_provider: str) -> None: + text_data = "aGVsbG8gd29ybGQ=" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": {"url": f"data:text/plain;base64,{text_data}"}}, + ], + } + ], + model="claude-opus-5-5", + llm_provider=llm_provider, + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "text/plain", "data": text_data}, + } + + +@pytest.mark.parametrize( + ("url_input", "format_param"), + ( + ("https://example.com/document.pdf", None), + ("https://example.com/document.pdf#page=1", None), + ("https://example.com/document.pdf?token=xyz#page=1", None), + ({"url": "https://example.com/document.pdf"}, None), + ({"url": "https://example.com/download?id=42", "format": "application/pdf"}, None), + ("https://example.com/download?id=42", "application/pdf"), + ), +) +def test_create_anthropic_image_param_converts_pdf_url_to_document( + monkeypatch: pytest.MonkeyPatch, url_input: str | dict, format_param: str | None +) -> None: + pdf_data = "JVBERi0xLjQK" + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64", + lambda url: f"data:application/pdf;base64,{pdf_data}", + ) + result = create_anthropic_image_param(url_input, format=format_param) + + assert result == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +def test_anthropic_messages_pt_converts_pdf_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None: + pdf_data = "JVBERi0xLjQK" + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64", + lambda url: f"data:application/pdf;base64,{pdf_data}", + ) + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "review"}, + {"type": "image_url", "image_url": "https://example.com/spec.pdf"}, + ], + } + ], + model="claude-3-7-sonnet", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +def test_anthropic_messages_pt_converts_plain_text_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None: + text_data = "aGVsbG8=" + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64", + lambda url: f"data:text/plain;base64,{text_data}", + ) + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "review"}, + {"type": "image_url", "image_url": {"url": "https://example.com/log.txt", "format": "text/plain"}}, + ], + } + ], + model="claude-3-7-sonnet", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "text/plain", "data": text_data}, + } + + EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize( "empty_user_message", [{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],