fix(anthropic): convert document and PDF URLs to Anthropic document blocks

- Ensure HTTPS URLs with format='application/pdf', ending in .pdf, or document MIME types are converted to base64 Anthropic document blocks instead of invalid image URL blocks
- Route text/plain and other document media types to AnthropicMessagesDocumentParam instead of AnthropicMessagesImageParam
- Add unit tests covering PDF URL conversion and plain text data URI handling
This commit is contained in:
Chau Vu / CPF-FAMILY 2026-10-30 19:42:37 +07:00
parent a44e44b980
commit 69d765d32c
2 changed files with 123 additions and 8 deletions

View file

@ -908,6 +908,9 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
)
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
def create_anthropic_image_param(
image_url_input: str | dict,
format: str | None = None,
@ -930,9 +933,18 @@ def create_anthropic_image_param(
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
clean_url: Final = image_url.split("#")[0].split("?")[0].lower()
is_document: Final = (
(format in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES)
or (format is not None and select_anthropic_content_block_type_for_file(format) == "document")
or clean_url.endswith(".pdf")
)
if is_bedrock_invoke or image_url.startswith("http://") or is_document:
inferred_format = format
if inferred_format is None and clean_url.endswith(".pdf"):
inferred_format = "application/pdf"
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=inferred_format)
return _anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
@ -1069,7 +1081,7 @@ def anthropic_messages_pt_xml(messages: list):
base64_source = cast(AnthropicContentParamSource, source)
user_content.append(
{
"type": "image",
"type": image_param.get("type", "image"),
"source": {
"type": "base64",
"media_type": base64_source["media_type"],
@ -1788,9 +1800,6 @@ def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
def _is_anthropic_document_data_uri(url: str) -> bool:
# Anthropic's base64 document source accepts only application/pdf and
# text/plain (see select_anthropic_content_block_type_for_file). Routing
@ -2184,13 +2193,17 @@ def add_cache_control_to_content(
def _anthropic_content_element_factory(
image_chunk: GenericImageParsingChunk,
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
if image_chunk["media_type"] == "application/pdf":
media_type = image_chunk.get("media_type")
if (
media_type in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
or (isinstance(media_type, str) and select_anthropic_content_block_type_for_file(media_type) == "document")
):
_anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
media_type=media_type,
data=image_chunk["data"],
),
)

View file

@ -1320,6 +1320,108 @@ def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None:
}
@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
def test_anthropic_messages_pt_converts_plain_text_data_uri_to_document(llm_provider: str) -> None:
text_data = "aGVsbG8gd29ybGQ="
result: Final = anthropic_messages_pt(
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "summarize"},
{"type": "image_url", "image_url": {"url": f"data:text/plain;base64,{text_data}"}},
],
}
],
model="claude-opus-5-5",
llm_provider=llm_provider,
)
assert result[0]["content"][1] == {
"type": "document",
"source": {"type": "base64", "media_type": "text/plain", "data": text_data},
}
@pytest.mark.parametrize(
("url_input", "format_param"),
(
("https://example.com/document.pdf", None),
("https://example.com/document.pdf#page=1", None),
("https://example.com/document.pdf?token=xyz#page=1", None),
({"url": "https://example.com/document.pdf"}, None),
({"url": "https://example.com/download?id=42", "format": "application/pdf"}, None),
("https://example.com/download?id=42", "application/pdf"),
),
)
def test_create_anthropic_image_param_converts_pdf_url_to_document(
monkeypatch: pytest.MonkeyPatch, url_input: str | dict, format_param: str | None
) -> None:
pdf_data = "JVBERi0xLjQK"
monkeypatch.setattr(
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
lambda url: f"data:application/pdf;base64,{pdf_data}",
)
result = create_anthropic_image_param(url_input, format=format_param)
assert result == {
"type": "document",
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
}
def test_anthropic_messages_pt_converts_pdf_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None:
pdf_data = "JVBERi0xLjQK"
monkeypatch.setattr(
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
lambda url: f"data:application/pdf;base64,{pdf_data}",
)
result: Final = anthropic_messages_pt(
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "review"},
{"type": "image_url", "image_url": "https://example.com/spec.pdf"},
],
}
],
model="claude-3-7-sonnet",
llm_provider="anthropic",
)
assert result[0]["content"][1] == {
"type": "document",
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
}
def test_anthropic_messages_pt_converts_plain_text_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None:
text_data = "aGVsbG8="
monkeypatch.setattr(
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
lambda url: f"data:text/plain;base64,{text_data}",
)
result: Final = anthropic_messages_pt(
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "review"},
{"type": "image_url", "image_url": {"url": "https://example.com/log.txt", "format": "text/plain"}},
],
}
],
model="claude-3-7-sonnet",
llm_provider="anthropic",
)
assert result[0]["content"][1] == {
"type": "document",
"source": {"type": "base64", "media_type": "text/plain", "data": text_data},
}
EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
"empty_user_message",
[{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],