mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(anthropic): convert document and PDF URLs to Anthropic document blocks
- Ensure HTTPS URLs with format='application/pdf', ending in .pdf, or document MIME types are converted to base64 Anthropic document blocks instead of invalid image URL blocks - Route text/plain and other document media types to AnthropicMessagesDocumentParam instead of AnthropicMessagesImageParam - Add unit tests covering PDF URL conversion and plain text data URI handling
This commit is contained in:
parent
a44e44b980
commit
69d765d32c
2 changed files with 123 additions and 8 deletions
|
|
@ -908,6 +908,9 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
|
|||
)
|
||||
|
||||
|
||||
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
|
||||
|
||||
|
||||
def create_anthropic_image_param(
|
||||
image_url_input: str | dict,
|
||||
format: str | None = None,
|
||||
|
|
@ -930,9 +933,18 @@ def create_anthropic_image_param(
|
|||
if image_url.startswith("http://") or image_url.startswith("https://"):
|
||||
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
|
||||
# as these providers don't support URL sources for images
|
||||
if is_bedrock_invoke or image_url.startswith("http://"):
|
||||
clean_url: Final = image_url.split("#")[0].split("?")[0].lower()
|
||||
is_document: Final = (
|
||||
(format in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES)
|
||||
or (format is not None and select_anthropic_content_block_type_for_file(format) == "document")
|
||||
or clean_url.endswith(".pdf")
|
||||
)
|
||||
if is_bedrock_invoke or image_url.startswith("http://") or is_document:
|
||||
inferred_format = format
|
||||
if inferred_format is None and clean_url.endswith(".pdf"):
|
||||
inferred_format = "application/pdf"
|
||||
base64_url: Final = convert_url_to_base64(url=image_url)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=inferred_format)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
else:
|
||||
# HTTPS URL - pass directly for regular Anthropic
|
||||
|
|
@ -1069,7 +1081,7 @@ def anthropic_messages_pt_xml(messages: list):
|
|||
base64_source = cast(AnthropicContentParamSource, source)
|
||||
user_content.append(
|
||||
{
|
||||
"type": "image",
|
||||
"type": image_param.get("type", "image"),
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": base64_source["media_type"],
|
||||
|
|
@ -1788,9 +1800,6 @@ def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
|
|||
return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
|
||||
|
||||
|
||||
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
|
||||
|
||||
|
||||
def _is_anthropic_document_data_uri(url: str) -> bool:
|
||||
# Anthropic's base64 document source accepts only application/pdf and
|
||||
# text/plain (see select_anthropic_content_block_type_for_file). Routing
|
||||
|
|
@ -2184,13 +2193,17 @@ def add_cache_control_to_content(
|
|||
def _anthropic_content_element_factory(
|
||||
image_chunk: GenericImageParsingChunk,
|
||||
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
|
||||
if image_chunk["media_type"] == "application/pdf":
|
||||
media_type = image_chunk.get("media_type")
|
||||
if (
|
||||
media_type in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
|
||||
or (isinstance(media_type, str) and select_anthropic_content_block_type_for_file(media_type) == "document")
|
||||
):
|
||||
_anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
|
||||
AnthropicMessagesDocumentParam(
|
||||
type="document",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
media_type=media_type,
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1320,6 +1320,108 @@ def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None:
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
|
||||
def test_anthropic_messages_pt_converts_plain_text_data_uri_to_document(llm_provider: str) -> None:
|
||||
text_data = "aGVsbG8gd29ybGQ="
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "summarize"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:text/plain;base64,{text_data}"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-opus-5-5",
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "text/plain", "data": text_data},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url_input", "format_param"),
|
||||
(
|
||||
("https://example.com/document.pdf", None),
|
||||
("https://example.com/document.pdf#page=1", None),
|
||||
("https://example.com/document.pdf?token=xyz#page=1", None),
|
||||
({"url": "https://example.com/document.pdf"}, None),
|
||||
({"url": "https://example.com/download?id=42", "format": "application/pdf"}, None),
|
||||
("https://example.com/download?id=42", "application/pdf"),
|
||||
),
|
||||
)
|
||||
def test_create_anthropic_image_param_converts_pdf_url_to_document(
|
||||
monkeypatch: pytest.MonkeyPatch, url_input: str | dict, format_param: str | None
|
||||
) -> None:
|
||||
pdf_data = "JVBERi0xLjQK"
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
|
||||
lambda url: f"data:application/pdf;base64,{pdf_data}",
|
||||
)
|
||||
result = create_anthropic_image_param(url_input, format=format_param)
|
||||
|
||||
assert result == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
|
||||
}
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_converts_pdf_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pdf_data = "JVBERi0xLjQK"
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
|
||||
lambda url: f"data:application/pdf;base64,{pdf_data}",
|
||||
)
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "review"},
|
||||
{"type": "image_url", "image_url": "https://example.com/spec.pdf"},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-3-7-sonnet",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
|
||||
}
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_converts_plain_text_url_to_document(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
text_data = "aGVsbG8="
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64",
|
||||
lambda url: f"data:text/plain;base64,{text_data}",
|
||||
)
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "review"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/log.txt", "format": "text/plain"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-3-7-sonnet",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "text/plain", "data": text_data},
|
||||
}
|
||||
|
||||
|
||||
EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
|
||||
"empty_user_message",
|
||||
[{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue