mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
fix(anthropic): send PDF image_url content to Claude as document blocks (#46984)
* fix(anthropic): convert PDF image_url content to documents Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prompt_templates): strip content-type parameters from fetched media Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(prompt_templates): format MIME normalization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang
GitHub
parent
4c8b90945f
commit
373f1752d3
4 changed files with 119 additions and 21 deletions
|
|
@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content(
|
|||
"""
|
||||
from litellm.litellm_core_utils.token_counter import get_image_type
|
||||
|
||||
media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None
|
||||
|
||||
# If we have a valid content type that's not generic, use it
|
||||
if current_content_type and current_content_type not in [
|
||||
if media_type and media_type not in [
|
||||
"binary/octet-stream",
|
||||
"application/octet-stream",
|
||||
]:
|
||||
return current_content_type
|
||||
return media_type
|
||||
|
||||
# Extension to MIME type mapping
|
||||
# Supports images, documents, and other common file types
|
||||
|
|
|
|||
|
|
@ -912,9 +912,9 @@ def create_anthropic_image_param(
|
|||
image_url_input: str | dict,
|
||||
format: str | None = None,
|
||||
is_bedrock_invoke: bool = False,
|
||||
) -> AnthropicMessagesImageParam:
|
||||
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
|
||||
"""
|
||||
Create an AnthropicMessagesImageParam from an image URL input.
|
||||
Create an Anthropic content block from an image URL input.
|
||||
|
||||
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
|
||||
"""
|
||||
|
|
@ -933,14 +933,7 @@ def create_anthropic_image_param(
|
|||
if is_bedrock_invoke or image_url.startswith("http://"):
|
||||
base64_url: Final = convert_url_to_base64(url=image_url)
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
else:
|
||||
# HTTPS URL - pass directly for regular Anthropic
|
||||
return AnthropicMessagesImageParam(
|
||||
|
|
@ -953,14 +946,7 @@ def create_anthropic_image_param(
|
|||
else:
|
||||
# Convert to base64 for data URIs or other formats
|
||||
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
|
||||
return AnthropicMessagesImageParam(
|
||||
type="image",
|
||||
source=AnthropicContentParamSource(
|
||||
type="base64",
|
||||
media_type=image_chunk["media_type"],
|
||||
data=image_chunk["data"],
|
||||
),
|
||||
)
|
||||
return _anthropic_content_element_factory(image_chunk)
|
||||
|
||||
|
||||
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
|
||||
|
|
@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result(
|
|||
anthropic_content_element=_anthropic_image_param,
|
||||
original_content_element=content,
|
||||
)
|
||||
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
|
||||
anthropic_content_list.append(
|
||||
cast(
|
||||
AnthropicMessagesImageParam | AnthropicMessagesDocumentParam,
|
||||
_anthropic_image_param,
|
||||
)
|
||||
)
|
||||
elif content["type"] == "tool_reference":
|
||||
anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"]))
|
||||
elif content["type"] == "file":
|
||||
|
|
|
|||
|
|
@ -1246,6 +1246,80 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block():
|
|||
assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
|
||||
def test_anthropic_messages_pt_converts_pdf_image_url_data_uri_to_document(llm_provider: str) -> None:
|
||||
pdf_data = "JVBERi0xLjQK"
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "summarize"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:application/pdf;base64,{pdf_data}"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-opus-5-5",
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai"))
|
||||
def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_provider: str) -> None:
|
||||
pdf_data = "JVBERi0xLjQK"
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "summarize"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:application/octet-stream;base64,{pdf_data}",
|
||||
"format": "application/pdf",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-opus-5-5",
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data},
|
||||
}
|
||||
|
||||
|
||||
def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None:
|
||||
image_data = "iVBORw0KGgo="
|
||||
result: Final = anthropic_messages_pt(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe"},
|
||||
{"type": "image_url", "image_url": f"data:image/png;base64,{image_data}"},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="claude-opus-5-5",
|
||||
llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert result[0]["content"][1] == {
|
||||
"type": "image",
|
||||
"source": {"type": "base64", "media_type": "image/png", "data": image_data},
|
||||
}
|
||||
|
||||
|
||||
EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
|
||||
"empty_user_message",
|
||||
[{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}],
|
||||
|
|
|
|||
|
|
@ -288,6 +288,37 @@ def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch):
|
|||
assert result.startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "content_type", "expected"),
|
||||
[
|
||||
(
|
||||
"https://docs.example/document.pdf",
|
||||
"application/pdf; qs=0.001",
|
||||
"data:application/pdf;base64,cGRm",
|
||||
),
|
||||
(
|
||||
"https://img.example/image.png",
|
||||
"image/png; charset=binary",
|
||||
"data:image/png;base64,cGRm",
|
||||
),
|
||||
(
|
||||
"https://docs.example/document.pdf",
|
||||
"application/octet-stream; charset=binary",
|
||||
"data:application/pdf;base64,cGRm",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_convert_url_to_base64_uses_bare_content_type(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
url: str,
|
||||
content_type: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(b"pdf", content_type))
|
||||
|
||||
assert convert_url_to_base64(url) == expected
|
||||
|
||||
|
||||
def test_convert_url_to_base64_raises_when_no_content_type_is_determinable(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue