scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge pull request #26710 from minznerjosh/fix/bedrock-anthropic-tool-result-pdf-content

Browse source 
fix(bedrock, anthropic): translate OpenAI file content on tool-result path
This commit is contained in:
Sameer Kankute 2026-04-29 12:59:08 +05:30 • committed by GitHub
commit d8d1444da4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 517 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
return sanitized
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"}
def _is_anthropic_document_data_uri(url: str) -> bool:
# Anthropic's base64 document source accepts only application/pdf and
# text/plain (see select_anthropic_content_block_type_for_file). Routing
# other mimes here would produce a document block the API rejects, so we
# leave them on the image code path.
match = re.match(r"data:([^;,]+)", url)
if not match:
return False
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def convert_to_anthropic_tool_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
force_base64: bool = False,
@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result(
"""
anthropic_content: Union[
str,
List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]],
List[
Union[
AnthropicMessagesToolResultContent,
AnthropicMessagesImageParam,
AnthropicMessagesDocumentParam,
]
],
] = ""
if isinstance(message["content"], str):
anthropic_content = message["content"]
elif isinstance(message["content"], List):
content_list = message["content"]
anthropic_content_list: List[
Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
Union[
AnthropicMessagesToolResultContent,
AnthropicMessagesImageParam,
AnthropicMessagesDocumentParam,
]
] = []
for content in content_list:
if content["type"] == "text":
@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result(
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
format = (
content["image_url"].get("format")
if isinstance(content["image_url"], dict)
image_url_value.get("format")
if isinstance(image_url_value, dict)
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format, is_bedrock_invoke=force_base64
url_str = (
image_url_value.get("url")
if isinstance(image_url_value, dict)
else image_url_value
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
# Data URIs with non-image mime types (e.g. application/pdf) must
# translate to Anthropic document blocks, not image blocks —
# wrapping a PDF in `type: "image"` is rejected by the API.
if isinstance(url_str, str) and _is_anthropic_document_data_uri(
url_str
):
synth_file_message: ChatCompletionFileObject = {
"type": "file",
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(
synth_file_message
)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(
AnthropicMessagesDocumentParam, _document_block
),
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesDocumentParam, _document_block)
)
else:
_anthropic_image_param = create_anthropic_image_param(
image_url_value,
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesImageParam, _anthropic_image_param)
)
elif content["type"] == "file":
file_content = cast(ChatCompletionFileObject, content)
_file_block = anthropic_process_openai_file_message(file_content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(
AnthropicMessagesDocumentParam, _file_block
),
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesImageParam, _anthropic_image_param)
)
anthropic_content_list.append(_file_block)
anthropic_content = anthropic_content_list
anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@ -3977,6 +4042,41 @@ def _convert_to_bedrock_tool_call_result(
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_block["image"])
)
elif "document" in _block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(document=_block["document"])
)
elif content["type"] == "file":
# Match the user-message path (_process_file_message): accept
# either file_data (base64 data URI) or file_id (server-side
# reference / URL) and hand off to BedrockImageProcessor. Raise
# BadRequestError on both-None rather than silently dropping.
file_obj = content.get("file") or {}
file_data = file_obj.get("file_data")
file_id = file_obj.get("file_id")
if file_data is None and file_id is None:
raise litellm.BadRequestError(
message="file_data and file_id cannot both be None. Got={}".format(
content
),
model="",
llm_provider="bedrock",
)
file_format = file_obj.get("format")
_file_block: BedrockContentBlock = (
BedrockImageProcessor.process_image_sync(
image_url=cast(str, file_id or file_data),
format=file_format,
)
)
if "document" in _file_block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(document=_file_block["document"])
)
elif "image" in _file_block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_file_block["image"])
)
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))

6
litellm/types/llms/anthropic.py
View file

@ -330,7 +330,11 @@ class AnthropicMessagesToolResultParam(TypedDict, total=False):
content: Union[
str,
Iterable[
Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
Union[
AnthropicMessagesToolResultContent,
AnthropicMessagesImageParam,
AnthropicMessagesDocumentParam,
]
],
]
cache_control: Optional[Union[dict, ChatCompletionCachedContent]]

183
tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -2476,3 +2476,186 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5():
cache_blocks_old = [b for b in result_old if "cachePoint" in b]
assert len(cache_blocks_old) == 1
assert "ttl" not in cache_blocks_old[0]["cachePoint"]
def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document():
"""
OpenAI `{type: "file", file: {file_data: "data:application/pdf;..."}}` inside
a tool-message content list should translate to an Anthropic document block
inside the tool_result content. Reuses anthropic_process_openai_file_message,
which already handles this for user messages.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
message = {
"tool_call_id": "toolu_pdf_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "file",
"file": {
"file_data": f"data:application/pdf;base64,{pdf_b64}",
"filename": "summary.pdf",
},
},
],
}
result = convert_to_anthropic_tool_result(message)
assert result["type"] == "tool_result"
assert result["tool_use_id"] == "toolu_pdf_1"
content = result["content"]
assert isinstance(content, list) and len(content) == 1
block = content[0]
assert block["type"] == "document"
assert block["source"]["type"] == "base64"
assert block["source"]["media_type"] == "application/pdf"
assert block["source"]["data"] == pdf_b64
def test_convert_to_anthropic_tool_result_image_url_pdf_data_uri_becomes_document():
"""
Regression: a PDF sent as an `image_url` data URI on the tool-result path
must translate to an Anthropic document block (not an image block — Anthropic
rejects image blocks whose media_type is a non-image like application/pdf).
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
message = {
"tool_call_id": "toolu_pdf_img_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:application/pdf;base64,{pdf_b64}",
},
},
],
}
result = convert_to_anthropic_tool_result(message)
content = result["content"]
assert isinstance(content, list) and len(content) == 1
block = content[0]
assert block["type"] == "document"
assert block["source"]["media_type"] == "application/pdf"
assert block["source"]["data"] == pdf_b64
def test_convert_to_anthropic_tool_result_image_url_unsupported_mime_stays_image_path():
"""
An `image_url` data URI whose mime is neither application/pdf nor text/plain
(e.g. application/json) must NOT be routed through the document path. Anthropic
only accepts application/pdf and text/plain as base64 document media_types —
anything else would produce a document block the API rejects. The old
(pre-fix) behavior was to wrap such data as an image block, which also
fails but stays on the image code path; preserve that failure mode rather
than switching to a document path that is equally broken.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
message = {
"tool_call_id": "toolu_json_1",
"role": "tool",
"name": "fetch_json",
"content": [
{
"type": "image_url",
"image_url": {
"url": "data:application/json;base64,eyJrIjoidiJ9",
},
},
],
}
result = convert_to_anthropic_tool_result(message)
content = result["content"]
assert isinstance(content, list) and len(content) == 1
block = content[0]
assert block["type"] == "image", (
f"unsupported mime {block.get('source', {}).get('media_type')!r} "
f"should not be routed to document path; got {block}"
)
def test_convert_to_anthropic_tool_result_image_url_text_plain_data_uri_becomes_document():
"""
text/plain is one of the two mimes Anthropic accepts as a base64 document
media_type. Confirm it routes through the document path so tightening the
gate to {application/pdf, text/plain} (not "application/*") covers both.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
txt_b64 = "aGVsbG8=" # "hello"
message = {
"tool_call_id": "toolu_txt_1",
"role": "tool",
"name": "fetch_text",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:text/plain;base64,{txt_b64}",
},
},
],
}
result = convert_to_anthropic_tool_result(message)
content = result["content"]
assert isinstance(content, list) and len(content) == 1
block = content[0]
assert block["type"] == "document"
assert block["source"]["media_type"] == "text/plain"
assert block["source"]["data"] == txt_b64
def test_convert_to_anthropic_tool_result_image_url_png_still_becomes_image():
"""
Regression: image_url with a real image mime type must continue to translate
to an Anthropic image block. Locks in existing behavior after the
data-URI-mime-type branching for PDFs.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_tool_result,
)
png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
message = {
"tool_call_id": "toolu_png_1",
"role": "tool",
"name": "fetch_image",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{png_b64}",
},
},
],
}
result = convert_to_anthropic_tool_result(message)
content = result["content"]
assert isinstance(content, list) and len(content) == 1
block = content[0]
assert block["type"] == "image"
assert block["source"]["media_type"] == "image/png"

218
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -4146,3 +4146,221 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(
# finish_reason must be "stop", not "tool_calls"
assert result.choices[0].finish_reason == "stop"
def test_bedrock_tool_message_openai_file_pdf_becomes_document():
"""
OpenAI Chat Completions `{type: "file", file: {file_data: "data:application/pdf;...", filename}}`
inside a tool message content list should translate to a Bedrock
toolResult.content[].document block.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK" # tiny "%PDF-1.4\n" header
messages = [
{"role": "user", "content": "Summarize the attached PDF."},
{
"tool_call_id": "tooluse_pdf_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "file",
"file": {
"file_data": f"data:application/pdf;base64,{pdf_b64}",
"filename": "summary.pdf",
},
},
],
},
]
translated_msg = _bedrock_converse_messages_pt(
messages=messages, model="", llm_provider=""
)
tool_result = translated_msg[-1]["content"][-1]["toolResult"]
assert tool_result["toolUseId"] == "tooluse_pdf_1"
assert len(tool_result["content"]) == 1
block = tool_result["content"][0]
assert "document" in block, f"expected document block, got {block}"
assert block["document"]["format"] == "pdf"
assert block["document"]["source"]["bytes"] == pdf_b64
assert block["document"]["name"].startswith("DocumentPDFmessages_")
assert block["document"]["name"].endswith("_pdf")
def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document():
"""
Regression for the processor-returns-document-but-wrapper-drops-it bug:
when a caller sends a PDF as an `image_url` data URI on the tool-result path,
BedrockImageProcessor correctly returns a {"document": ...} block, but the
tool-result wrapper only appended the "image" case, silently dropping documents.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
messages = [
{"role": "user", "content": "Summarize the attached PDF."},
{
"tool_call_id": "tooluse_pdf_img_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:application/pdf;base64,{pdf_b64}",
},
},
],
},
]
translated_msg = _bedrock_converse_messages_pt(
messages=messages, model="", llm_provider=""
)
tool_result = translated_msg[-1]["content"][-1]["toolResult"]
assert tool_result["toolUseId"] == "tooluse_pdf_img_1"
assert len(tool_result["content"]) == 1
block = tool_result["content"][0]
assert "document" in block, f"expected document block, got {block}"
assert block["document"]["format"] == "pdf"
assert block["document"]["source"]["bytes"] == pdf_b64
def test_bedrock_tool_message_file_id_http_url_becomes_document():
"""
OpenAI `file.file_id` is a server-side file reference. The Bedrock
user-message path (_process_file_message at factory.py:4796) accepts either
`file_data` or `file_id` and forwards to BedrockImageProcessor. The
tool-result path must match: when `file_id` is an http(s) PDF URL, it
should resolve to a Bedrock document block, not be silently dropped.
"""
from unittest.mock import patch
from litellm.litellm_core_utils.prompt_templates.factory import (
BedrockImageProcessor,
_bedrock_converse_messages_pt,
)
pdf_url = "https://example.com/whitepaper.pdf"
fake_document_block = {
"document": {
"format": "pdf",
"name": "fake_doc",
"source": {"bytes": "ZmFrZQ=="},
}
}
messages = [
{"role": "user", "content": "Summarize the attached PDF."},
{
"tool_call_id": "tooluse_fid_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "file",
"file": {
"file_id": pdf_url,
"filename": "whitepaper.pdf",
},
},
],
},
]
with patch.object(
BedrockImageProcessor,
"process_image_sync",
return_value=fake_document_block,
) as mock_proc:
translated_msg = _bedrock_converse_messages_pt(
messages=messages, model="", llm_provider=""
)
mock_proc.assert_called_once()
assert mock_proc.call_args.kwargs["image_url"] == pdf_url
tool_result = translated_msg[-1]["content"][-1]["toolResult"]
assert len(tool_result["content"]) == 1
block = tool_result["content"][0]
assert "document" in block, f"expected document block, got {block}"
assert block["document"]["source"]["bytes"] == "ZmFrZQ=="
def test_bedrock_tool_message_file_without_data_or_id_raises():
"""
The user-message path raises BadRequestError when a `type: "file"` block
has neither `file_data` nor `file_id` (factory.py:4802-4809). The
tool-result path must match — silently dropping the block makes the model
see an empty tool result and obscures the caller bug.
"""
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
messages = [
{"role": "user", "content": "Summarize."},
{
"tool_call_id": "tooluse_bad_1",
"role": "tool",
"name": "fetch_document",
"content": [
{
"type": "file",
"file": {"filename": "nothing.pdf"},
},
],
},
]
with pytest.raises(litellm.BadRequestError):
_bedrock_converse_messages_pt(messages=messages, model="", llm_provider="")
def test_bedrock_tool_message_image_url_png_still_becomes_image():
"""
Regression: image_url with an image mime type must continue to translate
to a Bedrock image block (not document). Locks in existing behavior after
the document-passthrough fix.
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
messages = [
{"role": "user", "content": "Describe the attached image."},
{
"tool_call_id": "tooluse_png_1",
"role": "tool",
"name": "fetch_image",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{png_b64}",
},
},
],
},
]
translated_msg = _bedrock_converse_messages_pt(
messages=messages, model="", llm_provider=""
)
tool_result = translated_msg[-1]["content"][-1]["toolResult"]
assert len(tool_result["content"]) == 1
block = tool_result["content"][0]
assert "image" in block, f"expected image block, got {block}"
assert "document" not in block
assert block["image"]["format"] == "png"
assert block["image"]["source"]["bytes"] == png_b64