diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fe8387476ee..a9df0895572 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: return sanitized +_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"} + + +def _is_anthropic_document_data_uri(url: str) -> bool: + # Anthropic's base64 document source accepts only application/pdf and + # text/plain (see select_anthropic_content_block_type_for_file). Routing + # other mimes here would produce a document block the API rejects, so we + # leave them on the image code path. + match = re.match(r"data:([^;,]+)", url) + if not match: + return False + return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES + + def convert_to_anthropic_tool_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], force_base64: bool = False, @@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result( """ anthropic_content: Union[ str, - List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]], + List[ + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] + ], ] = "" if isinstance(message["content"], str): anthropic_content = message["content"] elif isinstance(message["content"], List): content_list = message["content"] anthropic_content_list: List[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ] = [] for content in content_list: if content["type"] == "text": @@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result( text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": + image_url_value = content["image_url"] format = ( - content["image_url"].get("format") - if isinstance(content["image_url"], dict) + image_url_value.get("format") + if isinstance(image_url_value, dict) else None ) - _anthropic_image_param = create_anthropic_image_param( - content["image_url"], format=format, is_bedrock_invoke=force_base64 + url_str = ( + image_url_value.get("url") + if isinstance(image_url_value, dict) + else image_url_value ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, + # Data URIs with non-image mime types (e.g. application/pdf) must + # translate to Anthropic document blocks, not image blocks — + # wrapping a PDF in `type: "image"` is rejected by the API. + if isinstance(url_str, str) and _is_anthropic_document_data_uri( + url_str + ): + synth_file_message: ChatCompletionFileObject = { + "type": "file", + "file": {"file_data": url_str}, + } + _document_block = anthropic_process_openai_file_message( + synth_file_message + ) + _document_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _document_block + ), + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesDocumentParam, _document_block) + ) + else: + _anthropic_image_param = create_anthropic_image_param( + image_url_value, + format=format, + is_bedrock_invoke=force_base64, + ) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesImageParam, _anthropic_image_param) + ) + elif content["type"] == "file": + file_content = cast(ChatCompletionFileObject, content) + _file_block = anthropic_process_openai_file_message(file_content) + _file_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _file_block + ), original_content_element=content, ) - anthropic_content_list.append( - cast(AnthropicMessagesImageParam, _anthropic_image_param) - ) + anthropic_content_list.append(_file_block) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None @@ -3977,6 +4042,41 @@ def _convert_to_bedrock_tool_call_result( tool_result_content_blocks.append( BedrockToolResultContentBlock(image=_block["image"]) ) + elif "document" in _block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_block["document"]) + ) + elif content["type"] == "file": + # Match the user-message path (_process_file_message): accept + # either file_data (base64 data URI) or file_id (server-side + # reference / URL) and hand off to BedrockImageProcessor. Raise + # BadRequestError on both-None rather than silently dropping. + file_obj = content.get("file") or {} + file_data = file_obj.get("file_data") + file_id = file_obj.get("file_id") + if file_data is None and file_id is None: + raise litellm.BadRequestError( + message="file_data and file_id cannot both be None. Got={}".format( + content + ), + model="", + llm_provider="bedrock", + ) + file_format = file_obj.get("format") + _file_block: BedrockContentBlock = ( + BedrockImageProcessor.process_image_sync( + image_url=cast(str, file_id or file_data), + format=file_format, + ) + ) + if "document" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_file_block["document"]) + ) + elif "image" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=_file_block["image"]) + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index e3f63d05742..c376b8694af 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -330,7 +330,11 @@ class AnthropicMessagesToolResultParam(TypedDict, total=False): content: Union[ str, Iterable[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ], ] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index f8708dd2f75..d424cd8599f 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2476,3 +2476,186 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): cache_blocks_old = [b for b in result_old if "cachePoint" in b] assert len(cache_blocks_old) == 1 assert "ttl" not in cache_blocks_old[0]["cachePoint"] + + +def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document(): + """ + OpenAI `{type: "file", file: {file_data: "data:application/pdf;..."}}` inside + a tool-message content list should translate to an Anthropic document block + inside the tool_result content. Reuses anthropic_process_openai_file_message, + which already handles this for user messages. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + message = { + "tool_call_id": "toolu_pdf_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{pdf_b64}", + "filename": "summary.pdf", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "toolu_pdf_1" + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "document" + assert block["source"]["type"] == "base64" + assert block["source"]["media_type"] == "application/pdf" + assert block["source"]["data"] == pdf_b64 + + +def test_convert_to_anthropic_tool_result_image_url_pdf_data_uri_becomes_document(): + """ + Regression: a PDF sent as an `image_url` data URI on the tool-result path + must translate to an Anthropic document block (not an image block — Anthropic + rejects image blocks whose media_type is a non-image like application/pdf). + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + message = { + "tool_call_id": "toolu_pdf_img_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_b64}", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "document" + assert block["source"]["media_type"] == "application/pdf" + assert block["source"]["data"] == pdf_b64 + + +def test_convert_to_anthropic_tool_result_image_url_unsupported_mime_stays_image_path(): + """ + An `image_url` data URI whose mime is neither application/pdf nor text/plain + (e.g. application/json) must NOT be routed through the document path. Anthropic + only accepts application/pdf and text/plain as base64 document media_types — + anything else would produce a document block the API rejects. The old + (pre-fix) behavior was to wrap such data as an image block, which also + fails but stays on the image code path; preserve that failure mode rather + than switching to a document path that is equally broken. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + message = { + "tool_call_id": "toolu_json_1", + "role": "tool", + "name": "fetch_json", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "data:application/json;base64,eyJrIjoidiJ9", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "image", ( + f"unsupported mime {block.get('source', {}).get('media_type')!r} " + f"should not be routed to document path; got {block}" + ) + + +def test_convert_to_anthropic_tool_result_image_url_text_plain_data_uri_becomes_document(): + """ + text/plain is one of the two mimes Anthropic accepts as a base64 document + media_type. Confirm it routes through the document path so tightening the + gate to {application/pdf, text/plain} (not "application/*") covers both. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + txt_b64 = "aGVsbG8=" # "hello" + message = { + "tool_call_id": "toolu_txt_1", + "role": "tool", + "name": "fetch_text", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:text/plain;base64,{txt_b64}", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "document" + assert block["source"]["media_type"] == "text/plain" + assert block["source"]["data"] == txt_b64 + + +def test_convert_to_anthropic_tool_result_image_url_png_still_becomes_image(): + """ + Regression: image_url with a real image mime type must continue to translate + to an Anthropic image block. Locks in existing behavior after the + data-URI-mime-type branching for PDFs. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + message = { + "tool_call_id": "toolu_png_1", + "role": "tool", + "name": "fetch_image", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{png_b64}", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "image" + assert block["source"]["media_type"] == "image/png" diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 8e53e57f1e0..7ee2f0aa37c 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -4146,3 +4146,221 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools( # finish_reason must be "stop", not "tool_calls" assert result.choices[0].finish_reason == "stop" + + +def test_bedrock_tool_message_openai_file_pdf_becomes_document(): + """ + OpenAI Chat Completions `{type: "file", file: {file_data: "data:application/pdf;...", filename}}` + inside a tool message content list should translate to a Bedrock + toolResult.content[].document block. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" # tiny "%PDF-1.4\n" header + messages = [ + {"role": "user", "content": "Summarize the attached PDF."}, + { + "tool_call_id": "tooluse_pdf_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{pdf_b64}", + "filename": "summary.pdf", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert tool_result["toolUseId"] == "tooluse_pdf_1" + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "document" in block, f"expected document block, got {block}" + assert block["document"]["format"] == "pdf" + assert block["document"]["source"]["bytes"] == pdf_b64 + assert block["document"]["name"].startswith("DocumentPDFmessages_") + assert block["document"]["name"].endswith("_pdf") + + +def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document(): + """ + Regression for the processor-returns-document-but-wrapper-drops-it bug: + when a caller sends a PDF as an `image_url` data URI on the tool-result path, + BedrockImageProcessor correctly returns a {"document": ...} block, but the + tool-result wrapper only appended the "image" case, silently dropping documents. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + messages = [ + {"role": "user", "content": "Summarize the attached PDF."}, + { + "tool_call_id": "tooluse_pdf_img_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_b64}", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert tool_result["toolUseId"] == "tooluse_pdf_img_1" + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "document" in block, f"expected document block, got {block}" + assert block["document"]["format"] == "pdf" + assert block["document"]["source"]["bytes"] == pdf_b64 + + +def test_bedrock_tool_message_file_id_http_url_becomes_document(): + """ + OpenAI `file.file_id` is a server-side file reference. The Bedrock + user-message path (_process_file_message at factory.py:4796) accepts either + `file_data` or `file_id` and forwards to BedrockImageProcessor. The + tool-result path must match: when `file_id` is an http(s) PDF URL, it + should resolve to a Bedrock document block, not be silently dropped. + """ + from unittest.mock import patch + + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockImageProcessor, + _bedrock_converse_messages_pt, + ) + + pdf_url = "https://example.com/whitepaper.pdf" + fake_document_block = { + "document": { + "format": "pdf", + "name": "fake_doc", + "source": {"bytes": "ZmFrZQ=="}, + } + } + messages = [ + {"role": "user", "content": "Summarize the attached PDF."}, + { + "tool_call_id": "tooluse_fid_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": { + "file_id": pdf_url, + "filename": "whitepaper.pdf", + }, + }, + ], + }, + ] + + with patch.object( + BedrockImageProcessor, + "process_image_sync", + return_value=fake_document_block, + ) as mock_proc: + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + mock_proc.assert_called_once() + assert mock_proc.call_args.kwargs["image_url"] == pdf_url + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "document" in block, f"expected document block, got {block}" + assert block["document"]["source"]["bytes"] == "ZmFrZQ==" + + +def test_bedrock_tool_message_file_without_data_or_id_raises(): + """ + The user-message path raises BadRequestError when a `type: "file"` block + has neither `file_data` nor `file_id` (factory.py:4802-4809). The + tool-result path must match — silently dropping the block makes the model + see an empty tool result and obscures the caller bug. + """ + import litellm + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + {"role": "user", "content": "Summarize."}, + { + "tool_call_id": "tooluse_bad_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": {"filename": "nothing.pdf"}, + }, + ], + }, + ] + + with pytest.raises(litellm.BadRequestError): + _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="") + + +def test_bedrock_tool_message_image_url_png_still_becomes_image(): + """ + Regression: image_url with an image mime type must continue to translate + to a Bedrock image block (not document). Locks in existing behavior after + the document-passthrough fix. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + messages = [ + {"role": "user", "content": "Describe the attached image."}, + { + "tool_call_id": "tooluse_png_1", + "role": "tool", + "name": "fetch_image", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{png_b64}", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "image" in block, f"expected image block, got {block}" + assert "document" not in block + assert block["image"]["format"] == "png" + assert block["image"]["source"]["bytes"] == png_b64