diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fe8387476ee..2d149b3a70d 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1661,6 +1661,16 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str: return sanitized +def _is_anthropic_document_data_uri(url: str) -> bool: + # Anthropic document blocks cover non-image mimes the API accepts via base64 + # source (application/pdf, text/*). Match the mime-type prefix in a data URI. + match = re.match(r"data:([^;,]+)", url) + if not match: + return False + mime_type = match.group(1) + return mime_type.startswith("application/") or mime_type.startswith("text/") + + def convert_to_anthropic_tool_result( message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], force_base64: bool = False, @@ -1698,14 +1708,24 @@ def convert_to_anthropic_tool_result( """ anthropic_content: Union[ str, - List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]], + List[ + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] + ], ] = "" if isinstance(message["content"], str): anthropic_content = message["content"] elif isinstance(message["content"], List): content_list = message["content"] anthropic_content_list: List[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ] = [] for content in content_list: if content["type"] == "text": @@ -1720,21 +1740,62 @@ def convert_to_anthropic_tool_result( text_content["cache_control"] = cache_control_value anthropic_content_list.append(text_content) elif content["type"] == "image_url": + image_url_value = content["image_url"] format = ( - content["image_url"].get("format") - if isinstance(content["image_url"], dict) + image_url_value.get("format") + if isinstance(image_url_value, dict) else None ) - _anthropic_image_param = create_anthropic_image_param( - content["image_url"], format=format, is_bedrock_invoke=force_base64 + url_str = ( + image_url_value.get("url") + if isinstance(image_url_value, dict) + else image_url_value ) - _anthropic_image_param = add_cache_control_to_content( - anthropic_content_element=_anthropic_image_param, + # Data URIs with non-image mime types (e.g. application/pdf) must + # translate to Anthropic document blocks, not image blocks — + # wrapping a PDF in `type: "image"` is rejected by the API. + if isinstance(url_str, str) and _is_anthropic_document_data_uri( + url_str + ): + synth_file_message: ChatCompletionFileObject = { + "type": "file", + "file": {"file_data": url_str}, + } + _document_block = anthropic_process_openai_file_message( + synth_file_message + ) + _document_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _document_block + ), + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesDocumentParam, _document_block) + ) + else: + _anthropic_image_param = create_anthropic_image_param( + image_url_value, + format=format, + is_bedrock_invoke=force_base64, + ) + _anthropic_image_param = add_cache_control_to_content( + anthropic_content_element=_anthropic_image_param, + original_content_element=content, + ) + anthropic_content_list.append( + cast(AnthropicMessagesImageParam, _anthropic_image_param) + ) + elif content["type"] == "file": + file_content = cast(ChatCompletionFileObject, content) + _file_block = anthropic_process_openai_file_message(file_content) + _file_block = add_cache_control_to_content( + anthropic_content_element=cast( + AnthropicMessagesDocumentParam, _file_block + ), original_content_element=content, ) - anthropic_content_list.append( - cast(AnthropicMessagesImageParam, _anthropic_image_param) - ) + anthropic_content_list.append(_file_block) anthropic_content = anthropic_content_list anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None @@ -3977,6 +4038,27 @@ def _convert_to_bedrock_tool_call_result( tool_result_content_blocks.append( BedrockToolResultContentBlock(image=_block["image"]) ) + elif "document" in _block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(document=_block["document"]) + ) + elif content["type"] == "file": + file_obj = content.get("file") or {} + file_data = file_obj.get("file_data") + if isinstance(file_data, str): + _file_block: BedrockContentBlock = ( + BedrockImageProcessor.process_image_sync(image_url=file_data) + ) + if "document" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock( + document=_file_block["document"] + ) + ) + elif "image" in _file_block: + tool_result_content_blocks.append( + BedrockToolResultContentBlock(image=_file_block["image"]) + ) message.get("name", "") id = str(message.get("tool_call_id", str(uuid.uuid4()))) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index e3f63d05742..c376b8694af 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -330,7 +330,11 @@ class AnthropicMessagesToolResultParam(TypedDict, total=False): content: Union[ str, Iterable[ - Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam] + Union[ + AnthropicMessagesToolResultContent, + AnthropicMessagesImageParam, + AnthropicMessagesDocumentParam, + ] ], ] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index fdf8c24ac9e..75a9c4c39a1 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1885,3 +1885,113 @@ def test_metadata_filter_applies_to_azure_anthropic(): headers={}, ) assert data.get("metadata") == {"user_id": "u2"} + + +def test_anthropic_tool_result_openai_file_pdf_becomes_document(): + """ + OpenAI `{type: "file", file: {file_data: "data:application/pdf;..."}}` inside + a tool-message content list should translate to an Anthropic document block + inside the tool_result content. The existing helper + `anthropic_process_openai_file_message` already does this translation for + user messages; it must be reused on the tool-result path. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + message = { + "tool_call_id": "toolu_pdf_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{pdf_b64}", + "filename": "summary.pdf", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + assert result["type"] == "tool_result" + assert result["tool_use_id"] == "toolu_pdf_1" + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "document" + assert block["source"]["type"] == "base64" + assert block["source"]["media_type"] == "application/pdf" + assert block["source"]["data"] == pdf_b64 + + +def test_anthropic_tool_result_image_url_pdf_data_uri_becomes_document(): + """ + Regression: a PDF sent as an `image_url` data URI on the tool-result path + must translate to an Anthropic document block (not an image block — Anthropic + rejects image blocks whose media_type is a non-image like application/pdf). + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + message = { + "tool_call_id": "toolu_pdf_img_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_b64}", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "document" + assert block["source"]["media_type"] == "application/pdf" + assert block["source"]["data"] == pdf_b64 + + +def test_anthropic_tool_result_image_url_png_still_becomes_image(): + """ + Regression: image_url with a real image mime type must continue to translate + to an Anthropic image block. Locks in existing behavior after the + data-URI-mime-type branching for PDFs. + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_tool_result, + ) + + png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + message = { + "tool_call_id": "toolu_png_1", + "role": "tool", + "name": "fetch_image", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{png_b64}", + }, + }, + ], + } + + result = convert_to_anthropic_tool_result(message) + + content = result["content"] + assert isinstance(content, list) and len(content) == 1 + block = content[0] + assert block["type"] == "image" + assert block["source"]["media_type"] == "image/png" diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index ddfe383f2a5..c9a886d125e 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -1282,6 +1282,123 @@ def test_bedrock_converse_translation_tool_message(): ] +def test_bedrock_tool_message_openai_file_pdf_becomes_document(): + """ + OpenAI Chat Completions `{type: "file", file: {file_data: "data:application/pdf;...", filename}}` + inside a tool message content list should translate to a Bedrock + toolResult.content[].document block. This is the documented OpenAI shape for + PDFs on the Chat Completions API and what downstream callers emit. + """ + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" # tiny "%PDF-1.4\n" header + messages = [ + {"role": "user", "content": "Summarize the attached PDF."}, + { + "tool_call_id": "tooluse_pdf_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{pdf_b64}", + "filename": "summary.pdf", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert tool_result["toolUseId"] == "tooluse_pdf_1" + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "document" in block, f"expected document block, got {block}" + assert block["document"]["format"] == "pdf" + assert block["document"]["source"]["bytes"] == pdf_b64 + assert block["document"]["name"].startswith("DocumentPDFmessages_") + assert block["document"]["name"].endswith("_pdf") + + +def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document(): + """ + Regression for the processor-returns-document-but-wrapper-drops-it bug: + when a caller sends a PDF as an `image_url` data URI on the tool-result path, + BedrockImageProcessor correctly routes it through the document path and + returns a {"document": ...} block, but the tool-result wrapper only + appended the "image" case, silently dropping documents. + """ + pdf_b64 = "JVBERi0xLjQKJeLjz9MK" + messages = [ + {"role": "user", "content": "Summarize the attached PDF."}, + { + "tool_call_id": "tooluse_pdf_img_1", + "role": "tool", + "name": "fetch_document", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:application/pdf;base64,{pdf_b64}", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert tool_result["toolUseId"] == "tooluse_pdf_img_1" + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "document" in block, f"expected document block, got {block}" + assert block["document"]["format"] == "pdf" + assert block["document"]["source"]["bytes"] == pdf_b64 + + +def test_bedrock_tool_message_image_url_png_still_becomes_image(): + """ + Regression: image_url with an image mime type must continue to translate + to a Bedrock image block (not document). Locks in existing behavior after + the document-passthrough fix. + """ + png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg==" + messages = [ + {"role": "user", "content": "Describe the attached image."}, + { + "tool_call_id": "tooluse_png_1", + "role": "tool", + "name": "fetch_image", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{png_b64}", + }, + }, + ], + }, + ] + + translated_msg = _bedrock_converse_messages_pt( + messages=messages, model="", llm_provider="" + ) + + tool_result = translated_msg[-1]["content"][-1]["toolResult"] + assert len(tool_result["content"]) == 1 + block = tool_result["content"][0] + assert "image" in block, f"expected image block, got {block}" + assert "document" not in block + assert block["image"]["format"] == "png" + assert block["image"]["source"]["bytes"] == png_b64 + + def test_base_aws_llm_get_credentials(): import time