From 15013cec4b4fc977f2259f56eec2fe88498174d4 Mon Sep 17 00:00:00 2001 From: Ryne Carbone Date: Tue, 20 Jan 2026 22:54:12 -0500 Subject: [PATCH] feat(gemini): add file content support in tool results (#19416) Add support for 'file' and 'input_file' content types in convert_to_gemini_tool_call_result(). File content in tool results was previously silently dropped. Supports base64 data URIs and HTTP URLs, matching the existing image handling pattern. Enables PDF, audio, video, and other file types as inline_data for Gemini. --- .../prompt_templates/factory.py | 29 ++- .../test_vertex_ai_gemini_transformation.py | 181 +++++++++++++++++- 2 files changed, 208 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 26f5eb7e73b..30263543fc6 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1462,7 +1462,7 @@ def convert_to_gemini_tool_call_invoke( ) -def convert_to_gemini_tool_call_result( +def convert_to_gemini_tool_call_result( # noqa: PLR0915 message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], last_message_with_tool_calls: Optional[dict], ) -> Union[VertexPartType, List[VertexPartType]]: @@ -1529,6 +1529,33 @@ def convert_to_gemini_tool_call_result( verbose_logger.warning( f"Failed to process image in tool response: {e}" ) + elif content_type in ("file", "input_file"): + # Extract file for inline_data (for tool results with PDF, audio, video, etc.) + file_data = content.get("file_data", "") + if not file_data: + file_content = content.get("file", {}) + file_data = ( + file_content.get("file_data", "") + if isinstance(file_content, dict) + else file_content + if isinstance(file_content, str) + else "" + ) + + if file_data: + # Convert file to base64 blob format for Gemini + try: + file_obj = convert_to_anthropic_image_obj( + file_data, format=None + ) + inline_data = BlobType( + data=file_obj["data"], + mime_type=file_obj["media_type"], + ) + except Exception as e: + verbose_logger.warning( + f"Failed to process file in tool response: {e}" + ) name: Optional[str] = message.get("name", "") # type: ignore # Recover name from last message with tool calls diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 70e6e9452e5..33fd11dfbee 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -903,7 +903,186 @@ def test_extract_file_data_fallback_to_octet_stream(): # Verify MIME type falls back to octet-stream assert extracted["content_type"] == "application/octet-stream", \ f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" - + finally: # Clean up temporary file os.unlink(tmp_path) + + +def test_convert_tool_response_with_pdf_file(): + """Test tool response with PDF file content using file_data field.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with file + tool_message = { + "role": "tool", + "tool_call_id": "call_pdf_test", + "content": [ + { + "type": "text", + "text": '{"status": "success", "pages": 1}' + }, + { + "type": "file", + "file_data": file_data_uri + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_pdf_test", + "function": { + "name": "analyze_document", + "arguments": '{"path": "/tmp/doc.pdf"}' + } + } + ] + } + + # Convert tool response (returns list when file is present) + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results - should be a list with 2 parts (function_response + inline_data) + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find function_response part and inline_data part + function_response_part = None + inline_data_part = None + for part in result: + if "function_response" in part: + function_response_part = part + elif "inline_data" in part: + inline_data_part = part + + # Check function_response exists + assert function_response_part is not None, "Missing function_response part" + function_response = function_response_part["function_response"] + assert function_response["name"] == "analyze_document" + assert "response" in function_response + # Verify JSON response is parsed correctly + assert "status" in function_response["response"] + assert function_response["response"]["status"] == "success" + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64 + + +def test_convert_tool_response_with_input_file_type(): + """Test tool response with input_file content type (Responses API format).""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with input_file type + tool_message = { + "role": "tool", + "tool_call_id": "call_input_file_test", + "content": [ + { + "type": "input_file", + "file_data": file_data_uri + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_input_file_test", + "function": { + "name": "read_file", + "arguments": "{}" + } + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find inline_data part + inline_data_part = None + for part in result: + if "inline_data" in part: + inline_data_part = part + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + assert inline_data_part["inline_data"]["mime_type"] == "application/pdf" + + +def test_convert_tool_response_with_nested_file_object(): + """Test tool response with file content using nested file object format.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with nested file object (OpenAI Agents SDK format) + tool_message = { + "role": "tool", + "tool_call_id": "call_nested_test", + "content": [ + { + "type": "file", + "file": { + "file_data": file_data_uri + } + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_nested_test", + "function": { + "name": "process_document", + "arguments": "{}" + } + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results - should be a list with 2 parts + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find inline_data part + inline_data_part = None + for part in result: + if "inline_data" in part: + inline_data_part = part + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64