diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 26f5eb7e73b..30263543fc6 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1462,7 +1462,7 @@ def convert_to_gemini_tool_call_invoke( ) -def convert_to_gemini_tool_call_result( +def convert_to_gemini_tool_call_result( # noqa: PLR0915 message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage], last_message_with_tool_calls: Optional[dict], ) -> Union[VertexPartType, List[VertexPartType]]: @@ -1529,6 +1529,33 @@ def convert_to_gemini_tool_call_result( verbose_logger.warning( f"Failed to process image in tool response: {e}" ) + elif content_type in ("file", "input_file"): + # Extract file for inline_data (for tool results with PDF, audio, video, etc.) + file_data = content.get("file_data", "") + if not file_data: + file_content = content.get("file", {}) + file_data = ( + file_content.get("file_data", "") + if isinstance(file_content, dict) + else file_content + if isinstance(file_content, str) + else "" + ) + + if file_data: + # Convert file to base64 blob format for Gemini + try: + file_obj = convert_to_anthropic_image_obj( + file_data, format=None + ) + inline_data = BlobType( + data=file_obj["data"], + mime_type=file_obj["media_type"], + ) + except Exception as e: + verbose_logger.warning( + f"Failed to process file in tool response: {e}" + ) name: Optional[str] = message.get("name", "") # type: ignore # Recover name from last message with tool calls diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 70e6e9452e5..33fd11dfbee 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -903,7 +903,186 @@ def test_extract_file_data_fallback_to_octet_stream(): # Verify MIME type falls back to octet-stream assert extracted["content_type"] == "application/octet-stream", \ f"Expected 'application/octet-stream' for unknown type, got '{extracted['content_type']}'" - + finally: # Clean up temporary file os.unlink(tmp_path) + + +def test_convert_tool_response_with_pdf_file(): + """Test tool response with PDF file content using file_data field.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with file + tool_message = { + "role": "tool", + "tool_call_id": "call_pdf_test", + "content": [ + { + "type": "text", + "text": '{"status": "success", "pages": 1}' + }, + { + "type": "file", + "file_data": file_data_uri + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_pdf_test", + "function": { + "name": "analyze_document", + "arguments": '{"path": "/tmp/doc.pdf"}' + } + } + ] + } + + # Convert tool response (returns list when file is present) + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results - should be a list with 2 parts (function_response + inline_data) + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find function_response part and inline_data part + function_response_part = None + inline_data_part = None + for part in result: + if "function_response" in part: + function_response_part = part + elif "inline_data" in part: + inline_data_part = part + + # Check function_response exists + assert function_response_part is not None, "Missing function_response part" + function_response = function_response_part["function_response"] + assert function_response["name"] == "analyze_document" + assert "response" in function_response + # Verify JSON response is parsed correctly + assert "status" in function_response["response"] + assert function_response["response"]["status"] == "success" + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64 + + +def test_convert_tool_response_with_input_file_type(): + """Test tool response with input_file content type (Responses API format).""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with input_file type + tool_message = { + "role": "tool", + "tool_call_id": "call_input_file_test", + "content": [ + { + "type": "input_file", + "file_data": file_data_uri + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_input_file_test", + "function": { + "name": "read_file", + "arguments": "{}" + } + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find inline_data part + inline_data_part = None + for part in result: + if "inline_data" in part: + inline_data_part = part + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + assert inline_data_part["inline_data"]["mime_type"] == "application/pdf" + + +def test_convert_tool_response_with_nested_file_object(): + """Test tool response with file content using nested file object format.""" + # Create a minimal test PDF (base64 encoded) + test_pdf_base64 = "JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PC9UeXBlL0NhdGFsb2cvUGFnZXMgMiAwIFI+PgplbmRvYmoKdHJhaWxlcgo8PC9TaXplIDQvUm9vdCAxIDAgUj4+CnN0YXJ0eHJlZgoyMTYKJSVFT0Y=" + file_data_uri = f"data:application/pdf;base64,{test_pdf_base64}" + + # Create tool message with nested file object (OpenAI Agents SDK format) + tool_message = { + "role": "tool", + "tool_call_id": "call_nested_test", + "content": [ + { + "type": "file", + "file": { + "file_data": file_data_uri + } + } + ] + } + + # Mock last message with tool calls + last_message_with_tool_calls = { + "tool_calls": [ + { + "id": "call_nested_test", + "function": { + "name": "process_document", + "arguments": "{}" + } + } + ] + } + + # Convert tool response + result = convert_to_gemini_tool_call_result( + tool_message, last_message_with_tool_calls + ) + + # Verify results - should be a list with 2 parts + assert isinstance(result, list), f"Expected list when file present, got {type(result)}" + assert len(result) == 2, f"Expected 2 parts, got {len(result)}" + + # Find inline_data part + inline_data_part = None + for part in result: + if "inline_data" in part: + inline_data_part = part + + # Check inline_data exists + assert inline_data_part is not None, "Missing inline_data part" + inline_data: BlobType = inline_data_part["inline_data"] + assert "data" in inline_data + assert "mime_type" in inline_data + assert inline_data["mime_type"] == "application/pdf" + assert inline_data["data"] == test_pdf_base64