-- Basic support for indexing and scanning of PDF content --

This is still mostly proof-of-concept stage, but it seems to work on my Mac. It requires the pdftotext utility from the Xpdf project, which parses PDFs into plain text files. The Zotero fulltext indexer calls pdftotext on the PDF file and saves the plaintext version as .zotero-ft-cache in the attachment item's storage directory. It runs the fulltext word indexer on the plaintext file and also scans the plaintext file when doing a phrase search.

To try it out, install a copy of Xpdf (or just pdftotext) and either place pdftotext into the Zotero data directory or create a symlink. Either way, the file must be named pdftotext-{platform}[.exe], where {platform} is navigator.platform, with spaces replaced by hyphens (e.g. "Win32", "Linux-i686", "MacPPC", "MacIntel", etc.). On my Mac, with Xpdf installed via Darwin Ports, I create a symlink to /opt/local/bin/pdftotext named pdftotext-MacIntel. This setup will allow users to sync their Firefox profiles and still have Zotero use the appropriate platform-specific binary.

Assuming we go this pdftotext route, I think we'll instruct users to download and install Xpdf/pdftotext, possibly even providing binaries ourselves. The binaries are too big to include in the XPI. I'm going to look into creating a GUI to make linking Zotero to pdftotext easier. I also need to finish some of the other tickets related to indexer feedback and control.

There are also two new hidden prefs, fulltext.pdfMaxPages and fulltext.textMaxLength, currently set to 100 and 500K, respectively. The first determines how many pages of each PDF pdftotext processes, and the second determines how many characters and/or bytes of text files (the PDF cache files included) Zotero indexes and scans. These defaults may want to be adjusted higher or lower.

Closes #315, Hidden pref to set maximum file size to index/scan

chrome/content/zotero/xpcom/attachments.js

@@ -33,6 +33,7 @@ Zotero.Attachments = new function(){
 	this.linkFromURL = linkFromURL;
 	this.linkFromDocument = linkFromDocument;
 	this.importFromDocument = importFromDocument;
+	this.createDirectoryForItem = createDirectoryForItem;
 	this.getPath = getPath;
 	
 	var self = this;
@@ -52,9 +53,7 @@ Zotero.Attachments = new function(){
 			var itemID = attachmentItem.getID();
 			
 			// Create directory for attachment files within storage directory
-			var destDir = Zotero.getStorageDirectory();
-			destDir.append(itemID);
-			destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+			var destDir = this.createDirectoryForItem(itemID);
 			
 			file.copyTo(destDir, null);
 			
@@ -235,9 +234,7 @@ Zotero.Attachments = new function(){
 					}
 					
 					// Create a new folder for this item in the storage directory
-					var destDir = Zotero.getStorageDirectory();
-					destDir.append(itemID);
-					destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+					var destDir = Zotero.Attachments.createDirectoryForItem(itemID);
 					
 					var file = Components.classes["@mozilla.org/file/local;1"].
 							createInstance(Components.interfaces.nsILocalFile);
@@ -248,6 +245,16 @@ Zotero.Attachments = new function(){
 						try {
 							_addToDB(file, url, title, Zotero.Attachments.LINK_MODE_IMPORTED_URL,
 								mimeType, null, sourceItemID, itemID);
+							
+							// We don't have any way of knowing that the file
+							// is flushed to disk, so we just wait a second
+							// and hope for the best -- we'll index it later
+							// if it fails
+							var timer = Components.classes["@mozilla.org/timer;1"].
+								createInstance(Components.interfaces.nsITimer);
+							timer.initWithCallback({notify: function() {
+								Zotero.Fulltext.indexItems([itemID]);
+							}}, 1000, Components.interfaces.nsITimer.TYPE_ONE_SHOT);
 						}
 						catch (e) {
 							// Clean up
@@ -356,8 +363,13 @@ Zotero.Attachments = new function(){
 		
 		// Run the fulltext indexer asynchronously (actually, it hangs the UI
 		// thread, but at least it lets the menu close)
-		setTimeout(function(){
-			Zotero.Fulltext.indexDocument(document, itemID);
+		setTimeout(function() {
+			if (Zotero.Fulltext.isCachedMIMEType(mimeType)) {
+				Zotero.Fulltext.indexItems([itemID]);
+			}
+			else {
+				Zotero.Fulltext.indexDocument(document, itemID);
+			}
 		}, 50);
 		
 		return itemID;
@@ -403,9 +415,7 @@ Zotero.Attachments = new function(){
 			var itemID = attachmentItem.getID();
 			
 			// Create a new folder for this item in the storage directory
-			var destDir = Zotero.getStorageDirectory();
-			destDir.append(itemID);
-			destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+			var destDir = this.createDirectoryForItem(itemID);
 			
 			var file = Components.classes["@mozilla.org/file/local;1"].
 					createInstance(Components.interfaces.nsILocalFile);
@@ -499,6 +509,19 @@ Zotero.Attachments = new function(){
 	}
 	
 	
+	/*
+	 * Create directory for attachment files within storage directory
+	 */
+	function createDirectoryForItem(itemID) {
+		var dir = Zotero.getStorageDirectory();
+		dir.append(itemID);
+		if (!dir.exists()) {
+			dir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755);
+		}
+		return dir;
+	}
+	
+	
 	/*
 	 * Gets a relative path for imported attachments and an absolute path
 	 * for files outside the storage directory
@@ -635,10 +658,17 @@ Zotero.Attachments = new function(){
 	 * asynchronously after the fact
 	 */
 	function _postProcessFile(itemID, file, mimeType){
+		// MIME types that get cached by the fulltext indexer can just be
+		// indexed directly
+		if (Zotero.Fulltext.isCachedMIMEType(mimeType)) {
+			Zotero.Fulltext.indexItems([itemID]);
+			return;
+		}
+		
 		var ext = Zotero.File.getExtension(file);
 		if (mimeType.substr(0, 5)!='text/' ||
 			!Zotero.MIME.hasInternalHandler(mimeType, ext)){
-			return false;
+			return;
 		}
 		
 		var browser = Zotero.Browser.createHiddenBrowser();

chrome/content/zotero/xpcom/data_access.js

@@ -1972,9 +1972,8 @@ Zotero.Item.prototype.erase = function(deleteChildren){
 		// Delete associated files
 		var linkMode = this.getAttachmentLinkMode();
 		switch (linkMode){
-			case Zotero.Attachments.LINK_MODE_LINKED_FILE:
+			// Link only -- nothing to delete
 			case Zotero.Attachments.LINK_MODE_LINKED_URL:
-				// Links only -- nothing to delete
 				break;
 			default:
 				var file = Zotero.getStorageDirectory();

chrome/content/zotero/xpcom/file.js

@@ -74,7 +74,7 @@ Zotero.File = new function(){
 	}
 	
 	
-	function getContents(file, charset){
+	function getContents(file, charset, maxLength){
 		var fis = Components.classes["@mozilla.org/network/file-input-stream;1"].
 			createInstance(Components.interfaces.nsIFileInputStream);
 		fis.init(file, false, false, false);
@@ -92,9 +92,19 @@ Zotero.File = new function(){
 		var is = Components.classes["@mozilla.org/intl/converter-input-stream;1"]
 			.createInstance(Components.interfaces.nsIConverterInputStream);
 		is.init(fis, charset, 1024, replacementChar);
+		var chars = 1024;
 		
 		var contents = [], str = {};
 		while (is.readString(4096, str) != 0) {
+			if (maxLength) {
+				chars += 4096;
+				if (chars >= maxLength) {
+					Zotero.debug('Stopping at ' + (chars - 4096)
+						+ ' characters in File.getContents()');
+					break;
+				}
+			}
+			
 			contents.push(str.value);
 		}
 		

chrome/content/zotero/xpcom/fulltext.js

@@ -21,23 +21,79 @@
 */
 
 Zotero.Fulltext = new function(){
+	const FULLTEXT_VERSION = 1;
+	const CACHE_FILE = '.zotero-ft-cache';
+	
+	this.init = init;
+	this.registerPDFToText = registerPDFToText;
+	this.cacheIsOutdated = cacheIsOutdated;
+	this.rebuildCache = rebuildCache;
+	this.isCachedMIMEType = isCachedMIMEType;
 	this.indexWord = indexWord;
 	this.indexWords = indexWords;
 	this.indexDocument = indexDocument;
 	this.indexString = indexString;
 	this.indexFile = indexFile;
+	this.indexPDF = indexPDF;
 	this.indexItems = indexItems;
 	this.findTextInFile = findTextInFile;
 	this.findTextInItems = findTextInItems;
-	this.cacheIsOutdated = cacheIsOutdated;
-	this.rebuildCache = rebuildCache;
 	this.clearItemWords = clearItemWords;
 	//this.clearItemContent = clearItemContent;
 	this.purgeUnusedWords = purgeUnusedWords;
 	this.HTMLToText = HTMLToText;
 	this.semanticSplitter = semanticSplitter;
 	
-	const FULLTEXT_VERSION = 1;
+	var _pdftotext = null;
+	
+	
+	function init() {
+		this.registerPDFToText();
+	}
+	
+	
+	/*
+	 * Looks for pdftotext-{platform}[.exe] in the Zotero data directory
+	 *
+	 * {platform} is navigator.platform, with spaces replaced by hyphens
+	 *   e.g. "Win32", "Linux-i686", "MacPPC", "MacIntel", etc.
+	 */
+	function registerPDFToText() {
+		var exec = Zotero.getZoteroDirectory();
+		var fileName = 'pdftotext-' + Zotero.platform.replace(' ', '-');
+		if (Zotero.isWin) {
+			fileName += '.exe';
+		}
+		
+		var errMsg = false;
+		
+		exec.append(fileName);
+		if (exec.exists()) {
+			if (exec.isSymlink()) {
+				exec = exec.target;
+				if (!exec.target) {
+					errMsg = fileName + ' symlink target not found';
+				}
+				else {
+					_pdftotext = exec;
+				}
+			}
+			else {
+				_pdftotext = exec;
+			}
+		}
+		else {
+			errMsg = fileName + ' not found';
+		}
+		
+		if (_pdftotext) {
+			Zotero.debug('pdftotext registered at ' + _pdftotext.path);
+		}
+		else {
+			_pdftotext = null;
+			Zotero.debug(errMsg + ' -- PDF indexing disabled');
+		}
+	}
 	
 	
 	function cacheIsOutdated(){
@@ -54,14 +110,25 @@ Zotero.Fulltext = new function(){
 		
 		var sql = "SELECT itemID FROM itemAttachments";
 		var items = Zotero.DB.columnQuery(sql);
-		indexItems(items);
+		this.indexItems(items);
 		
 		Zotero.DB.commitTransaction();
 	}
 	
 	
+	function isCachedMIMEType(mimeType) {
+		switch (mimeType) {
+			case 'application/pdf':
+				return true;
+		}
+		return false;
+	}
+	
+	
 	/*
 	 * Index a single word
+	 *
+	 * Note: not used
 	 */
 	function indexWord(itemID, word){
 		Zotero.DB.beginTransaction();
@@ -140,8 +207,8 @@ Zotero.Fulltext = new function(){
 		
 		Zotero.DB.beginTransaction();
 		
-		clearItemWords(itemID);
-		indexWords(itemID, words);
+		this.clearItemWords(itemID);
+		this.indexWords(itemID, words);
 		
 		/*
 		var sql = "REPLACE INTO fulltextContent (itemID, textContent) VALUES (?,?)";
@@ -169,9 +236,18 @@ Zotero.Fulltext = new function(){
 			return false;
 		}
 		
-		var text = document.body.innerHTML.replace(/(>)/g, '$1 ');
-		text = HTMLToText(text);
-		indexString(text, document.characterSet, itemID);
+		var text = document.body.innerHTML;
+		
+		var max = Zotero.Prefs.get('fulltext.textMaxLength');
+		if (text.length > max) {
+			Zotero.debug('Only indexing first ' + max + ' characters of item '
+				+ itemID + ' in indexDocument()');
+			text = text.substr(0, max);
+		}
+		
+		text = text.replace(/(>)/g, '$1 ');
+		text = this.HTMLToText(text);
+		this.indexString(text, document.characterSet, itemID);
 	}
 	
 	
@@ -188,6 +264,10 @@ Zotero.Fulltext = new function(){
 			return false;
 		}
 		
+		if (mimeType == 'application/pdf') {
+			return this.indexPDF(file, itemID);
+		}
+		
 		if (mimeType.substr(0, 5)!='text/'){
 			Zotero.debug('File is not text in indexFile()', 2);
 			return false;
@@ -198,15 +278,62 @@ Zotero.Fulltext = new function(){
 			return false;
 		}
 		
-		var text = Zotero.File.getContents(file, charset);
+		Zotero.debug('Indexing file ' + file.path);
+		
+		var maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
+		var text = Zotero.File.getContents(file, charset, maxLength);
 		// Split elements to avoid word concatentation
 		text = text.replace(/(>)/g, '$1 ');
-		text = HTMLToText(text);
-		indexString(text, charset, itemID);
+		text = this.HTMLToText(text);
+		this.indexString(text, charset, itemID);
+		
+		return true;
+	}
+	
+	
+	/*
+	 * Run PDF through pdftotext to generate .zotero-ft-cache and pass the
+	 * text file back to indexFile()
+	 */
+	function indexPDF(file, itemID) {
+		if (!_pdftotext) {
+			return false;
+		}
+		
+		var item = Zotero.Items.get(itemID);
+		var linkMode = item.getAttachmentLinkMode();
+		// If file is stored outside of Zotero, create a directory for the item
+		// in the storage directory and save the cache file there
+		if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE) {
+			var cacheFile = Zotero.Attachments.createDirectoryForItem(itemID);
+		}
+		else {
+			var cacheFile = file.parent;
+		}
+		cacheFile.append(CACHE_FILE);
+		
+		var proc = Components.classes["@mozilla.org/process/util;1"].
+				createInstance(Components.interfaces.nsIProcess);
+		proc.init(_pdftotext);
+		var maxPages = Zotero.Prefs.get('fulltext.pdfMaxPages');
+		Zotero.debug('Running pdftotext -nopgbrk -l ' + maxPages +
+			' "' + file.path + '" "' + cacheFile.path + '"');
+		var args = ['-nopgbrk', '-l', maxPages, file.path, cacheFile.path];
+		Zotero.debug(args);
+		proc.run(true, args, args.length);
+		
+		if (cacheFile.exists()) {
+			return this.indexFile(cacheFile, 'text/plain', 'utf-8', itemID);
+		}
+		return false;
 	}
 	
 	
 	function indexItems(items){
+		if (items.constructor.name != 'Array') {
+			items = [items];
+		}
+		
 		var items = Zotero.Items.get(items);
 		var found = [];
 		
@@ -222,7 +349,7 @@ Zotero.Fulltext = new function(){
 				continue;
 			}
 			
-			indexFile(file, i.getAttachmentMimeType(),
+			this.indexFile(file, i.getAttachmentMimeType(),
 				i.getAttachmentCharset(), i.getID());
 		}
 		
@@ -247,7 +374,8 @@ Zotero.Fulltext = new function(){
 	function findTextInFile(file, charset, searchText, mode){
 		Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
 		
-		var str = Zotero.File.getContents(file, charset);
+		var maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
+		var str = Zotero.File.getContents(file, charset, maxLength);
 		
 		// If not binary mode, convert HTML to text
 		if (!mode || mode.indexOf('Binary')==-1){
@@ -255,7 +383,7 @@ Zotero.Fulltext = new function(){
 			str = str.replace(/(>)/g, '$1 ');
 			
 			// Parse to avoid searching on HTML
-			str = HTMLToText(str);
+			str = this.HTMLToText(str);
 		}
 		
 		switch (mode){
@@ -332,17 +460,30 @@ Zotero.Fulltext = new function(){
 				continue;
 			}
 			
-			// If not binary mode, only scan plaintext files
-			if (!mode || mode.indexOf('Binary')==-1){
-				if (!Zotero.MIME.isTextType(i.getAttachmentMimeType())) {
-					Zotero.debug('Not scanning MIME type ' + mimeType, 4);
+			var mimeType = i.getAttachmentMimeType();
+			
+			if (isCachedMIMEType(mimeType)) {
+				var file = _getItemCacheFile(i.getID());
+				if (!file.exists()) {
 					continue;
 				}
+				
+				mimeType = 'text/plain';
+				var charset = 'utf-8';
+			}
+			else {
+				// If not binary mode, only scan plaintext files
+				if (!mode || mode.indexOf('Binary') == -1) {
+					if (!Zotero.MIME.isTextType(mimeType)) {
+						Zotero.debug('Not scanning MIME type ' + mimeType, 4);
+						continue;
+					}
+				}
+				
+				var charset = i.getAttachmentCharset();
 			}
 			
-			var charset = i.getAttachmentCharset();
-			
-			var match = findTextInFile(file, charset, searchText, mode);
+			var match = this.findTextInFile(file, charset, searchText, mode);
 			
 			if (match != -1){
 				found.push({id:i.getID(), match:match});
@@ -355,6 +496,17 @@ Zotero.Fulltext = new function(){
 	
 	function clearItemWords(itemID){
 		Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=" + itemID);
+		
+		// Delete fulltext cache file if there is one
+		var item = Zotero.Items.get(itemID);
+		switch (item.getAttachmentMimeType()) {
+			case 'application/pdf':
+				var cacheFile = _getItemCacheFile();
+				if (cacheFile.exists()) {
+					cacheFile.remove(null);
+				}
+				break;
+		}
 	}
 	
 	
@@ -467,4 +619,12 @@ Zotero.Fulltext = new function(){
 		text = text.replace("zoteroapostrophe", "'");
 		return text;
 	}
+	
+	
+	function _getItemCacheFile(itemID) {
+		var cacheFile = Zotero.getStorageDirectory();
+		cacheFile.append(itemID);
+		cacheFile.append(CACHE_FILE);
+		return cacheFile;
+	}
 }

chrome/content/zotero/xpcom/zotero.js

@@ -98,7 +98,6 @@ var Zotero = new function(){
 		var gExtensionManager =
 			Components.classes["@mozilla.org/extensions/manager;1"]
 				.getService(Components.interfaces.nsIExtensionManager);
-		var itemType = nsIUpdateItem.TYPE_EXTENSION;
 		this.version
 			= gExtensionManager.getItemForID(ZOTERO_CONFIG['GUID']).version;
 		
@@ -131,6 +130,8 @@ var Zotero = new function(){
 		Zotero.DB.addCallback('commit', Zotero.Notifier.commit);
 		Zotero.DB.addCallback('rollback', Zotero.Notifier.reset);
 		
+		Zotero.Fulltext.init();
+		
 		// Trigger updating of schema and scrapers
 		Zotero.Schema.updateSchema();
 		Zotero.Schema.updateScrapersRemote();

defaults/preferences/zotero.js

@@ -16,7 +16,6 @@ pref("extensions.zotero.automaticTags",true);
 pref("extensions.zotero.fontSize", "1.0");
 pref("extensions.zotero.recursiveCollections", false);
 
-
 pref("extensions.zotero.lastCreatorFieldMode",0);
 pref("extensions.zotero.lastAbstractExpand",0);
 pref("extensions.zotero.lastRenameAssociatedFile", false);
@@ -32,6 +31,10 @@ pref("extensions.zotero.keys.newNote", 'O');
 pref("extensions.zotero.keys.toggleTagSelector", 'T');
 pref("extensions.zotero.keys.copySelectedItemsToClipboard", 'C');
 
+// Fulltext indexing
+pref("extensions.zotero.fulltext.textMaxLength", 500000);
+pref("extensions.zotero.fulltext.pdfMaxPages", 100);
+
 // Export and citation settings
 pref("extensions.zotero.export.lastTranslator", '14763d24-8ba0-45df-8f52-b8d1108e7ac9');
 pref("extensions.zotero.export.translatorSettings", 'true,false');