diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index 9069970ba5..652a6d647e 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -33,6 +33,7 @@ Zotero.Attachments = new function(){ this.linkFromURL = linkFromURL; this.linkFromDocument = linkFromDocument; this.importFromDocument = importFromDocument; + this.createDirectoryForItem = createDirectoryForItem; this.getPath = getPath; var self = this; @@ -52,9 +53,7 @@ Zotero.Attachments = new function(){ var itemID = attachmentItem.getID(); // Create directory for attachment files within storage directory - var destDir = Zotero.getStorageDirectory(); - destDir.append(itemID); - destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755); + var destDir = this.createDirectoryForItem(itemID); file.copyTo(destDir, null); @@ -235,9 +234,7 @@ Zotero.Attachments = new function(){ } // Create a new folder for this item in the storage directory - var destDir = Zotero.getStorageDirectory(); - destDir.append(itemID); - destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755); + var destDir = Zotero.Attachments.createDirectoryForItem(itemID); var file = Components.classes["@mozilla.org/file/local;1"]. createInstance(Components.interfaces.nsILocalFile); @@ -248,6 +245,16 @@ Zotero.Attachments = new function(){ try { _addToDB(file, url, title, Zotero.Attachments.LINK_MODE_IMPORTED_URL, mimeType, null, sourceItemID, itemID); + + // We don't have any way of knowing that the file + // is flushed to disk, so we just wait a second + // and hope for the best -- we'll index it later + // if it fails + var timer = Components.classes["@mozilla.org/timer;1"]. + createInstance(Components.interfaces.nsITimer); + timer.initWithCallback({notify: function() { + Zotero.Fulltext.indexItems([itemID]); + }}, 1000, Components.interfaces.nsITimer.TYPE_ONE_SHOT); } catch (e) { // Clean up @@ -356,8 +363,13 @@ Zotero.Attachments = new function(){ // Run the fulltext indexer asynchronously (actually, it hangs the UI // thread, but at least it lets the menu close) - setTimeout(function(){ - Zotero.Fulltext.indexDocument(document, itemID); + setTimeout(function() { + if (Zotero.Fulltext.isCachedMIMEType(mimeType)) { + Zotero.Fulltext.indexItems([itemID]); + } + else { + Zotero.Fulltext.indexDocument(document, itemID); + } }, 50); return itemID; @@ -403,9 +415,7 @@ Zotero.Attachments = new function(){ var itemID = attachmentItem.getID(); // Create a new folder for this item in the storage directory - var destDir = Zotero.getStorageDirectory(); - destDir.append(itemID); - destDir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755); + var destDir = this.createDirectoryForItem(itemID); var file = Components.classes["@mozilla.org/file/local;1"]. createInstance(Components.interfaces.nsILocalFile); @@ -499,6 +509,19 @@ Zotero.Attachments = new function(){ } + /* + * Create directory for attachment files within storage directory + */ + function createDirectoryForItem(itemID) { + var dir = Zotero.getStorageDirectory(); + dir.append(itemID); + if (!dir.exists()) { + dir.create(Components.interfaces.nsIFile.DIRECTORY_TYPE, 0755); + } + return dir; + } + + /* * Gets a relative path for imported attachments and an absolute path * for files outside the storage directory @@ -635,10 +658,17 @@ Zotero.Attachments = new function(){ * asynchronously after the fact */ function _postProcessFile(itemID, file, mimeType){ + // MIME types that get cached by the fulltext indexer can just be + // indexed directly + if (Zotero.Fulltext.isCachedMIMEType(mimeType)) { + Zotero.Fulltext.indexItems([itemID]); + return; + } + var ext = Zotero.File.getExtension(file); if (mimeType.substr(0, 5)!='text/' || !Zotero.MIME.hasInternalHandler(mimeType, ext)){ - return false; + return; } var browser = Zotero.Browser.createHiddenBrowser(); diff --git a/chrome/content/zotero/xpcom/data_access.js b/chrome/content/zotero/xpcom/data_access.js index e147b3859e..d6731dd5c8 100644 --- a/chrome/content/zotero/xpcom/data_access.js +++ b/chrome/content/zotero/xpcom/data_access.js @@ -1972,9 +1972,8 @@ Zotero.Item.prototype.erase = function(deleteChildren){ // Delete associated files var linkMode = this.getAttachmentLinkMode(); switch (linkMode){ - case Zotero.Attachments.LINK_MODE_LINKED_FILE: + // Link only -- nothing to delete case Zotero.Attachments.LINK_MODE_LINKED_URL: - // Links only -- nothing to delete break; default: var file = Zotero.getStorageDirectory(); diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js index 95169c17da..206603b00e 100644 --- a/chrome/content/zotero/xpcom/file.js +++ b/chrome/content/zotero/xpcom/file.js @@ -74,7 +74,7 @@ Zotero.File = new function(){ } - function getContents(file, charset){ + function getContents(file, charset, maxLength){ var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]. createInstance(Components.interfaces.nsIFileInputStream); fis.init(file, false, false, false); @@ -92,9 +92,19 @@ Zotero.File = new function(){ var is = Components.classes["@mozilla.org/intl/converter-input-stream;1"] .createInstance(Components.interfaces.nsIConverterInputStream); is.init(fis, charset, 1024, replacementChar); + var chars = 1024; var contents = [], str = {}; while (is.readString(4096, str) != 0) { + if (maxLength) { + chars += 4096; + if (chars >= maxLength) { + Zotero.debug('Stopping at ' + (chars - 4096) + + ' characters in File.getContents()'); + break; + } + } + contents.push(str.value); } diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index 5c63c76d69..50dd22e3c9 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -21,23 +21,79 @@ */ Zotero.Fulltext = new function(){ + const FULLTEXT_VERSION = 1; + const CACHE_FILE = '.zotero-ft-cache'; + + this.init = init; + this.registerPDFToText = registerPDFToText; + this.cacheIsOutdated = cacheIsOutdated; + this.rebuildCache = rebuildCache; + this.isCachedMIMEType = isCachedMIMEType; this.indexWord = indexWord; this.indexWords = indexWords; this.indexDocument = indexDocument; this.indexString = indexString; this.indexFile = indexFile; + this.indexPDF = indexPDF; this.indexItems = indexItems; this.findTextInFile = findTextInFile; this.findTextInItems = findTextInItems; - this.cacheIsOutdated = cacheIsOutdated; - this.rebuildCache = rebuildCache; this.clearItemWords = clearItemWords; //this.clearItemContent = clearItemContent; this.purgeUnusedWords = purgeUnusedWords; this.HTMLToText = HTMLToText; this.semanticSplitter = semanticSplitter; - const FULLTEXT_VERSION = 1; + var _pdftotext = null; + + + function init() { + this.registerPDFToText(); + } + + + /* + * Looks for pdftotext-{platform}[.exe] in the Zotero data directory + * + * {platform} is navigator.platform, with spaces replaced by hyphens + * e.g. "Win32", "Linux-i686", "MacPPC", "MacIntel", etc. + */ + function registerPDFToText() { + var exec = Zotero.getZoteroDirectory(); + var fileName = 'pdftotext-' + Zotero.platform.replace(' ', '-'); + if (Zotero.isWin) { + fileName += '.exe'; + } + + var errMsg = false; + + exec.append(fileName); + if (exec.exists()) { + if (exec.isSymlink()) { + exec = exec.target; + if (!exec.target) { + errMsg = fileName + ' symlink target not found'; + } + else { + _pdftotext = exec; + } + } + else { + _pdftotext = exec; + } + } + else { + errMsg = fileName + ' not found'; + } + + if (_pdftotext) { + Zotero.debug('pdftotext registered at ' + _pdftotext.path); + } + else { + _pdftotext = null; + Zotero.debug(errMsg + ' -- PDF indexing disabled'); + } + } function cacheIsOutdated(){ @@ -54,14 +110,25 @@ Zotero.Fulltext = new function(){ var sql = "SELECT itemID FROM itemAttachments"; var items = Zotero.DB.columnQuery(sql); - indexItems(items); + this.indexItems(items); Zotero.DB.commitTransaction(); } + function isCachedMIMEType(mimeType) { + switch (mimeType) { + case 'application/pdf': + return true; + } + return false; + } + + /* * Index a single word + * + * Note: not used */ function indexWord(itemID, word){ Zotero.DB.beginTransaction(); @@ -140,8 +207,8 @@ Zotero.Fulltext = new function(){ Zotero.DB.beginTransaction(); - clearItemWords(itemID); - indexWords(itemID, words); + this.clearItemWords(itemID); + this.indexWords(itemID, words); /* var sql = "REPLACE INTO fulltextContent (itemID, textContent) VALUES (?,?)"; @@ -169,9 +236,18 @@ Zotero.Fulltext = new function(){ return false; } - var text = document.body.innerHTML.replace(/(>)/g, '$1 '); - text = HTMLToText(text); - indexString(text, document.characterSet, itemID); + var text = document.body.innerHTML; + + var max = Zotero.Prefs.get('fulltext.textMaxLength'); + if (text.length > max) { + Zotero.debug('Only indexing first ' + max + ' characters of item ' + + itemID + ' in indexDocument()'); + text = text.substr(0, max); + } + + text = text.replace(/(>)/g, '$1 '); + text = this.HTMLToText(text); + this.indexString(text, document.characterSet, itemID); } @@ -188,6 +264,10 @@ Zotero.Fulltext = new function(){ return false; } + if (mimeType == 'application/pdf') { + return this.indexPDF(file, itemID); + } + if (mimeType.substr(0, 5)!='text/'){ Zotero.debug('File is not text in indexFile()', 2); return false; @@ -198,15 +278,62 @@ Zotero.Fulltext = new function(){ return false; } - var text = Zotero.File.getContents(file, charset); + Zotero.debug('Indexing file ' + file.path); + + var maxLength = Zotero.Prefs.get('fulltext.textMaxLength'); + var text = Zotero.File.getContents(file, charset, maxLength); // Split elements to avoid word concatentation text = text.replace(/(>)/g, '$1 '); - text = HTMLToText(text); - indexString(text, charset, itemID); + text = this.HTMLToText(text); + this.indexString(text, charset, itemID); + + return true; + } + + + /* + * Run PDF through pdftotext to generate .zotero-ft-cache and pass the + * text file back to indexFile() + */ + function indexPDF(file, itemID) { + if (!_pdftotext) { + return false; + } + + var item = Zotero.Items.get(itemID); + var linkMode = item.getAttachmentLinkMode(); + // If file is stored outside of Zotero, create a directory for the item + // in the storage directory and save the cache file there + if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE) { + var cacheFile = Zotero.Attachments.createDirectoryForItem(itemID); + } + else { + var cacheFile = file.parent; + } + cacheFile.append(CACHE_FILE); + + var proc = Components.classes["@mozilla.org/process/util;1"]. + createInstance(Components.interfaces.nsIProcess); + proc.init(_pdftotext); + var maxPages = Zotero.Prefs.get('fulltext.pdfMaxPages'); + Zotero.debug('Running pdftotext -nopgbrk -l ' + maxPages + + ' "' + file.path + '" "' + cacheFile.path + '"'); + var args = ['-nopgbrk', '-l', maxPages, file.path, cacheFile.path]; + Zotero.debug(args); + proc.run(true, args, args.length); + + if (cacheFile.exists()) { + return this.indexFile(cacheFile, 'text/plain', 'utf-8', itemID); + } + return false; } function indexItems(items){ + if (items.constructor.name != 'Array') { + items = [items]; + } + var items = Zotero.Items.get(items); var found = []; @@ -222,7 +349,7 @@ Zotero.Fulltext = new function(){ continue; } - indexFile(file, i.getAttachmentMimeType(), + this.indexFile(file, i.getAttachmentMimeType(), i.getAttachmentCharset(), i.getID()); } @@ -247,7 +374,8 @@ Zotero.Fulltext = new function(){ function findTextInFile(file, charset, searchText, mode){ Zotero.debug("Searching for text '" + searchText + "' in " + file.path); - var str = Zotero.File.getContents(file, charset); + var maxLength = Zotero.Prefs.get('fulltext.textMaxLength'); + var str = Zotero.File.getContents(file, charset, maxLength); // If not binary mode, convert HTML to text if (!mode || mode.indexOf('Binary')==-1){ @@ -255,7 +383,7 @@ Zotero.Fulltext = new function(){ str = str.replace(/(>)/g, '$1 '); // Parse to avoid searching on HTML - str = HTMLToText(str); + str = this.HTMLToText(str); } switch (mode){ @@ -332,17 +460,30 @@ Zotero.Fulltext = new function(){ continue; } - // If not binary mode, only scan plaintext files - if (!mode || mode.indexOf('Binary')==-1){ - if (!Zotero.MIME.isTextType(i.getAttachmentMimeType())) { - Zotero.debug('Not scanning MIME type ' + mimeType, 4); + var mimeType = i.getAttachmentMimeType(); + + if (isCachedMIMEType(mimeType)) { + var file = _getItemCacheFile(i.getID()); + if (!file.exists()) { continue; } + + mimeType = 'text/plain'; + var charset = 'utf-8'; + } + else { + // If not binary mode, only scan plaintext files + if (!mode || mode.indexOf('Binary') == -1) { + if (!Zotero.MIME.isTextType(mimeType)) { + Zotero.debug('Not scanning MIME type ' + mimeType, 4); + continue; + } + } + + var charset = i.getAttachmentCharset(); } - var charset = i.getAttachmentCharset(); - - var match = findTextInFile(file, charset, searchText, mode); + var match = this.findTextInFile(file, charset, searchText, mode); if (match != -1){ found.push({id:i.getID(), match:match}); @@ -355,6 +496,17 @@ Zotero.Fulltext = new function(){ function clearItemWords(itemID){ Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=" + itemID); + + // Delete fulltext cache file if there is one + var item = Zotero.Items.get(itemID); + switch (item.getAttachmentMimeType()) { + case 'application/pdf': + var cacheFile = _getItemCacheFile(); + if (cacheFile.exists()) { + cacheFile.remove(null); + } + break; + } } @@ -467,4 +619,12 @@ Zotero.Fulltext = new function(){ text = text.replace("zoteroapostrophe", "'"); return text; } + + + function _getItemCacheFile(itemID) { + var cacheFile = Zotero.getStorageDirectory(); + cacheFile.append(itemID); + cacheFile.append(CACHE_FILE); + return cacheFile; + } } diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index f53e9c67a7..26c37cbf20 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -98,7 +98,6 @@ var Zotero = new function(){ var gExtensionManager = Components.classes["@mozilla.org/extensions/manager;1"] .getService(Components.interfaces.nsIExtensionManager); - var itemType = nsIUpdateItem.TYPE_EXTENSION; this.version = gExtensionManager.getItemForID(ZOTERO_CONFIG['GUID']).version; @@ -131,6 +130,8 @@ var Zotero = new function(){ Zotero.DB.addCallback('commit', Zotero.Notifier.commit); Zotero.DB.addCallback('rollback', Zotero.Notifier.reset); + Zotero.Fulltext.init(); + // Trigger updating of schema and scrapers Zotero.Schema.updateSchema(); Zotero.Schema.updateScrapersRemote(); diff --git a/defaults/preferences/zotero.js b/defaults/preferences/zotero.js index f4b2fb18c7..f3efea89a1 100644 --- a/defaults/preferences/zotero.js +++ b/defaults/preferences/zotero.js @@ -16,7 +16,6 @@ pref("extensions.zotero.automaticTags",true); pref("extensions.zotero.fontSize", "1.0"); pref("extensions.zotero.recursiveCollections", false); - pref("extensions.zotero.lastCreatorFieldMode",0); pref("extensions.zotero.lastAbstractExpand",0); pref("extensions.zotero.lastRenameAssociatedFile", false); @@ -32,6 +31,10 @@ pref("extensions.zotero.keys.newNote", 'O'); pref("extensions.zotero.keys.toggleTagSelector", 'T'); pref("extensions.zotero.keys.copySelectedItemsToClipboard", 'C'); +// Fulltext indexing +pref("extensions.zotero.fulltext.textMaxLength", 500000); +pref("extensions.zotero.fulltext.pdfMaxPages", 100); + // Export and citation settings pref("extensions.zotero.export.lastTranslator", '14763d24-8ba0-45df-8f52-b8d1108e7ac9'); pref("extensions.zotero.export.translatorSettings", 'true,false');