From 2820add3d16d82c4214b63ead509a82c650e64e0 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Sun, 9 Oct 2022 14:01:50 +0200 Subject: [PATCH 1/5] Add bulk folder import #2252 --- .../zotero/components/progressQueueTable.jsx | 8 +- chrome/content/zotero/components/utils.js | 38 ++- chrome/content/zotero/fileInterface.js | 9 + chrome/content/zotero/import/folderImport.js | 241 ++++++++++++++++++ chrome/content/zotero/xpcom/data/relations.js | 3 +- package.json | 1 + resource/config.js | 7 +- scripts/config.js | 9 +- test/tests/folderImportTest.js | 136 ++++++++++ 9 files changed, 435 insertions(+), 17 deletions(-) create mode 100644 chrome/content/zotero/import/folderImport.js create mode 100644 test/tests/folderImportTest.js diff --git a/chrome/content/zotero/components/progressQueueTable.jsx b/chrome/content/zotero/components/progressQueueTable.jsx index 4b08ee4c44..c010c7639e 100644 --- a/chrome/content/zotero/components/progressQueueTable.jsx +++ b/chrome/content/zotero/components/progressQueueTable.jsx @@ -27,7 +27,7 @@ import PropTypes from 'prop-types'; import { getDOMElement } from 'components/icons'; import VirtualizedTable, { renderCell } from 'components/virtualized-table'; -import { noop } from './utils'; +import { nextHTMLID, noop } from './utils'; function getImageByStatus(status) { @@ -45,8 +45,9 @@ function getImageByStatus(status) { const ProgressQueueTable = ({ onActivate = noop, progressQueue }) => { const treeRef = useRef(null); + const htmlID = useRef(nextHTMLID()); - const getRowCount = useCallback(() => progressQueue.getRows().length, [progressQueue]); + const getRowCount = useCallback(() => progressQueue.getTotal(), [progressQueue]); const rowToTreeItem = useCallback((index, selection, oldDiv = null, columns) => { let rows = progressQueue.getRows(); @@ -92,6 +93,7 @@ const ProgressQueueTable = ({ onActivate = noop, progressQueue }) => { progressQueue.addListener('rowadded', refreshTree); progressQueue.addListener('rowupdated', refreshTree); progressQueue.addListener('rowdeleted', refreshTree); + return () => { progressQueue.removeListener('rowadded', refreshTree); progressQueue.removeListener('rowupdated', refreshTree); @@ -103,7 +105,7 @@ const ProgressQueueTable = ({ onActivate = noop, progressQueue }) => { . ***** END LICENSE BLOCK ***** */ -'use strict'; - const noop = () => {}; + function getDragTargetOrient(event, target) { const elem = target || event.target; const {y, height} = elem.getBoundingClientRect(); @@ -72,9 +71,30 @@ function createDragHandler({ handleDrag, handleDragStop }) { return { start: onDragStart, stop: onDragStop - } + }; } -export { - noop, getDragTargetOrient, createDragHandler +var _htmlID = 1; + +const nextHTMLID = (prefix = 'id-') => prefix + _htmlID++; + +const scrollIntoViewIfNeeded = (element, container, opts = {}) => { + const containerTop = container.scrollTop; + const containerBottom = containerTop + container.clientHeight; + const elementTop = element.offsetTop; + const elementBottom = elementTop + element.clientHeight; + + if (elementTop < containerTop || elementBottom > containerBottom) { + const before = container.scrollTop; + element.scrollIntoView(opts); + const after = container.scrollTop; + return after - before; + } + return 0; +}; + +const stopPropagation = ev => ev.stopPropagation(); + +export { + nextHTMLID, noop, getDragTargetOrient, createDragHandler, scrollIntoViewIfNeeded, stopPropagation }; diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 86b9ca2a9b..f78306aa86 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -448,6 +448,15 @@ var Zotero_File_Interface = new function() { translation.createNewCollection = createNewCollection; translation.mendeleyCode = options.mendeleyCode; } + else if (options.folder) { + Components.utils.import("chrome://zotero/content/import/folderImport.js"); + translation = new Zotero_Import_Folder({ + folder: options.folder, + recreateStructure: options.recreateStructure, + fileTypes: options.fileTypes, + mimeTypes: options.mimeTypes, + }); + } else { // Check if the file is an SQLite database var sample = yield Zotero.File.getSample(file.path); diff --git a/chrome/content/zotero/import/folderImport.js b/chrome/content/zotero/import/folderImport.js new file mode 100644 index 0000000000..e3bfc0f693 --- /dev/null +++ b/chrome/content/zotero/import/folderImport.js @@ -0,0 +1,241 @@ +var EXPORTED_SYMBOLS = ["Zotero_Import_Folder"]; // eslint-disable-line no-unused-vars + +Components.utils.import("resource://gre/modules/Services.jsm"); +Services.scriptloader.loadSubScript("chrome://zotero/content/include.js"); +const multimatch = require('multimatch'); + +const collectFilesRecursive = async (dirPath, parents = [], files = []) => { + await Zotero.File.iterateDirectory(dirPath, async ({ isDir, _isSymlink, name, path }) => { + if (isDir) { + await collectFilesRecursive(path, [...parents, name], files); + } + // TODO: Also check for hidden file attribute on windows? + else if (!name.startsWith('.')) { + files.push({ parents, path, name }); + } + }); + return files; +}; + +const findCollection = (libraryID, parentCollectionID, collectionName) => { + const collections = parentCollectionID + ? Zotero.Collections.getByParent(parentCollectionID) + : Zotero.Collections.getByLibrary(libraryID); + + return collections.find(c => c.name === collectionName); +}; + +const findItemByHash = async (libraryID, hash) => { + let items = (await Zotero.Relations.getByPredicateAndObject('item', 'zotero:attachmentHash', hash)) + .filter(item => item.libraryID == libraryID && !item.deleted && item.isTopLevelItem()); + + if (!items.length) { + items = (await Zotero.Relations.getByPredicateAndObject('item', 'zotero:fileHash', hash)) + .filter(item => item.libraryID == libraryID && !item.deleted && item.isTopLevelItem()); + } + + if (!items.length) { + return null; + } + + return items[0]; +}; + +class Zotero_Import_Folder { // eslint-disable-line camelcase,no-unused-vars + constructor({ mimeTypes = ['application/pdf'], fileTypes, folder, libraryID, recreateStructure }) { + this.folder = folder; + this.libraryID = libraryID; + this.newItems = []; + this.recreateStructure = recreateStructure; + this.fileTypes = fileTypes && fileTypes.length ? fileTypes.split(',').map(ft => ft.trim()) : []; + this._progress = 0; + this._progressMax = 0; + this._itemDone = () => {}; + this.types = mimeTypes; // whitelist of mime types to process + } + + setLocation(folder) { + this.folder = folder; + } + + setHandler(name, handler) { + switch (name) { + case 'itemDone': + this._itemDone = handler; + break; + } + } + + setTranslator() {} + + getProgress() { + return this._progress / this._progressMax * 100; + } + + async getTranslators() { + return [{ label: 'Folder import' }]; + } + + async translate({ collections = [], linkFiles = false } = {}) { + const libraryID = this.libraryID || Zotero.Libraries.userLibraryID; + const files = await collectFilesRecursive(this.folder); + + // import is done in four phases: sniff for mime type, calculate md5, import as attachment, recognize. + // hence number of files is multiplied by 4 to determine max progress + this._progressMax = files.length * 4; + + const mimeTypes = await Promise.all(files.map( + async ({ path }) => { + const mimeType = Zotero.MIME.sniffForMIMEType(await Zotero.File.getSample(path)); + this._progress++; + this._itemDone(); + return mimeType; + } + )); + + const fileHashes = await Promise.all(files.map( + async ({ name, path }, index) => { + const contentType = mimeTypes[index]; + this._progress++; + if (!(this.types.includes(contentType) || multimatch(name, this.fileTypes, { nocase: true }).length > 0)) { + // don't bother calculating a hash for file that will be ignored + return null; + } + const md5Hash = await Zotero.Utilities.Internal.md5Async(path); + this._itemDone(); + return md5Hash; + } + )); + + files.forEach((fileData, index) => { + fileData.parentCollectionIDs = (collections && collections.length) ? [...collections] : []; + fileData.mimeType = mimeTypes[index]; + }); + + if (this.recreateStructure) { + for (const fileData of files) { + const { parents } = fileData; + let prevParentCollectionID = null; + if (parents.length) { + prevParentCollectionID = (collections && collections.length) ? collections[0] : null; + for (const parentName of parents) { + const parentCollection = findCollection(libraryID, prevParentCollectionID, parentName) || new Zotero.Collection; + parentCollection.libraryID = libraryID; + parentCollection.name = parentName; + if (prevParentCollectionID) { + parentCollection.parentID = prevParentCollectionID; + } + await parentCollection.saveTx({ skipSelect: true }); //eslint-disable-line no-await-in-loop + prevParentCollectionID = parentCollection.id; + } + } + if (prevParentCollectionID) { + fileData.parentCollectionIDs = [prevParentCollectionID]; + } + } + } + + // index files by hash to avoid importing duplicate files. Keep track of where duplicates were found so that + // duplicate item is still added to one collection per folder + const fileDataByHash = {}; + files.forEach((fileData, index) => { + const hash = fileHashes[index]; + if (hash in fileDataByHash) { + fileDataByHash[hash].parentCollectionIDs.push(...fileData.parentCollectionIDs); + } + else { + fileDataByHash[hash] = fileData; + } + }); + + // advance progress to account for duplicates found within file structure + // these files won't be imported nor recognized so advance 2 ticks per file + this._progress += 2 * (files.length - Object.keys(fileDataByHash).length); + this._itemDone(); + + const attachmentItemHashLookup = {}; + const attachmentItems = await Promise.all(Object.entries(fileDataByHash).map( + async ([hash, { name, path, parentCollectionIDs, mimeType }]) => { + const options = { + collections: parentCollectionIDs, + contentType: mimeType, + file: path, + libraryID, + }; + + let attachmentItem = null; + + if ((this.types.includes(mimeType) || multimatch(name, this.fileTypes, { nocase: true }).length > 0)) { + const existingItem = await findItemByHash(libraryID, hash); + + if (existingItem) { + existingItem.setCollections([...existingItem.getCollections(), ...parentCollectionIDs]); + existingItem.saveTx({ skipSelect: true }); + } + else { + if (linkFiles) { + attachmentItem = await Zotero.Attachments.linkFromFile(options); + } + else { + attachmentItem = await Zotero.Attachments.importFromFile(options); + } + + this.newItems.push(attachmentItem); + attachmentItemHashLookup[attachmentItem.id] = hash; + } + } + + if (attachmentItem && !Zotero.RecognizePDF.canRecognize(attachmentItem)) { + attachmentItem.setRelations({ 'zotero:fileHash': hash }); + await attachmentItem.saveTx({ skipSelect: true }); + attachmentItem = null; + } + this._progress++; + this._itemDone(); + return attachmentItem; + } + )); + + + // discard unrecognizable items, increase progress for discarded items + const recognizableItems = attachmentItems.filter(item => item !== null); + this._progress += attachmentItems.length - recognizableItems.length; + this._itemDone(); + + const recognizeQueue = Zotero.ProgressQueues.get('recognize'); + const itemsToSavePostRecognize = []; + + const processRecognizedItem = ({ status, id }) => { + const updatedItem = recognizableItems.find(i => i.id === id); + if (status === Zotero.ProgressQueue.ROW_SUCCEEDED) { + const recognizedItem = updatedItem.parentItem; + if (recognizedItem && id in attachmentItemHashLookup) { + recognizedItem.setRelations({ 'zotero:attachmentHash': attachmentItemHashLookup[id] }); + itemsToSavePostRecognize.push(recognizedItem); + } + } + if (status === Zotero.ProgressQueue.ROW_FAILED) { + if (updatedItem && id in attachmentItemHashLookup) { + updatedItem.setRelations({ 'zotero:fileHash': attachmentItemHashLookup[id] }); + itemsToSavePostRecognize.push(updatedItem); + } + } + if ([Zotero.ProgressQueue.ROW_FAILED, Zotero.ProgressQueue.ROW_SUCCEEDED].includes(status)) { + this._progress++; + this._itemDone(); + } + }; + + recognizeQueue.addListener('rowupdated', processRecognizedItem); + try { + await Zotero.RecognizePDF.recognizeItems(recognizableItems); + } + finally { + recognizeQueue.removeListener('rowupdated', processRecognizedItem); + } + + await Zotero.Promise.all( + itemsToSavePostRecognize.map(async item => item.saveTx({ skipSelect: true })) + ); + } +} diff --git a/chrome/content/zotero/xpcom/data/relations.js b/chrome/content/zotero/xpcom/data/relations.js index c830091fb1..ab14cd38fe 100644 --- a/chrome/content/zotero/xpcom/data/relations.js +++ b/chrome/content/zotero/xpcom/data/relations.js @@ -33,7 +33,8 @@ Zotero.Relations = new function () { this._namespaces = { dc: 'http://purl.org/dc/elements/1.1/', owl: 'http://www.w3.org/2002/07/owl#', - mendeleyDB: 'http://zotero.org/namespaces/mendeleyDB#' + mendeleyDB: 'http://zotero.org/namespaces/mendeleyDB#', + zotero: 'http://zotero.org/namespaces/zotero' }; var _types = ['collection', 'item']; diff --git a/package.json b/package.json index 23e96cae40..61b3f32b5c 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "colors": "^1.4.0", "eslint": "^8.5.0", "eslint-plugin-react": "^7.28.0", + "eslint-plugin-react-hooks": "^4.0.4", "fs-extra": "^3.0.1", "globby": "^6.1.0", "jspath": "^0.4.0", diff --git a/resource/config.js b/resource/config.js index 452fe1870b..67c3a5760f 100644 --- a/resource/config.js +++ b/resource/config.js @@ -33,8 +33,9 @@ var ZOTERO_CONFIG = { PLUGINS_URL: 'https://www.zotero.org/support/plugins', }; -if (typeof process === 'object' && process + '' === '[object process]'){ +if (typeof exports === 'object' && typeof module !== 'undefined') { module.exports = ZOTERO_CONFIG; -} else { +} +else { var EXPORTED_SYMBOLS = ["ZOTERO_CONFIG"]; -} \ No newline at end of file +} diff --git a/scripts/config.js b/scripts/config.js index d09363f581..dab51d0a0d 100644 --- a/scripts/config.js +++ b/scripts/config.js @@ -97,7 +97,14 @@ const browserifyConfigs = [ config: { standalone: 'chaiAsPromised' } - } + }, + { + src: 'node_modules/multimatch/index.js', + dest: 'resource/multimatch.js', + config: { + standalone: 'multimatch' + } + }, ]; // exclude mask used for js, copy, symlink and sass tasks diff --git a/test/tests/folderImportTest.js b/test/tests/folderImportTest.js new file mode 100644 index 0000000000..d6609876ec --- /dev/null +++ b/test/tests/folderImportTest.js @@ -0,0 +1,136 @@ +/* global Zotero_Import_Folder: false */ + +describe('Zotero_Import_Folder', function () { + var tmpDir; + const uc = (name) => 'Zotero_Import_Folder_' + name; + + before(async () => { + tmpDir = await getTempDirectory(); + + await OS.File.makeDir(OS.Path.join(tmpDir, uc('dir1'))); + await OS.File.makeDir(OS.Path.join(tmpDir, uc('dir1'), uc('subdir1'))); + await OS.File.makeDir(OS.Path.join(tmpDir, uc('dir2'))); + + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'recognizePDF_test_title.pdf'), + OS.Path.join(tmpDir, 'recognizePDF_test_title.pdf') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'recognizePDF_test_title.pdf'), + OS.Path.join(tmpDir, uc('dir1'), 'recognizePDF_test_title.pdf') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'recognizePDF_test_arXiv.pdf'), + OS.Path.join(tmpDir, uc('dir1'), uc('subdir1'), 'recognizePDF_test_arXiv.pdf') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'recognizePDF_test_title.pdf'), + OS.Path.join(tmpDir, uc('dir2'), 'recognizePDF_test_title.pdf') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'test.png'), + OS.Path.join(tmpDir, uc('dir2'), 'test.png') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'test.html'), + OS.Path.join(tmpDir, uc('dir2'), 'test.html') + ); + await OS.File.copy( + OS.Path.join(getTestDataDirectory().path, 'test.txt'), + OS.Path.join(tmpDir, uc('dir2'), 'test.txt') + ); + + Components.utils.import('chrome://zotero/content/import/folderImport.js'); + }); + + describe('#import', () => { + it('should import PDFs from a folder and recreate structure without creating duplicates', async function () { + this.timeout(30000); + if (Zotero.automatedTest) { + this.skip(); + } + + const importer = new Zotero_Import_Folder({ + folder: tmpDir, + recreateStructure: true, + }); + + await importer.translate({ + libraryID: Zotero.Libraries.userLibraryID, + linkFiles: true, + }); + + assert.equal(importer.newItems.length, 2); + + const firstPDFAttachment = importer.newItems.find(ni => ni.getField('title') === 'recognizePDF_test_arXiv.pdf'); + const firstPDFItem = await Zotero.Items.getAsync(firstPDFAttachment.parentID); + const firstPDFCollections = await Zotero.Collections.getAsync(firstPDFItem.getCollections()); + assert.equal(firstPDFItem.getField('title'), 'Scaling study of an improved fermion action on quenched lattices'); + assert.equal(firstPDFCollections.length, 1); + assert.equal(firstPDFCollections[0].name, uc('subdir1')); + assert.equal((await Zotero.Collections.getAsync(firstPDFCollections[0].parentID)).name, uc('dir1')); + + const secondPDFAttachment = importer.newItems.find(ni => ni.getField('title') === 'recognizePDF_test_title.pdf'); + const secondPDFItem = await Zotero.Items.getAsync(secondPDFAttachment.parentID); + const secondPDFCollections = await Zotero.Collections.getAsync(secondPDFItem.getCollections()); + assert.equal(secondPDFItem.getField('title'), 'Bitcoin: A Peer-to-Peer Electronic Cash System'); + assert.equal(secondPDFCollections.length, 2); + assert.sameMembers(secondPDFCollections.map(c => c.name), [uc('dir1'), uc('dir2')]); + + assert.sameMembers( + Zotero.Collections.getByLibrary(Zotero.Libraries.userLibraryID, true) + .map(c => c.name) + .filter(c => c.startsWith('Zotero_Import_Folder')), + [uc('dir1'), uc('dir2'), uc('subdir1')] + ); + + const importer2 = new Zotero_Import_Folder({ + folder: tmpDir, + recreateStructure: true, + }); + + await importer2.translate({ + libraryID: Zotero.Libraries.userLibraryID, + linkFiles: true, + }); + + assert.lengthOf(importer2.newItems, 0); + assert.sameMembers( + Zotero.Collections.getByLibrary(Zotero.Libraries.userLibraryID, true) + .map(c => c.name) + .filter(c => c.startsWith('Zotero_Import_Folder')), + [uc('dir1'), uc('dir2'), uc('subdir1')] + ); + }); + + it('should only import specified file types from a folder', async function () { + this.timeout(30000); + if (Zotero.automatedTest) { + this.skip(); + } + const importer = new Zotero_Import_Folder({ + folder: tmpDir, + recreateStructure: false, + fileTypes: '*.png,*.txt', + mimeTypes: [] + }); + + await importer.translate({ + libraryID: Zotero.Libraries.userLibraryID, + linkFiles: true, + }); + + assert.equal(importer.newItems.length, 2); + const pngItem = importer.newItems.find(ni => ni.getField('title') === 'test.png'); + assert.isDefined(pngItem); + assert.isFalse(pngItem.parentID); + + const txtItem = importer.newItems.find(ni => ni.getField('title') === 'test.txt'); + assert.isDefined(txtItem); + assert.isFalse(txtItem.parentID); + + const htmlItem = importer.newItems.find(ni => ni.getField('title') === 'test.html'); + assert.isUndefined(htmlItem); + }); + }); +}); From 5ea43bd65cf896a0f300d08b173ca978cdf6333d Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Mon, 15 Aug 2022 12:04:52 +0200 Subject: [PATCH 2/5] fx-compat: Convert import wizard to use CE --- chrome/content/zotero/fileInterface.js | 67 +- chrome/content/zotero/import/folderImport.js | 31 +- chrome/content/zotero/import/importWizard.js | 599 +++++++++--------- .../content/zotero/import/importWizard.xhtml | 92 +++ chrome/content/zotero/import/importWizard.xul | 98 --- .../zotero/import/mendeley/authViewer.js | 47 ++ .../zotero/standalone/basicViewer.xhtml | 16 +- chrome/locale/en-US/zotero/zotero.dtd | 2 + chrome/locale/en-US/zotero/zotero.ftl | 64 ++ chrome/locale/en-US/zotero/zotero.properties | 8 - chrome/skin/default/zotero/importWizard.css | 82 --- scss/_zotero-react-client.scss | 1 + scss/components/_import-wizard.scss | 142 +++++ test/tests/folderImportTest.js | 4 +- 14 files changed, 683 insertions(+), 570 deletions(-) create mode 100644 chrome/content/zotero/import/importWizard.xhtml delete mode 100644 chrome/content/zotero/import/importWizard.xul create mode 100644 chrome/content/zotero/import/mendeley/authViewer.js create mode 100644 chrome/locale/en-US/zotero/zotero.ftl delete mode 100644 chrome/skin/default/zotero/importWizard.css create mode 100644 scss/components/_import-wizard.scss diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index f78306aa86..3db3587504 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -389,7 +389,7 @@ var Zotero_File_Interface = new function() { }; args.wrappedJSObject = args; - Services.ww.openWindow(null, "chrome://zotero/content/import/importWizard.xul", + Services.ww.openWindow(null, "chrome://zotero/content/import/importWizard.xhtml", "importFile", "chrome,dialog=yes,centerscreen,width=600,height=400,modal", args); }; @@ -985,71 +985,6 @@ var Zotero_File_Interface = new function() { } } - this.authenticateMendeleyOnlinePoll = function (win) { - if (win && win[0] && win[0].location) { - const matchResult = win[0].location.toString().match(/mendeley_oauth_redirect.html(?:.*?)(?:\?|&)code=(.*?)(?:&|$)/i); - if (matchResult) { - const mendeleyCode = matchResult[1]; - Zotero.getMainWindow().setTimeout(() => this.showImportWizard({ mendeleyCode }), 0); - - // Clear all cookies to remove access - // - // This includes unrelated cookies in the central cookie store, but that's fine for - // the moment, since we're not purposely using cookies for anything else. - // - // TODO: Switch to removeAllSince() once >Fx60 - try { - Cc["@mozilla.org/cookiemanager;1"] - .getService(Ci.nsICookieManager) - .removeAll(); - } - catch (e) { - Zotero.logError(e); - } - - win.close(); - return; - } - } - - if (win && !win.closed) { - Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); - } - }; - - this.authenticateMendeleyOnline = function () { - const uri = `https://api.mendeley.com/oauth/authorize?client_id=5907&redirect_uri=https%3A%2F%2Fzotero-static.s3.amazonaws.com%2Fmendeley_oauth_redirect.html&response_type=code&state=&scope=all`; - var win = Services.wm.getMostRecentWindow("zotero:basicViewer"); - if (win) { - win.loadURI(uri); - } - else { - const ww = Services.ww; - const arg = Components.classes["@mozilla.org/supports-string;1"] - .createInstance(Components.interfaces.nsISupportsString); - arg.data = uri; - win = ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xhtml", - "basicViewer", "chrome,dialog=yes,resizable,centerscreen,menubar,scrollbars", arg); - } - - let browser; - let func = function () { - win.removeEventListener("load", func); - browser = win.document.documentElement.getElementsByTagName('browser')[0]; - browser.addEventListener("pageshow", innerFunc); - }; - let innerFunc = function () { - browser.removeEventListener("pageshow", innerFunc); - win.outerWidth = Math.max(640, Math.min(1000, win.screen.availHeight)); - win.outerHeight = Math.max(480, Math.min(800, win.screen.availWidth)); - }; - - win.addEventListener("load", func); - - // polling executed by the main window because current (wizard) window will be closed - Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); - }; - /** * Generate an error string reporting a translation failure. Includes the * label of the running translator if available. diff --git a/chrome/content/zotero/import/folderImport.js b/chrome/content/zotero/import/folderImport.js index e3bfc0f693..6bc38ba093 100644 --- a/chrome/content/zotero/import/folderImport.js +++ b/chrome/content/zotero/import/folderImport.js @@ -2,7 +2,22 @@ var EXPORTED_SYMBOLS = ["Zotero_Import_Folder"]; // eslint-disable-line no-unuse Components.utils.import("resource://gre/modules/Services.jsm"); Services.scriptloader.loadSubScript("chrome://zotero/content/include.js"); -const multimatch = require('multimatch'); + +// matches "*" and "?" wildcards of a glob pattern, case-insensitive +function simpleGlobMatch(filename, patterns) { + for (const pattern of patterns) { + // Convert glob pattern to regex pattern + const regexPattern = pattern + .replace(/[.+^${}()|[\]\\]/g, '\\$&') // Escape special regex characters + .replace(/\*/g, '.*') // Replace * with regex equivalent + .replace(/\?/g, '.'); // Replace ? with regex equivalent + + if (new RegExp(`^${regexPattern}$`, 'i').test(filename)) { + return true; + } + } + return false; +} const collectFilesRecursive = async (dirPath, parents = [], files = []) => { await Zotero.File.iterateDirectory(dirPath, async ({ isDir, _isSymlink, name, path }) => { @@ -97,7 +112,7 @@ class Zotero_Import_Folder { // eslint-disable-line camelcase,no-unused-vars async ({ name, path }, index) => { const contentType = mimeTypes[index]; this._progress++; - if (!(this.types.includes(contentType) || multimatch(name, this.fileTypes, { nocase: true }).length > 0)) { + if (!(this.types.includes(contentType) || simpleGlobMatch(name, this.fileTypes))) { // don't bother calculating a hash for file that will be ignored return null; } @@ -165,12 +180,12 @@ class Zotero_Import_Folder { // eslint-disable-line camelcase,no-unused-vars let attachmentItem = null; - if ((this.types.includes(mimeType) || multimatch(name, this.fileTypes, { nocase: true }).length > 0)) { + if ((this.types.includes(mimeType) || simpleGlobMatch(name, this.fileTypes))) { const existingItem = await findItemByHash(libraryID, hash); if (existingItem) { existingItem.setCollections([...existingItem.getCollections(), ...parentCollectionIDs]); - existingItem.saveTx({ skipSelect: true }); + await existingItem.saveTx({ skipSelect: true }); } else { if (linkFiles) { @@ -233,9 +248,9 @@ class Zotero_Import_Folder { // eslint-disable-line camelcase,no-unused-vars finally { recognizeQueue.removeListener('rowupdated', processRecognizedItem); } - - await Zotero.Promise.all( - itemsToSavePostRecognize.map(async item => item.saveTx({ skipSelect: true })) - ); + + for (const item of itemsToSavePostRecognize) { + await item.saveTx({ skipSelect: true }); + } } } diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 48b1c83cb2..28ea2181d5 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -1,211 +1,249 @@ -import FilePicker from 'zotero/modules/filePicker'; +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2022 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ -var Zotero_Import_Wizard = { - _wizard: null, - _dbs: null, - _file: null, - _translation: null, - _mendeleyOnlineRedirectURLWithCode: null, - _mendeleyCode: null, - - - init: async function () { - this._wizard = document.getElementById('import-wizard'); - var dbs = await Zotero_File_Interface.findMendeleyDatabases(); - if (dbs.length) { - // Local import disabled - //document.getElementById('radio-import-source-mendeley').hidden = false; +import FilePicker from 'zotero/filePicker'; +import React from 'react'; +import ReactDOM from 'react-dom'; +import ProgressQueueTable from 'components/progressQueueTable'; + +/* eslint camelcase: ["error", {allow: ["Zotero_File_Interface", "Zotero_Import_Wizard"]} ] */ +/* global Zotero_File_Interface: false */ + + +const Zotero_Import_Wizard = { // eslint-disable-line no-unused-vars + wizard: null, + folder: null, + file: null, + mendeleyCode: null, + libraryID: null, + translation: null, + + async getShouldCreateCollection() { + const sql = "SELECT ROWID FROM collections WHERE libraryID=?1 " + + "UNION " + + "SELECT ROWID FROM items WHERE libraryID=?1 " + // Not in trash + + "AND itemID NOT IN (SELECT itemID FROM deletedItems) " + // And not a child item (which doesn't necessarily show up in the trash) + + "AND itemID NOT IN (SELECT itemID FROM itemNotes WHERE parentItemID IS NOT NULL) " + + "AND itemID NOT IN (SELECT itemID FROM itemAttachments WHERE parentItemID IS NOT NULL) " + + "LIMIT 1"; + return Zotero.DB.valueQueryAsync(sql, this.libraryID); + }, + + async init() { + const { mendeleyCode, libraryID } = window.arguments[0].wrappedJSObject ?? {}; + + this.libraryID = libraryID; + this.mendeleyCode = mendeleyCode; + + this.wizard = document.getElementById('import-wizard'); + this.wizard.getPageById('page-start') + .addEventListener('pageadvanced', this.onImportSourceAdvance.bind(this)); + this.wizard.getPageById('page-mendeley-online-intro') + .addEventListener('pagerewound', this.goToStart.bind(this)); + this.wizard.getPageById('page-mendeley-online-intro') + .addEventListener('pageadvanced', this.openMendeleyAuthWindow.bind(this)); + this.wizard.getPageById('page-options') + .addEventListener('pageshow', this.onOptionsPageShow.bind(this)); + this.wizard.getPageById('page-options') + .addEventListener('pageadvanced', this.startImport.bind(this)); + this.wizard.getPageById('page-progress') + .addEventListener('pageshow', this.onProgressPageShow.bind(this)); + + document + .getElementById('other-files') + .addEventListener('keyup', (ev) => { + document.getElementById('import-other').checked = ev.currentTarget.value.length > 0; + }); + document + .querySelector('#page-done-error-mendeley > a') + .addEventListener('click', this.onURLInteract.bind(this)); + document + .querySelector('#page-done-error-mendeley > a') + .addEventListener('keydown', this.onURLInteract.bind(this)); + document + .querySelector('#page-done-error > button') + .addEventListener('click', this.onReportErrorInteract.bind(this)); + document + .querySelector('#page-done-error > button') + .addEventListener('keydown', this.onReportErrorInteract.bind(this)); + + this.wizard.addEventListener('pageshow', this.updateFocus.bind(this)); + this.wizard.addEventListener('wizardcancel', this.onCancel.bind(this)); + + const shouldCreateCollection = await this.getShouldCreateCollection(); + document.getElementById('create-collection').checked = shouldCreateCollection; + + // wizard.shadowRoot content isn't exposed to our css + this.wizard.shadowRoot + .querySelector('.wizard-header-label').style.fontSize = '16px'; + + if (mendeleyCode) { + this.wizard.goTo('page-options'); } - - // If no existing collections or non-trash items in the library, don't create a new - // collection by default - var args = window.arguments[0].wrappedJSObject; - if (args && args.libraryID) { - let sql = "SELECT ROWID FROM collections WHERE libraryID=?1 " - + "UNION " - + "SELECT ROWID FROM items WHERE libraryID=?1 " - // Not in trash - + "AND itemID NOT IN (SELECT itemID FROM deletedItems) " - // And not a child item (which doesn't necessarily show up in the trash) - + "AND itemID NOT IN (SELECT itemID FROM itemNotes WHERE parentItemID IS NOT NULL) " - + "AND itemID NOT IN (SELECT itemID FROM itemAttachments WHERE parentItemID IS NOT NULL) " - + "LIMIT 1"; - if (!await Zotero.DB.valueQueryAsync(sql, args.libraryID)) { - document.getElementById('create-collection-checkbox').removeAttribute('checked'); + }, + + skipToDonePage(label, description, showReportErrorButton = false, isMendeleyError = false) { + this.wizard.getPageById('page-done').dataset.headerLabelId = label; + + if (!isMendeleyError) { + if (Array.isArray(description)) { + document.getElementById('page-done-description').dataset.l10nId = description[0]; + document.getElementById('page-done-description').dataset.l10nArgs = JSON.stringify(description[1]); + } + else { + document.getElementById('page-done-description').dataset.l10nId = description; } } - if (args && args.mendeleyCode) { - this._mendeleyCode = args.mendeleyCode; - this._wizard.goTo('page-options'); - } - - // Update labels - document.getElementById('radio-import-source-mendeley-online').label - = `Mendeley Reference Manager (${Zotero.getString('import.onlineImport')})`; - document.getElementById('radio-import-source-mendeley').label - = `Mendeley Desktop (${Zotero.getString('import.localImport')})`; - document.getElementById('file-handling-store').label = Zotero.getString( - 'import.fileHandling.store', - Zotero.appName - ); - document.getElementById('file-handling-link').label = Zotero.getString('import.fileHandling.link'); - document.getElementById('file-handling-description').textContent = Zotero.getString( - 'import.fileHandling.description', - Zotero.appName - ); - - Zotero.Translators.init(); // async - }, + document.getElementById('page-done-error-mendeley').style.display = isMendeleyError ? 'block' : 'none'; + document.getElementById('page-done-error').style.display = showReportErrorButton ? 'block' : 'none'; - onCancel: function () { - if (this._translation && this._translation.interrupt) { - this._translation.interrupt(); - } - }, - - onModeChosen: async function () { - var wizard = this._wizard; + const doneQueueContainer = document.getElementById('done-queue-container'); + const doneQueue = document.getElementById('done-queue'); - var mode = document.getElementById('import-source').selectedItem.id; - try { - switch (mode) { - case 'radio-import-source-file': - await this.chooseFile(); - break; - - case 'radio-import-source-mendeley-online': - wizard.goTo('mendeley-online-explanation'); - wizard.canRewind = true; - break; - - case 'radio-import-source-mendeley': - this._dbs = await Zotero_File_Interface.findMendeleyDatabases(); - // This shouldn't happen, because we only show the wizard if there are databases - if (!this._dbs.length) { - throw new Error("No databases found"); - } - this._populateFileList(this._dbs); - document.getElementById('file-options-header').textContent - = Zotero.getString('fileInterface.chooseAppDatabaseToImport', 'Mendeley') - wizard.goTo('page-file-list'); - wizard.canRewind = true; - this._enableCancel(); - break; - - default: - throw new Error(`Unknown mode ${mode}`); - } - } - catch (e) { - this._onDone( - Zotero.getString('general.error'), - Zotero.getString('fileInterface.importError'), - true + if (this.folder && !showReportErrorButton) { + doneQueueContainer.style.display = 'flex'; + ReactDOM.render( + , + doneQueue ); - throw e; } + else { + doneQueueContainer.style.display = 'none'; + } + + this.wizard.goTo('page-done'); + this.wizard.canRewind = false; }, - onMendeleyOnlineShow: async function () { - document.getElementById('mendeley-online-description').textContent = Zotero.getString( - 'import.online.intro', [Zotero.appName, 'Mendeley Reference Manager', 'Mendeley'] - ); - document.getElementById('mendeley-online-description2').textContent = Zotero.getString( - 'import.online.intro2', [Zotero.appName, 'Mendeley'] - ); + goToStart() { + this.wizard.goTo('page-start'); + this.wizard.canAdvance = true; }, - onMendeleyOnlineAdvance: function () { - if (!this._mendeleyOnlineRedirectURLWithCode) { - Zotero_File_Interface.authenticateMendeleyOnline(); - window.close(); - } - }, - - goToStart: function () { - this._wizard.goTo('page-start'); - this._wizard.canAdvance = true; - return false; - }, - - - chooseFile: async function (translation) { - var translation = new Zotero.Translate.Import(); - var translators = await translation.getTranslators(); - var fp = new FilePicker(); + async chooseFile() { + const translation = new Zotero.Translate.Import(); + const translators = await translation.getTranslators(); + const fp = new FilePicker(); fp.init(window, Zotero.getString("fileInterface.import"), fp.modeOpen); - fp.appendFilters(fp.filterAll); - var collation = Zotero.getLocaleCollation(); - + // Add Mendeley DB, which isn't a translator - var mendeleyFilter = { + const mendeleyFilter = { label: "Mendeley Database", // TODO: Localize target: "*.sqlite" }; - var filters = [...translators]; + const filters = [...translators]; filters.push(mendeleyFilter); - + filters.sort((a, b) => collation.compareString(1, a.label, b.label)); for (let filter of filters) { fp.appendFilter(filter.label, "*." + filter.target); } - - var rv = await fp.show(); + + const rv = await fp.show(); if (rv !== fp.returnOK && rv !== fp.returnReplace) { - return false; + return; } - + Zotero.debug(`File is ${fp.file}`); - - this._file = fp.file; - this._wizard.canAdvance = true; - this._wizard.goTo('page-options'); + this.file = fp.file; + this.wizard.canAdvance = true; + this.wizard.goTo('page-options'); }, - - - /** - * When a file is clicked on in the file list - */ - onFileSelected: async function () { - var index = document.getElementById('file-list').selectedIndex; - if (index != -1) { - this._file = this._dbs[index].path; - this._wizard.canAdvance = true; + + async chooseFolder() { + const fp = new FilePicker(); + fp.init(window, Zotero.getString('attachmentBasePath.selectDir'), fp.modeGetFolder); + fp.appendFilters(fp.filterAll); + + const rv = await fp.show(); + if (rv !== fp.returnOK && rv !== fp.returnReplace) { + return; } - else { - this._file = null; - this._wizard.canAdvance = false; + + Zotero.debug(`Folder is ${fp.file}`); + + this.folder = fp.file; + this.wizard.canAdvance = true; + this.wizard.goTo('page-options'); + }, + + async onImportSourceAdvance(ev) { + const selectedMode = document.getElementById('import-source-group').selectedItem.value; + ev.preventDefault(); + ev.stopPropagation(); + try { + switch (selectedMode) { + case 'file': + this.folder = null; + await this.chooseFile(); + break; + case 'folder': + this.file = null; + await this.chooseFolder(); + break; + case 'mendeleyOnline': + this.file = null; + this.folder = null; + this.wizard.goTo('page-mendeley-online-intro'); + this.wizard.canRewind = true; + break; + default: + throw new Error(`Unknown mode ${selectedMode}`); + } + } + catch (e) { + this.skipToDonePage('general-error', 'file-interface-import-error', true); + throw e; } }, - - - /** - * When the user clicks "Other…" to choose a file not in the list - */ - chooseMendeleyDB: async function () { - document.getElementById('file-list').selectedIndex = -1; - var fp = new FilePicker(); - fp.init(window, Zotero.getString('fileInterface.import'), fp.modeOpen); - fp.appendFilter("Mendeley Database", "*.sqlite"); // TODO: Localize - var rv = await fp.show(); - if (rv != fp.returnOK) { - return false; - } - this._file = fp.file; - this._wizard.canAdvance = true; - this._wizard.advance(); + + onOptionsPageShow() { + document.getElementById('page-options-folder-import').style.display = this.folder ? 'block' : 'none'; + document.getElementById('page-options-file-handling').style.display = this.mendeleyCode ? 'none' : 'block'; + this.wizard.canRewind = false; }, - - - onOptionsShown: function () { - document.getElementById('file-handling-options').hidden = !!this._mendeleyCode; + + openMendeleyAuthWindow(ev) { + ev.preventDefault(); + + const arg = Components.classes["@mozilla.org/supports-string;1"] + .createInstance(Components.interfaces.nsISupportsString); + arg.data = 'mendeleyImport'; + + window.close(); + + Services.ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xhtml", + "basicViewer", "chrome,dialog=yes,centerscreen,width=1000,height=700,modal", arg); }, - - - onBeforeImport: async function (translation) { + + async onBeforeImport(translation) { // Unrecognized translator if (!translation) { // Allow error dialog to be displayed, and then close window @@ -214,159 +252,112 @@ var Zotero_Import_Wizard = { }); return; } - - this._translation = translation; - + + this.translation = translation; + // Switch to progress pane - this._wizard.goTo('page-progress'); - var pm = document.getElementById('import-progressmeter'); - - translation.setHandler('itemDone', function () { - pm.value = translation.getProgress(); + this.wizard.goTo('page-progress'); + translation.setHandler('itemDone', () => { + document.getElementById('import-progress').value = translation.getProgress(); }); }, - - - onImportStart: async function () { - if (!this._file && !this._mendeleyCode) { - let index = document.getElementById('file-list').selectedIndex; - this._file = this._dbs[index].path; + + async onProgressPageShow() { + this.wizard.canAdvance = false; + this.wizard.canRewind = false; + const progressQueueContainer = document.getElementById('progress-queue-container'); + const progressQueue = document.getElementById('progress-queue'); + if (this.folder) { + progressQueueContainer.style.display = 'flex'; + ReactDOM.render( + , + progressQueue + ); } - this._disableCancel(); - this._wizard.canRewind = false; - this._wizard.canAdvance = false; + else { + progressQueueContainer.style.display = 'none'; + } + }, + + onURLInteract(ev) { + if (ev.type === 'click' || (ev.type === 'keydown' && ev.key === ' ')) { + Zotero.launchURL(ev.currentTarget.getAttribute('href')); + window.close(); + ev.preventDefault(); + } + }, + + onReportErrorInteract(ev) { + if (ev.type === 'click' || (ev.type === 'keydown' && ev.key === ' ')) { + Zotero.getActiveZoteroPane().reportErrors(); + window.close(); + } + }, + + onCancel() { + if (this.translation && this.translation.interrupt) { + this.translation.interrupt(); + } + }, + + updateFocus() { + (this.wizard.currentPage.querySelector('radiogroup:not([disabled]),checkbox:not([disabled])') ?? this.wizard.currentPage).focus(); + }, + + async startImport() { + this.wizard.canAdvance = false; + this.wizard.canRewind = false; + + const linkFiles = document.getElementById('file-handling').selectedItem.id === 'link'; + const recreateStructure = document.getElementById('recreate-structure').checked; + const shouldCreateCollection = document.getElementById('create-collection').checked; + const mimeTypes = document.getElementById('import-pdf').checked + ? ['application/pdf'] + : []; + const fileTypes = document.getElementById('import-other').checked + ? document.getElementById('other-files').value + : null; try { - let result = await Zotero_File_Interface.importFile({ - file: this._file, + const result = await Zotero_File_Interface.importFile({ + addToLibraryRoot: !shouldCreateCollection, + file: this.file, + fileTypes, + folder: this.folder, + linkFiles, + mendeleyCode: this.mendeleyCode, + mimeTypes, onBeforeImport: this.onBeforeImport.bind(this), - addToLibraryRoot: !document.getElementById('create-collection-checkbox') - .hasAttribute('checked'), - linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1, - mendeleyCode: this._mendeleyCode + recreateStructure }); - + // Cancelled by user or due to error if (!result) { window.close(); return; } - - let numItems = this._translation.newItems.length; - this._onDone( - Zotero.getString('fileInterface.importComplete'), - Zotero.getString(`fileInterface.itemsWereImported`, numItems, numItems) + + const numItems = this.translation.newItems.length; + this.skipToDonePage( + 'file-interface-import-complete', + ['file-interface-items-were-imported', { numItems }] ); } catch (e) { if (e.message == 'Encrypted Mendeley database') { - let url = 'https://www.zotero.org/support/kb/mendeley_import'; - let elem = document.createElement('div'); - elem.innerHTML = `The selected Mendeley database cannot be read, likely because it ` - + `is encrypted. See How do I import a ` - + `Mendeley library into Zotero? for more information.` - this._onDone(Zotero.getString('general.error'), elem); + this.skipToDonePage('general.error', [], false, true); } else { - this._onDone( - Zotero.getString('general.error'), - Zotero_File_Interface.makeImportErrorString(this._translation), + const translatorLabel = this.translation?.translator?.[0]?.label; + this.skipToDonePage( + 'general.error', + translatorLabel + ? ['file-interface-import-error-translator', { translator: translatorLabel }] + : 'file-interface-import-error', true ); } throw e; } }, - - - reportError: function () { - Zotero.getActiveZoteroPane().reportErrors(); - window.close(); - }, - - - _populateFileList: async function (files) { - var listbox = document.getElementById('file-list'); - - // Remove existing entries - var items = listbox.getElementsByTagName('listitem'); - for (let item of items) { - listbox.removeChild(item); - } - - for (let file of files) { - let li = document.createElement('listitem'); - - let name = document.createElement('listcell'); - // Simply filenames - let nameStr = file.name - .replace(/\.sqlite$/, '') - .replace(/@www\.mendeley\.com$/, ''); - if (nameStr == 'online') { - nameStr = Zotero.getString('dataDir.default', 'online.sqlite'); - } - name.setAttribute('label', nameStr + ' '); - li.appendChild(name); - - let lastModified = document.createElement('listcell'); - lastModified.setAttribute('label', file.lastModified.toLocaleString() + ' '); - li.appendChild(lastModified); - - let size = document.createElement('listcell'); - size.setAttribute( - 'label', - Zotero.getString('general.nMegabytes', (file.size / 1024 / 1024).toFixed(1)) + ' ' - ); - li.appendChild(size); - - listbox.appendChild(li); - } - - if (files.length == 1) { - listbox.selectedIndex = 0; - } - }, - - - _enableCancel: function () { - this._wizard.getButton('cancel').disabled = false; - }, - - - _disableCancel: function () { - this._wizard.getButton('cancel').disabled = true; - }, - - - _onDone: function (label, description, showReportErrorButton) { - var wizard = this._wizard; - wizard.getPageById('page-done').setAttribute('label', label); - - var xulElem = document.getElementById('result-description'); - var htmlElem = document.getElementById('result-description-html'); - - if (description instanceof HTMLElement) { - htmlElem.appendChild(description); - Zotero.Utilities.Internal.updateHTMLInXUL(htmlElem, { callback: () => window.close() }); - xulElem.hidden = true; - htmlElem.setAttribute('display', 'block'); - } - else { - xulElem.textContent = description; - xulElem.hidden = false; - htmlElem.setAttribute('display', 'none'); - } - document.getElementById('result-description'); - - if (showReportErrorButton) { - let button = document.getElementById('result-report-error'); - button.setAttribute('label', Zotero.getString('errorReport.reportError')); - button.hidden = false; - } - - // When done, move to last page and allow closing - wizard.canAdvance = true; - wizard.goTo('page-done'); - wizard.canRewind = false; - } }; diff --git a/chrome/content/zotero/import/importWizard.xhtml b/chrome/content/zotero/import/importWizard.xhtml new file mode 100644 index 0000000000..348ab78fda --- /dev/null +++ b/chrome/content/zotero/import/importWizard.xhtml @@ -0,0 +1,92 @@ + + + + + + + + + + + + + @@ -165,6 +170,13 @@ - + + + diff --git a/chrome/locale/en-US/zotero/zotero.dtd b/chrome/locale/en-US/zotero/zotero.dtd index e8e6b3de3e..abec546bbb 100644 --- a/chrome/locale/en-US/zotero/zotero.dtd +++ b/chrome/locale/en-US/zotero/zotero.dtd @@ -180,6 +180,8 @@ + + diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl new file mode 100644 index 0000000000..c725dfb585 --- /dev/null +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -0,0 +1,64 @@ +-app-name = Zotero + +import-wizard = + .title = "Import" + +import-where-from = Where do you want to import from? +import-online-intro-title = Introduction + +import-source-file = + .label = A file (BibTeX, RIS, Zotero RDF, etc.) + +import-source-folder = + .label = A folder of PDFs or other files + +import-source-online = + .label = { $targetApp } online import + +import-options = Options +import-importing = Importing… + +import-create-collection = + .label = Place imported collections and items into new collection + +import-recreate-structure = + .label = Recreate folder structure as collections + +import-fileTypes-header = File Types to Import: + +import-fileTypes-pdf = + .label = PDFs + +import-fileTypes-other = + .placeholder = Other files by pattern, comma-separated (e.g., *.jpg,*.png) + +import-file-handling = File Handling +import-file-handling-store = + .label = Copy files to the { -app-name } storage folder +import-file-handling-link = + .label = Link to files in original location +import-fileHandling-description = Linked files cannot be synced by { -app-name }. + +general-error = Error +file-interface-import-error = An error occurred while trying to import the selected file. Please ensure that the file is valid and try again. +file-interface-import-complete = Import Complete +file-interface-items-were-imported = { $numItems -> + [one] item was imported + *[other] { $numItems } items were imported + } + +import-mendeley-encrypted = The selected Mendeley database cannot be read, likely because it is encrypted. + See How do I import a Mendeley library into Zotero? for more information. + + +file-interface-import-error = = An error occurred while trying to import the selected file. Please ensure that the file is valid and try again. +file-interface-import-error-translator = An error occurred importing the selected file with “{ $translator }”. Please ensure that the file is valid and try again. + +# Variables: +# $targetAppOnline (String) +# $targetApp (String) +import-online-intro=In the next step you will be asked to log in to { $targetAppOnline } and grant { -app-name } access. This is necessary to import your { $targetApp } library into { -app-name }. +import-online-intro2={ -app-name } will never see or store your { $targetApp } password. + +report-error = + .label = Report Error… \ No newline at end of file diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index adb5b8ee7b..40bc7bc4ff 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -808,14 +808,6 @@ fileInterface.exportError = An error occurred while trying to export the selecte fileInterface.importOPML = Import Feeds from OPML fileInterface.OPMLFeedFilter = OPML Feed List -import.onlineImport = online import -import.localImport = local import -import.fileHandling.store = Copy files to the %S storage folder -import.fileHandling.link = Link to files in original location -import.fileHandling.description = Linked files cannot be synced by %S. -import.online.intro = In the next step you will be asked to log in to %2$S and grant %1$S access. This is necessary to import your %3$S library into %1$S. -import.online.intro2 = %1$S will never see or store your %2$S password. - quickCopy.copyAs = Copy as %S quickSearch.mode.titleCreatorYear = Title, Creator, Year diff --git a/chrome/skin/default/zotero/importWizard.css b/chrome/skin/default/zotero/importWizard.css deleted file mode 100644 index ec8d6d27a8..0000000000 --- a/chrome/skin/default/zotero/importWizard.css +++ /dev/null @@ -1,82 +0,0 @@ -.wizard-header-label { - font-size: 16px; - font-weight: bold; -} - -/* Start */ -wizard[currentpageid="page-start"] .wizard-header-label { - padding-top: 24px; -} - -wizard[currentpageid="page-start"] .wizard-page-box { - margin-top: -2px; - padding-top: 0; -} - -radiogroup { - font-size: 14px; - margin-top: 4px; -} - -radio { - padding-top: 5px; -} - -/* File list */ -wizard[currentpageid="page-file-list"] .wizard-header { - display: none; -} - -#file-options-header { - font-size: 15px; - font-weight: bold; - margin-bottom: 6px; -} - -#file-handling-options { - margin-top: 2em; -} - -#file-handling-options > label { - font-size: 14px; -} - -#file-handling-options radiogroup { - font-size: 13px; - margin-left: 1em; -} - -#file-handling-options description { - margin-top: .6em; - font-size: 11px; -} - -listbox, #result-description, #result-description-html { - font-size: 13px; -} - -#result-description-html { - line-height: 1.5; -} - -#mendeley-online-description, #mendeley-online-description2 { - font-size: 13px; - line-height: 1.5; -} - -#mendeley-online-description2 { - margin-top: 1em; -} - -#result-description-html a { - text-decoration: underline; -} - -button, checkbox { - font-size: 13px; -} - -#result-report-error { - margin-top: 13px; - margin-left: 0; -} diff --git a/scss/_zotero-react-client.scss b/scss/_zotero-react-client.scss index bd0ceb0c9d..3ee50b97a8 100644 --- a/scss/_zotero-react-client.scss +++ b/scss/_zotero-react-client.scss @@ -31,6 +31,7 @@ @import "components/editable"; @import "components/exportOptions"; @import "components/icons"; +@import "components/import-wizard"; @import "components/item-tree"; @import "components/longTagFixer"; @import "components/mainWindow"; diff --git a/scss/components/_import-wizard.scss b/scss/components/_import-wizard.scss new file mode 100644 index 0000000000..f485ca8cf4 --- /dev/null +++ b/scss/components/_import-wizard.scss @@ -0,0 +1,142 @@ +.import-wizard { + font-size: 12px; + + radiogroup { + radio { + padding-top: 5px; + } + } + + wizardpage { + display: flex; + flex-direction: column; + + > div { + display: block; + } + } + + a { + display: inline; + color: -moz-nativehyperlinktext; + cursor: pointer; + + &:hover, &:active, &:focus { + outline:none; + text-decoration: $link-hover-decoration; + } + } + + checkbox { + margin-left: 4px; // indent required for correct focus ring rendering + } + + button { + font: menu; + } + + h2 { + font-size: 14px; + font-weight: normal; + margin-bottom: 6px; + margin: 0; + } + + #other-files { + margin-left: -1px; + width: 400px; + } + + + #page-start { + radiogroup { + margin-top: 4px; + } + } + + #page-options-folder-import { + fieldset { + display: block; + border: none; + margin-left: 1em; + } + + #recreate-structure { + margin-top: .25em; + } + + .import-file-types-header { + margin-top: 1em; + } + + .page-options-file-type { + display: flex; + margin-top: .25em; + + &:first-child { + margin-top: 1em; + } + } + } + + #page-options-file-handling { + padding-top: 1.5em; + + .radioset { + // font-size: 13px; + margin-left: 1em; + } + } + + .page-options-file-handling-description { + margin-top: .6em; + font-size: 11px; + } + + #page-done-error { + margin-top: 2em; + text-align: right; + } + + .mendeley-online-intro { + // font-size: 13px; + + & +.mendeley-online-intro { + margin-top: 1em; + } + } + + #import-progress { + margin-top: 1em; + } + + #page-progress { + display: flex; + flex-direction: column; + } + + // TODO: deduplicate with rtfscan + .table-container { + display: flex; + height: 0; + flex-direction: column; + flex: 1 0 auto; + margin-top: 1.5em; + + > div { + display: flex; + flex: 1 0 auto; + background-color: -moz-field; + overflow: hidden; + position: relative; + } + + .virtualized-table-body { + display: flex; + + .windowed-list { + flex: 1 0 auto; + } + } + } +} \ No newline at end of file diff --git a/test/tests/folderImportTest.js b/test/tests/folderImportTest.js index d6609876ec..737f9d33b4 100644 --- a/test/tests/folderImportTest.js +++ b/test/tests/folderImportTest.js @@ -2,7 +2,7 @@ describe('Zotero_Import_Folder', function () { var tmpDir; - const uc = (name) => 'Zotero_Import_Folder_' + name; + const uc = name => 'Zotero_Import_Folder_' + name; before(async () => { tmpDir = await getTempDirectory(); @@ -111,7 +111,7 @@ describe('Zotero_Import_Folder', function () { const importer = new Zotero_Import_Folder({ folder: tmpDir, recreateStructure: false, - fileTypes: '*.png,*.txt', + fileTypes: '*.png,*.TXT', // should match case-insensitively mimeTypes: [] }); From c65e8f1621ae8780fc4c98ed295b227efa39b641 Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Mon, 19 Sep 2022 14:21:49 +0200 Subject: [PATCH 3/5] fx-compat: Convert rtfScan to use CE wizards Also: * Adds Style Configurator CE * Extends "base" CE to enable fluent l10n --- chrome/content/zotero/bibliography.js | 2 +- chrome/content/zotero/elements/base.js | 5 + .../zotero/elements/styleConfigurator.js | 262 ++++++ chrome/content/zotero/rtfScan.js | 758 +++++++++++++++++ chrome/content/zotero/rtfScan.jsx | 779 ------------------ chrome/content/zotero/rtfScan.xhtml | 72 ++ chrome/content/zotero/rtfScan.xul | 95 --- chrome/content/zotero/zoteroPane.xhtml | 2 +- chrome/locale/en-US/zotero/zotero.ftl | 55 +- scss/_zotero-react-client.scss | 1 + scss/components/_rtfScan.scss | 85 ++ scss/elements/style-configurator.scss | 58 ++ scss/themes/_light.scss | 2 + 13 files changed, 1297 insertions(+), 879 deletions(-) create mode 100644 chrome/content/zotero/elements/styleConfigurator.js create mode 100644 chrome/content/zotero/rtfScan.js delete mode 100644 chrome/content/zotero/rtfScan.jsx create mode 100644 chrome/content/zotero/rtfScan.xhtml delete mode 100644 chrome/content/zotero/rtfScan.xul create mode 100644 scss/components/_rtfScan.scss create mode 100644 scss/elements/style-configurator.scss diff --git a/chrome/content/zotero/bibliography.js b/chrome/content/zotero/bibliography.js index 431f0e75b1..a7df980cd0 100644 --- a/chrome/content/zotero/bibliography.js +++ b/chrome/content/zotero/bibliography.js @@ -225,7 +225,7 @@ var Zotero_File_Interface_Bibliography = new function() { updateLocaleMenu(selectedStyleObj); // - // For integrationDocPrefs.xul and rtfScan.xul + // For integrationDocPrefs.xul and rtfScan.xhtml // if (isDocPrefs) { // update status of displayAs box based on style class diff --git a/chrome/content/zotero/elements/base.js b/chrome/content/zotero/elements/base.js index 35ed5839bc..ebe1a5049c 100644 --- a/chrome/content/zotero/elements/base.js +++ b/chrome/content/zotero/elements/base.js @@ -58,6 +58,11 @@ class XULElementBase extends XULElement { shadow.append(content); } + MozXULElement.insertFTLIfNeeded("zotero.ftl"); + if (document.l10n) { + document.l10n.connectRoot(this.shadowRoot); + } + this.init(); } diff --git a/chrome/content/zotero/elements/styleConfigurator.js b/chrome/content/zotero/elements/styleConfigurator.js new file mode 100644 index 0000000000..3ac1e52b3a --- /dev/null +++ b/chrome/content/zotero/elements/styleConfigurator.js @@ -0,0 +1,262 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2022 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/* global XULElementBase: false */ + +{ + Services.scriptloader.loadSubScript("chrome://zotero/content/elements/base.js", this); + + class StyleSelector extends XULElementBase { + get stylesheets() { + return [ + 'chrome://global/skin/global.css', + 'chrome://zotero/skin/elements/style-configurator.css' + ]; + } + + content = MozXULElement.parseXULToFragment(` +
+
+ +
+
+ `); + + set value(val) { + this.shadowRoot.getElementById('style-list').value = val; + } + + get value() { + return this.shadowRoot.getElementById('style-list').value; + } + + async init() { + await Zotero.Styles.init(); + const styleListEl = this.shadowRoot.getElementById('style-list'); + + Zotero.Styles.getVisible().forEach((so) => { + const value = so.styleID; + // Add acronyms to APA and ASA to avoid confusion + // https://forums.zotero.org/discussion/comment/357135/#Comment_357135 + const label = so.title + .replace(/^American Psychological Association/, "American Psychological Association (APA)") + .replace(/^American Sociological Association/, "American Sociological Association (ASA)"); + + styleListEl.appendChild(MozXULElement.parseXULToFragment(` + ${label} + `)); + }); + this.value = this.getAttribute('value'); + this.shadowRoot.getElementById('style-list').addEventListener("select", () => { + const event = document.createEvent("Events"); + event.initEvent("select", true, true); + this.dispatchEvent(event); + }); + } + } + + class LocaleSelector extends XULElementBase { + content = MozXULElement.parseXULToFragment(` +
+
+ + + +
+
+ `); + + get stylesheets() { + return [ + 'chrome://global/skin/global.css', + 'chrome://zotero/skin/elements/style-configurator.css' + ]; + } + + get value() { + return this.localeListEl.value; + } + + set value(val) { + this._value = val; + const styleData = this._style ? Zotero.Styles.get(this._style) : null; + this.localeListEl.value = styleData && styleData.locale || this._value; + } + + get style() { + return this.style; + } + + set style(style) { + this._style = style; + const styleData = style ? Zotero.Styles.get(style) : null; + this.localeListEl.disabled = !style || !!styleData.locale; + this.localeListEl.value = styleData && styleData.locale || this._value || this.fallbackLocale; + } + + connectedCallback() { + super.connectedCallback(); + this.localeListEl = this.shadowRoot.getElementById('locale-list'); + this.localePopupEl = this.shadowRoot.querySelector('#locale-list > menupopup'); + } + + async init() { + this._style = this.getAttribute('style'); + this._value = this.getAttribute('value'); + + await Zotero.Styles.init(); + this.fallbackLocale = Zotero.Styles?.primaryDialects[Zotero.locale] || Zotero.locale; + + const menuLocales = Zotero.Utilities.deepCopy(Zotero.Styles.locales); + const menuLocalesKeys = Object.keys(menuLocales).sort(); + + // Make sure that client locale is always available as a choice + if (this.fallbackLocale && !(this.fallbackLocale in menuLocales)) { + menuLocales[this.fallbackLocale] = this.fallbackLocale; + menuLocalesKeys.unshift(this.fallbackLocale); + } + + menuLocalesKeys.forEach((key) => { + const label = menuLocales[key]; + + this.localePopupEl.appendChild(MozXULElement.parseXULToFragment(` + + `)); + }); + + this.value = this._value; + this.style = this._style; + + this.localeListEl.addEventListener("command", (_event) => { + this._value = this.localeListEl.value; + const event = document.createEvent("Events"); + event.initEvent("select", true, true); + this.dispatchEvent(event); + }); + } + } + + class StyleConfigurator extends XULElementBase { + content = MozXULElement.parseXULToFragment(` +
+
+ `); + + get stylesheets() { + return [ + 'chrome://global/skin/global.css', + 'chrome://zotero/skin/elements/style-configurator.css' + ]; + } + + set style(val) { + this.shadowRoot.getElementById('style-selector').value = val; + this.handleStyleChanged(val); + } + + get style() { + return this.shadowRoot.getElementById('style-selector').value; + } + + set locale(val) { + this.shadowRoot.getElementById('locale-selector').value = val; + } + + get locale() { + return this.shadowRoot.getElementById('locale-selector').value; + } + + set displayAs(val) { + this.shadowRoot.getElementById('display-as').value = val; + } + + get displayAs() { + return this.shadowRoot.getElementById('display-as').value; + } + + async init() { + this.shadowRoot.getElementById('style-configurator').style.display = 'none'; + await Zotero.Styles.init(); + this.shadowRoot.getElementById('style-configurator').style.display = ''; + this.shadowRoot.getElementById('style-selector').addEventListener('select', (_event) => { + this.handleStyleChanged(_event.target.value); + + const event = document.createEvent("Events"); + event.initEvent("select", true, true); + this.dispatchEvent(event); + }); + + this.shadowRoot.getElementById('locale-selector').addEventListener('select', (_event) => { + const event = document.createEvent("Events"); + event.initEvent("select", true, true); + this.dispatchEvent(event); + }); + + this.shadowRoot.getElementById('display-as').addEventListener('select', (_event) => { + const event = document.createEvent("Events"); + event.initEvent("select", true, true); + this.dispatchEvent(event); + }); + } + + handleStyleChanged(style) { + this.shadowRoot.getElementById('locale-selector').style = style; + const styleData = style ? Zotero.Styles.get(style) : null; + const isNoteStyle = (styleData || {}).class === 'note'; + this.shadowRoot.getElementById('display-as-wrapper').style.display = isNoteStyle ? '' : 'none'; + } + } + + customElements.define('locale-selector', LocaleSelector); + customElements.define('style-selector', StyleSelector); + customElements.define('style-configurator', StyleConfigurator); +} diff --git a/chrome/content/zotero/rtfScan.js b/chrome/content/zotero/rtfScan.js new file mode 100644 index 0000000000..793b8bcb0c --- /dev/null +++ b/chrome/content/zotero/rtfScan.js @@ -0,0 +1,758 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; + +import FilePicker from 'zotero/modules/filePicker'; +import VirtualizedTable from 'components/virtualized-table'; +import { getDOMElement } from 'components/icons'; + +var { Services } = ChromeUtils.import('resource://gre/modules/Services.jsm'); +Services.scriptloader.loadSubScript('chrome://zotero/content/elements/styleConfigurator.js', this); + +function _generateItem(citationString, itemName, action) { + return { + rtf: citationString, + item: itemName, + action + }; +} + +function _matchesItemCreators(creators, item, etAl) { + var itemCreators = item.getCreators(); + var primaryCreators = []; + var primaryCreatorTypeID = Zotero.CreatorTypes.getPrimaryIDForType(item.itemTypeID); + + // use only primary creators if primary creators exist + for (let i = 0; i < itemCreators.length; i++) { + if (itemCreators[i].creatorTypeID == primaryCreatorTypeID) { + primaryCreators.push(itemCreators[i]); + } + } + // if primaryCreators matches the creator list length, or if et al is being used, use only + // primary creators + if (primaryCreators.length == creators.length || etAl) itemCreators = primaryCreators; + + // for us to have an exact match, either the citation creator list length has to match the + // item creator list length, or et al has to be used + if (itemCreators.length == creators.length || (etAl && itemCreators.length > creators.length)) { + var matched = true; + for (let i = 0; i < creators.length; i++) { + // check each item creator to see if it matches + matched = matched && _matchesItemCreator(creators[i], itemCreators[i]); + if (!matched) break; + } + return matched; + } + + return false; +} + +function _matchesItemCreator(creator, itemCreator) { + // make sure last name matches + var lowerLast = itemCreator.lastName.toLowerCase(); + if (lowerLast != creator.substr(-lowerLast.length).toLowerCase()) return false; + + // make sure first name matches, if it exists + if (creator.length > lowerLast.length) { + var firstName = Zotero.Utilities.trim(creator.substr(0, creator.length - lowerLast.length)); + if (firstName.length) { + // check to see whether the first name is all initials + const initialRe = /^(?:[A-Z]\.? ?)+$/; + var m = initialRe.exec(firstName); + if (m) { + var initials = firstName.replace(/[^A-Z]/g, ""); + var itemInitials = itemCreator.firstName.split(/ +/g) + .map(name => name[0].toUpperCase()) + .join(""); + if (initials != itemInitials) return false; + } + else { + // not all initials; verify that the first name matches + var firstWord = firstName.substr(0, itemCreator.firstName).toLowerCase(); + var itemFirstWord = itemCreator.firstName.substr(0, itemCreator.firstName.indexOf(" ")).toLowerCase(); + if (firstWord != itemFirstWord) return false; + } + } + } + + return true; +} + + +const columns = [ + { dataKey: 'rtf', label: "zotero.rtfScan.citation.label", primary: true, flex: 4 }, + { dataKey: 'item', label: "zotero.rtfScan.itemName.label", flex: 5 }, + { dataKey: 'action', label: "", fixedWidth: true, width: "26px" }, +]; + +const BIBLIOGRAPHY_PLACEHOLDER = "\\{Bibliography\\}"; + +const initialRows = [ + { id: 'unmapped', rtf: Zotero.Intl.strings['zotero.rtfScan.unmappedCitations.label'], collapsed: false }, + { id: 'ambiguous', rtf: Zotero.Intl.strings['zotero.rtfScan.ambiguousCitations.label'], collapsed: false }, + { id: 'mapped', rtf: Zotero.Intl.strings['zotero.rtfScan.mappedCitations.label'], collapsed: false }, +]; +Object.freeze(initialRows); + +// const initialRowMap = {}; +// initialRows.forEach((row, index) => initialRowMap[row.id] = index); +const initialRowMap = initialRows.reduce((aggr, row, index) => { + aggr[row.id] = index; + return aggr; +}, {}); +Object.freeze(initialRowMap); + + +const Zotero_RTFScan = { // eslint-disable-line no-unused-vars, camelcase + wizard: null, + inputFile: null, + outputFile: null, + contents: null, + tree: null, + styleConfig: null, + citations: null, + citationItemIDs: null, + ids: 0, + rows: [...initialRows], + rowMap: { ...initialRowMap }, + + + async init() { + this.wizard = document.getElementById('rtfscan-wizard'); + + this.wizard.getPageById('page-start') + .addEventListener('pageshow', this.onIntroShow.bind(this)); + this.wizard.getPageById('page-start') + .addEventListener('pageadvanced', this.onIntroAdvanced.bind(this)); + this.wizard.getPageById('scan-page') + .addEventListener('pageshow', this.onScanPageShow.bind(this)); + this.wizard.getPageById('style-page') + .addEventListener('pageadvanced', this.onStylePageAdvanced.bind(this)); + this.wizard.getPageById('style-page') + .addEventListener('pagerewound', this.onStylePageRewound.bind(this)); + this.wizard.getPageById('format-page') + .addEventListener('pageshow', this.onFormatPageShow.bind(this)); + this.wizard.getPageById('citations-page') + .addEventListener('pageshow', this.onCitationsPageShow.bind(this)); + this.wizard.getPageById('citations-page') + .addEventListener('pagerewound', this.onCitationsPageRewound.bind(this)); + this.wizard.getPageById('complete-page') + .addEventListener('pageshow', this.onCompletePageShow.bind(this)); + + document + .getElementById('choose-input-file') + .addEventListener('click', this.onChooseInputFile.bind(this)); + document + .getElementById('choose-output-file') + .addEventListener('click', this.onChooseOutputFile.bind(this)); + + ReactDOM.render(( + this.rows.length} + id="rtfScan-table" + ref={ref => this.tree = ref} + renderItem={this.renderItem.bind(this)} + showHeader={true} + columns={columns} + containerWidth={document.getElementById('tree').clientWidth} + disableFontSizeScaling={true} + /> + ), document.getElementById('tree')); + + const lastInputFile = Zotero.Prefs.get("rtfScan.lastInputFile"); + if (lastInputFile) { + document.getElementById('input-path').value = lastInputFile; + this.inputFile = Zotero.File.pathToFile(lastInputFile); + } + const lastOutputFile = Zotero.Prefs.get("rtfScan.lastOutputFile"); + if (lastOutputFile) { + document.getElementById('output-path').value = lastOutputFile; + this.outputFile = Zotero.File.pathToFile(lastOutputFile); + } + + // wizard.shadowRoot content isn't exposed to our css + this.wizard.shadowRoot + .querySelector('.wizard-header-label').style.fontSize = '16px'; + + this.updatePath(); + document.getElementById("choose-input-file").focus(); + }, + + async onChooseInputFile(ev) { + if (ev.type === 'keydown' && ev.key !== ' ') { + return; + } + ev.stopPropagation(); + const fp = new FilePicker(); + fp.init(window, Zotero.getString("rtfScan.openTitle"), fp.modeOpen); + fp.appendFilters(fp.filterAll); + fp.appendFilter(Zotero.getString("rtfScan.rtf"), "*.rtf"); + const rv = await fp.show(); + + if (rv == fp.returnOK || rv == fp.returnReplace) { + this.inputFile = Zotero.File.pathToFile(fp.file); + this.updatePath(); + } + }, + + async onChooseOutputFile(ev) { + if (ev.type === 'keydown' && ev.key !== ' ') { + return; + } + ev.stopPropagation(); + const fp = new FilePicker(); + fp.init(window, Zotero.getString("rtfScan.saveTitle"), fp.modeSave); + fp.appendFilter(Zotero.getString("rtfScan.rtf"), "*.rtf"); + if (this.inputFile) { + let leafName = this.inputFile.leafName; + let dotIndex = leafName.lastIndexOf("."); + if (dotIndex !== -1) { + leafName = leafName.substr(0, dotIndex); + } + fp.defaultString = leafName + " " + Zotero.getString("rtfScan.scannedFileSuffix") + ".rtf"; + } + else { + fp.defaultString = "Untitled.rtf"; + } + + var rv = await fp.show(); + + if (rv == fp.returnOK || rv == fp.returnReplace) { + this.outputFile = Zotero.File.pathToFile(fp.file); + this.updatePath(); + } + }, + + onIntroShow() { + this.wizard.canRewind = false; + this.updatePath(); + }, + + onIntroAdvanced() { + Zotero.Prefs.set("rtfScan.lastInputFile", this.inputFile.path); + Zotero.Prefs.set("rtfScan.lastOutputFile", this.outputFile.path); + }, + + async onScanPageShow() { + this.wizard.canRewind = false; + this.wizard.canAdvance = false; + + // wait a ms so that UI thread gets updated + try { + await this.scanRTF(); + this.tree.invalidate(); + this.wizard.canRewind = true; + this.wizard.canAdvance = true; + this.wizard.advance(); + } + catch (e) { + Zotero.logError(e); + Zotero.debug(e); + } + }, + + onStylePageRewound(ev) { + ev.preventDefault(); + this.rows = [...initialRows]; + this.rowMap = { ...initialRowMap }; + this.wizard.goTo('page-start'); + }, + + onStylePageAdvanced() { + const styleConfigurator = document.getElementById('style-configurator'); + this.styleConfig = { + style: styleConfigurator.style, + locale: styleConfigurator.locale, + displayAs: styleConfigurator.displayAs + }; + Zotero.Prefs.set("export.lastStyle", this.styleConfig.style); + }, + + onCitationsPageShow() { + this.refreshCanAdvanceIfCitationsReady(); + }, + + onCitationsPageRewound(ev) { + ev.preventDefault(); + this.rows = [...initialRows]; + this.rowMap = { ...initialRowMap }; + this.wizard.goTo('page-start'); + }, + + onFormatPageShow() { + this.wizard.canAdvance = false; + this.wizard.canRewind = false; + + window.setTimeout(() => { + this.formatRTF(); + this.wizard.canRewind = true; + this.wizard.canAdvance = true; + this.wizard.advance(); + }, 0); + }, + + onCompletePageShow() { + this.wizard.canRewind = false; + }, + + onRowTwistyMouseUp(event, index) { + const row = this.rows[index]; + if (!row.collapsed) { + // Store children rows on the parent when collapsing + row.children = []; + const depth = this.getRowLevel(index); + for (let childIndex = index + 1; childIndex < this.rows.length && this.getRowLevel(this.rows[childIndex]) > depth; childIndex++) { + row.children.push(this.rows[childIndex]); + } + // And then remove them + this.removeRows(row.children.map((_, childIndex) => index + 1 + childIndex)); + } + else { + // Insert children rows from the ones stored on the parent + this.insertRows(row.children, index + 1); + delete row.children; + } + row.collapsed = !row.collapsed; + this.tree.invalidate(); + }, + + onActionMouseUp(event, index) { + let row = this.rows[index]; + if (!row.parent) return; + let level = this.getRowLevel(row); + if (level == 2) { // ambiguous citation item + let parentIndex = this.rowMap[row.parent.id]; + // Update parent item + row.parent.item = row.item; + + // Remove children + let children = []; + for (let childIndex = parentIndex + 1; childIndex < this.rows.length && this.getRowLevel(this.rows[childIndex]) >= level; childIndex++) { + children.push(this.rows[childIndex]); + } + this.removeRows(children.map((_, childIndex) => parentIndex + 1 + childIndex)); + + // Move citation to mapped rows + row.parent.parent = this.rows[this.rowMap.mapped]; + this.removeRows(parentIndex); + this.insertRows(row.parent, this.rows.length); + + // update array + this.citationItemIDs[row.parent.rtf] = [this.citationItemIDs[row.parent.rtf][index - parentIndex - 1]]; + } + else { // mapped or unmapped citation, or ambiguous citation parent + var citation = row.rtf; + var io = { singleSelection: true }; + if (this.citationItemIDs[citation] && this.citationItemIDs[citation].length == 1) { // mapped citation + // specify that item should be selected in window + io.select = this.citationItemIDs[citation][0]; + } + + window.openDialog('chrome://zotero/content/selectItemsDialog.xhtml', '', 'chrome,modal', io); + + if (io.dataOut && io.dataOut.length) { + var selectedItemID = io.dataOut[0]; + var selectedItem = Zotero.Items.get(selectedItemID); + // update item name + row.item = selectedItem.getField("title"); + + // Remove children + let children = []; + for (let childIndex = index + 1; childIndex < this.rows.length && this.getRowLevel(this.rows[childIndex]) > level; childIndex++) { + children.push(this.rows[childIndex]); + } + this.removeRows(children.map((_, childIndex) => index + 1 + childIndex)); + + if (row.parent.id != 'mapped') { + // Move citation to mapped rows + row.parent = this.rows[this.rowMap.mapped]; + this.removeRows(index); + this.insertRows(row, this.rows.length); + } + + // update array + this.citationItemIDs[citation] = [selectedItemID]; + } + } + this.tree.invalidate(); + this.refreshCanAdvanceIfCitationsReady(); + }, + + async scanRTF() { + // set up globals + this.citations = []; + this.citationItemIDs = {}; + + let unmappedRow = this.rows[this.rowMap.unmapped]; + let ambiguousRow = this.rows[this.rowMap.ambiguous]; + let mappedRow = this.rows[this.rowMap.mapped]; + + // set up regular expressions + // this assumes that names are >=2 chars or only capital initials and that there are no + // more than 4 names + const nameRe = "(?:[^ .,;]{2,} |[A-Z].? ?){0,3}[A-Z][^ .,;]+"; + const creatorRe = '((?:(?:' + nameRe + ', )*' + nameRe + '(?:,? and|,? \\&|,) )?' + nameRe + ')(,? et al\\.?)?'; + // TODO: localize "and" term + const creatorSplitRe = /(?:,| *(?:and|&)) +/g; + var citationRe = new RegExp('(\\\\\\{|; )(' + creatorRe + ',? (?:"([^"]+)(?:,"|",) )?([0-9]{4})[a-z]?)(?:,(?: pp?.?)? ([^ )]+))?(?=;|\\\\\\})|(([A-Z][^ .,;]+)(,? et al\\.?)? (\\\\\\{([0-9]{4})[a-z]?\\\\\\}))', "gm"); + + // read through RTF file and display items as they're found + // we could read the file in chunks, but unless people start having memory issues, it's + // probably faster and definitely simpler if we don't + this.contents = Zotero.File.getContents(this.inputFile) + .replace(/([^\\\r])\r?\n/, "$1 ") + .replace("\\'92", "'", "g") + .replace("\\rquote ", "’"); + var m; + var lastCitation = false; + while ((m = citationRe.exec(this.contents))) { + // determine whether suppressed or standard regular expression was used + if (m[2]) { // standard parenthetical + var citationString = m[2]; + var creators = m[3]; + // var etAl = !!m[4]; + var title = m[5]; + var date = m[6]; + var pages = m[7]; + var start = citationRe.lastIndex - m[0].length; + var end = citationRe.lastIndex + 2; + } + else { // suppressed + citationString = m[8]; + creators = m[9]; + // etAl = !!m[10]; + title = false; + date = m[12]; + pages = false; + start = citationRe.lastIndex - m[11].length; + end = citationRe.lastIndex; + } + citationString = citationString.replace("\\{", "{", "g").replace("\\}", "}", "g"); + var suppressAuthor = !m[2]; + + if (lastCitation && lastCitation.end >= start) { + // if this citation is just an extension of the last, add items to it + lastCitation.citationStrings.push(citationString); + lastCitation.pages.push(pages); + lastCitation.end = end; + } + else { + // otherwise, add another citation + lastCitation = { + citationStrings: [citationString], pages: [pages], + start, end, suppressAuthor + }; + this.citations.push(lastCitation); + } + + // only add each citation once + if (this.citationItemIDs[citationString]) continue; + Zotero.debug("Found citation " + citationString); + + // for each individual match, look for an item in the database + var s = new Zotero.Search; + creators = creators.replace(".", ""); + // TODO: localize "et al." term + creators = creators.split(creatorSplitRe); + + for (let i = 0; i < creators.length; i++) { + if (!creators[i]) { + if (i == creators.length - 1) { + break; + } + else { + creators.splice(i, 1); + } + } + + var spaceIndex = creators[i].lastIndexOf(" "); + var lastName = spaceIndex == -1 ? creators[i] : creators[i].substr(spaceIndex + 1); + s.addCondition("lastName", "contains", lastName); + } + if (title) s.addCondition("title", "contains", title); + s.addCondition("date", "is", date); + var ids = await s.search(); // eslint-disable-line no-await-in-loop + Zotero.debug("Mapped to " + ids); + this.citationItemIDs[citationString] = ids; + + if (!ids) { // no mapping found + let row = _generateItem(citationString, ""); + row.parent = unmappedRow; + this.insertRows(row, this.rowMap.ambiguous); + } + else { // some mapping found + var items = await Zotero.Items.getAsync(ids); // eslint-disable-line no-await-in-loop + if (items.length > 1) { + // check to see how well the author list matches the citation + var matchedItems = []; + for (let item of items) { + await item.loadAllData(); // eslint-disable-line no-await-in-loop + if (_matchesItemCreators(creators, item)) matchedItems.push(item); + } + + if (matchedItems.length != 0) items = matchedItems; + } + + if (items.length == 1) { // only one mapping + await items[0].loadAllData(); // eslint-disable-line no-await-in-loop + let row = _generateItem(citationString, items[0].getField("title")); + row.parent = mappedRow; + this.insertRows(row, this.rows.length); + this.citationItemIDs[citationString] = [items[0].id]; + } + else { // ambiguous mapping + let row = _generateItem(citationString, ""); + row.parent = ambiguousRow; + this.insertRows(row, this.rowMap.mapped); + + // generate child items + let children = []; + for (let item of items) { + let childRow = _generateItem("", item.getField("title"), true); + childRow.parent = row; + children.push(childRow); + } + this.insertRows(children, this.rowMap[row.id] + 1); + } + } + } + }, + + formatRTF() { + // load style and create ItemSet with all items + var zStyle = Zotero.Styles.get(this.styleConfig.style); + var cslEngine = zStyle.getCiteProc(this.styleConfig.locale, 'rtf'); + var isNote = zStyle.class == "note"; + + // create citations + // var k = 0; + var cslCitations = []; + var itemIDs = {}; + // var shouldBeSubsequent = {}; + for (let i = 0; i < this.citations.length; i++) { + let citation = this.citations[i]; + var cslCitation = { citationItems: [], properties: {} }; + if (isNote) { + cslCitation.properties.noteIndex = i; + } + + // create citation items + for (var j = 0; j < citation.citationStrings.length; j++) { + var citationItem = {}; + citationItem.id = this.citationItemIDs[citation.citationStrings[j]][0]; + itemIDs[citationItem.id] = true; + citationItem.locator = citation.pages[j]; + citationItem.label = "page"; + citationItem["suppress-author"] = citation.suppressAuthor && !isNote; + cslCitation.citationItems.push(citationItem); + } + + cslCitations.push(cslCitation); + } + Zotero.debug(cslCitations); + + itemIDs = Object.keys(itemIDs); + Zotero.debug(itemIDs); + + // prepare the list of rendered citations + var citationResults = cslEngine.rebuildProcessorState(cslCitations, "rtf"); + + // format citations + var contentArray = []; + var lastEnd = 0; + for (let i = 0; i < this.citations.length; i++) { + let citation = citationResults[i][2]; + Zotero.debug("Formatted " + citation); + + // if using notes, we might have to move the note after the punctuation + if (isNote && this.citations[i].start != 0 && this.contents[this.citations[i].start - 1] == " ") { + contentArray.push(this.contents.substring(lastEnd, this.citations[i].start - 1)); + } + else { + contentArray.push(this.contents.substring(lastEnd, this.citations[i].start)); + } + + lastEnd = this.citations[i].end; + if (isNote && this.citations[i].end < this.contents.length && ".,!?".indexOf(this.contents[this.citations[i].end]) !== -1) { + contentArray.push(this.contents[this.citations[i].end]); + lastEnd++; + } + + if (isNote) { + if (this.styleConfig.displayAs === 'endnotes') { + contentArray.push("{\\super\\chftn}\\ftnbj {\\footnote\\ftnalt {\\super\\chftn } " + citation + "}"); + } + else { // footnotes + contentArray.push("{\\super\\chftn}\\ftnbj {\\footnote {\\super\\chftn } " + citation + "}"); + } + } + else { + contentArray.push(citation); + } + } + contentArray.push(this.contents.substring(lastEnd)); + this.contents = contentArray.join(""); + + // add bibliography + if (zStyle.hasBibliography) { + var bibliography = Zotero.Cite.makeFormattedBibliography(cslEngine, "rtf"); + bibliography = bibliography.substring(5, bibliography.length - 1); + // fix line breaks + var linebreak = "\r\n"; + if (this.contents.indexOf("\r\n") == -1) { + bibliography = bibliography.replace("\r\n", "\n", "g"); + linebreak = "\n"; + } + + if (this.contents.indexOf(BIBLIOGRAPHY_PLACEHOLDER) !== -1) { + this.contents = this.contents.replace(BIBLIOGRAPHY_PLACEHOLDER, bibliography); + } + else { + // add two newlines before bibliography + bibliography = linebreak + "\\" + linebreak + "\\" + linebreak + bibliography; + + // add bibliography automatically inside last set of brackets closed + const bracketRe = /^\{+/; + var m = bracketRe.exec(this.contents); + if (m) { + var closeBracketRe = new RegExp("(\\}{" + m[0].length + "}\\s*)$"); + this.contents = this.contents.replace(closeBracketRe, bibliography + "$1"); + } + else { + this.contents += bibliography; + } + } + } + + cslEngine.free(); + + Zotero.File.putContents(this.outputFile, this.contents); + + // save locale + if (!zStyle.locale && this.styleConfig.locale) { + Zotero.Prefs.set("export.lastLocale", this.styleConfig.locale); + } + }, + + refreshCanAdvanceIfCitationsReady() { + let newCanAdvance = true; + for (let i in this.citationItemIDs) { + let itemList = this.citationItemIDs[i]; + if (itemList.length !== 1) { + newCanAdvance = false; + break; + } + } + this.wizard.canAdvance = newCanAdvance; + }, + + updatePath() { + this.wizard.canAdvance = this.inputFile && this.outputFile; + document.getElementById('input-path').value = this.inputFile ? this.inputFile.path : ''; + document.getElementById('output-path').value = this.outputFile ? this.outputFile.path : ''; + }, + + insertRows(newRows, beforeRow) { + if (!Array.isArray(newRows)) { + newRows = [newRows]; + } + this.rows.splice(beforeRow, 0, ...newRows); + newRows.forEach(row => row.id = this.ids++); + + // Refresh the row map + this.rowMap = {}; + this.rows.forEach((row, index) => this.rowMap[row.id] = index); + }, + + removeRows(indices) { + if (!Array.isArray(indices)) { + indices = [indices]; + } + // Reverse sort so we can safely splice out the entries from the rows array + indices.sort((a, b) => b - a); + for (const index of indices) { + this.rows.splice(index, 1); + } + // Refresh the row map + this.rowMap = {}; + this.rows.forEach((row, index) => this.rowMap[row.id] = index); + }, + + getRowLevel(row, depth = 0) { + if (typeof row == 'number') { + row = this.rows[row]; + } + if (!row.parent) { + return depth; + } + return this.getRowLevel(row.parent, depth + 1); + }, + + renderItem(index, selection, oldDiv = null, columns) { + const row = this.rows[index]; + let div; + if (oldDiv) { + div = oldDiv; + div.innerHTML = ""; + } + else { + div = document.createElement('div'); + div.className = "row"; + } + + for (const column of columns) { + if (column.primary) { + let twisty; + if (row.children || (this.rows[index + 1] && this.rows[index + 1].parent == row)) { + twisty = getDOMElement("IconTwisty"); + twisty.classList.add('twisty'); + if (!row.collapsed) { + twisty.classList.add('open'); + } + twisty.style.pointerEvents = 'auto'; + twisty.addEventListener('mousedown', event => event.stopPropagation()); + twisty.addEventListener('mouseup', event => this.onRowTwistyMouseUp(event, index), + { passive: true }); + } + else { + twisty = document.createElement('span'); + twisty.classList.add("spacer-twisty"); + } + + let textSpan = document.createElement('span'); + textSpan.className = "cell-text"; + textSpan.innerText = row[column.dataKey] || ""; + textSpan.style.paddingLeft = (5 + 20 * this.getRowLevel(row)) + 'px'; + + let span = document.createElement('span'); + span.className = `cell primary ${column.className}`; + span.appendChild(twisty); + span.appendChild(textSpan); + div.appendChild(span); + } + else if (column.dataKey == 'action') { + let span = document.createElement('span'); + span.className = `cell action ${column.className}`; + if (row.parent) { + if (row.action) { + span.appendChild(getDOMElement('IconRTFScanAccept')); + } + else { + span.appendChild(getDOMElement('IconRTFScanLink')); + } + span.addEventListener('mouseup', e => this.onActionMouseUp(e, index), { passive: true }); + span.style.pointerEvents = 'auto'; + } + + div.appendChild(span); + } + else { + let span = document.createElement('span'); + span.className = `cell ${column.className}`; + span.innerText = row[column.dataKey] || ""; + div.appendChild(span); + } + } + return div; + }, + +}; diff --git a/chrome/content/zotero/rtfScan.jsx b/chrome/content/zotero/rtfScan.jsx deleted file mode 100644 index 3cf30bd8ae..0000000000 --- a/chrome/content/zotero/rtfScan.jsx +++ /dev/null @@ -1,779 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2009 Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - -/** - * @fileOverview Tools for automatically retrieving a citation for the given PDF - */ - -import FilePicker from 'zotero/modules/filePicker'; -import React from 'react'; -import ReactDOM from 'react-dom'; -import VirtualizedTable from 'components/virtualized-table'; -import { getDOMElement } from 'components/icons'; - -/** - * Front end for recognizing PDFs - * @namespace - */ -var Zotero_RTFScan = new function() { - const ACCEPT_ICON = "chrome://zotero/skin/rtfscan-accept.png"; - const LINK_ICON = "chrome://zotero/skin/rtfscan-link.png"; - const BIBLIOGRAPHY_PLACEHOLDER = "\\{Bibliography\\}"; - - const columns = [ - { dataKey: 'rtf', label: "zotero.rtfScan.citation.label", primary: true, flex: 4 }, - { dataKey: 'item', label: "zotero.rtfScan.itemName.label", flex: 5 }, - { dataKey: 'action', label: "", fixedWidth: true, width: "26px" }, - ]; - var ids = 0; - var tree; - this._rows = [ - { id: 'unmapped', rtf: Zotero.getString('zotero.rtfScan.unmappedCitations.label'), collapsed: false }, - { id: 'ambiguous', rtf: Zotero.getString('zotero.rtfScan.ambiguousCitations.label'), collapsed: false }, - { id: 'mapped', rtf: Zotero.getString('zotero.rtfScan.mappedCitations.label'), collapsed: false }, - ]; - this._rowMap = {}; - this._rows.forEach((row, index) => this._rowMap[row.id] = index); - - var inputFile = null, outputFile = null; - var citations, citationItemIDs, contents; - - /** INTRO PAGE UI **/ - - /** - * Called when the first page is shown; loads target file from preference, if one is set - */ - this.introPageShowing = function() { - var path = Zotero.Prefs.get("rtfScan.lastInputFile"); - if(path) { - inputFile = Zotero.File.pathToFile(path); - } - var path = Zotero.Prefs.get("rtfScan.lastOutputFile"); - if(path) { - outputFile = Zotero.File.pathToFile(path); - } - _updatePath(); - document.getElementById("choose-input-file").focus(); - } - - /** - * Called when the first page is hidden - */ - this.introPageAdvanced = function() { - Zotero.Prefs.set("rtfScan.lastInputFile", inputFile.path); - Zotero.Prefs.set("rtfScan.lastOutputFile", outputFile.path); - } - - /** - * Called to select the file to be processed - */ - this.chooseInputFile = async function () { - // display file picker - var fp = new FilePicker(); - fp.init(window, Zotero.getString("rtfScan.openTitle"), fp.modeOpen); - - fp.appendFilters(fp.filterAll); - fp.appendFilter(Zotero.getString("rtfScan.rtf"), "*.rtf"); - - var rv = await fp.show(); - if (rv == fp.returnOK || rv == fp.returnReplace) { - inputFile = Zotero.File.pathToFile(fp.file); - _updatePath(); - } - } - - /** - * Called to select the output file - */ - this.chooseOutputFile = async function () { - var fp = new FilePicker(); - fp.init(window, Zotero.getString("rtfScan.saveTitle"), fp.modeSave); - fp.appendFilter(Zotero.getString("rtfScan.rtf"), "*.rtf"); - if(inputFile) { - var leafName = inputFile.leafName; - var dotIndex = leafName.lastIndexOf("."); - if(dotIndex != -1) { - leafName = leafName.substr(0, dotIndex); - } - fp.defaultString = leafName+" "+Zotero.getString("rtfScan.scannedFileSuffix")+".rtf"; - } else { - fp.defaultString = "Untitled.rtf"; - } - - var rv = await fp.show(); - if (rv == fp.returnOK || rv == fp.returnReplace) { - outputFile = Zotero.File.pathToFile(fp.file); - _updatePath(); - } - } - - /** - * Called to update the path label in the dialog box - * @private - */ - function _updatePath() { - document.documentElement.canAdvance = inputFile && outputFile; - if(inputFile) document.getElementById("input-path").value = inputFile.path; - if(outputFile) document.getElementById("output-path").value = outputFile.path; - } - - /** SCAN PAGE UI **/ - - /** - * Called when second page is shown. - */ - this.scanPageShowing = async function () { - // can't advance - document.documentElement.canAdvance = false; - - // wait a ms so that UI thread gets updated - try { - await this._scanRTF(); - } - catch (e) { - Zotero.logError(e); - Zotero.debug(e); - } - }; - - /** - * Scans file for citations, then proceeds to next wizard page. - */ - this._scanRTF = async () => { - // set up globals - citations = []; - citationItemIDs = {}; - - let unmappedRow = this._rows[this._rowMap['unmapped']]; - let ambiguousRow = this._rows[this._rowMap['ambiguous']]; - let mappedRow = this._rows[this._rowMap['mapped']]; - - // set up regular expressions - // this assumes that names are >=2 chars or only capital initials and that there are no - // more than 4 names - const nameRe = "(?:[^ .,;]{2,} |[A-Z].? ?){0,3}[A-Z][^ .,;]+"; - const creatorRe = '((?:(?:'+nameRe+', )*'+nameRe+'(?:,? and|,? \\&|,) )?'+nameRe+')(,? et al\\.?)?'; - // TODO: localize "and" term - const creatorSplitRe = /(?:,| *(?:and|\&)) +/g; - var citationRe = new RegExp('(\\\\\\{|; )('+creatorRe+',? (?:"([^"]+)(?:,"|",) )?([0-9]{4})[a-z]?)(?:,(?: pp?\.?)? ([^ )]+))?(?=;|\\\\\\})|(([A-Z][^ .,;]+)(,? et al\\.?)? (\\\\\\{([0-9]{4})[a-z]?\\\\\\}))', "gm"); - - // read through RTF file and display items as they're found - // we could read the file in chunks, but unless people start having memory issues, it's - // probably faster and definitely simpler if we don't - contents = Zotero.File.getContents(inputFile).replace(/([^\\\r])\r?\n/, "$1 ").replace("\\'92", "'", "g").replace("\\rquote ", "’"); - var m; - var lastCitation = false; - while ((m = citationRe.exec(contents))) { - // determine whether suppressed or standard regular expression was used - if (m[2]) { // standard parenthetical - var citationString = m[2]; - var creators = m[3]; - var etAl = !!m[4]; - var title = m[5]; - var date = m[6]; - var pages = m[7]; - var start = citationRe.lastIndex - m[0].length; - var end = citationRe.lastIndex + 2; - } - else { // suppressed - citationString = m[8]; - creators = m[9]; - etAl = !!m[10]; - title = false; - date = m[12]; - pages = false; - start = citationRe.lastIndex - m[11].length; - end = citationRe.lastIndex; - } - citationString = citationString.replace("\\{", "{", "g").replace("\\}", "}", "g"); - var suppressAuthor = !m[2]; - - if (lastCitation && lastCitation.end >= start) { - // if this citation is just an extension of the last, add items to it - lastCitation.citationStrings.push(citationString); - lastCitation.pages.push(pages); - lastCitation.end = end; - } - else { - // otherwise, add another citation - lastCitation = { citationStrings: [citationString], pages: [pages], - start, end, suppressAuthor }; - citations.push(lastCitation); - } - - // only add each citation once - if (citationItemIDs[citationString]) continue; - Zotero.debug("Found citation " + citationString); - - // for each individual match, look for an item in the database - var s = new Zotero.Search; - creators = creators.replace(".", ""); - // TODO: localize "et al." term - creators = creators.split(creatorSplitRe); - - for (let i = 0; i < creators.length; i++) { - if (!creators[i]) { - if (i == creators.length - 1) { - break; - } - else { - creators.splice(i, 1); - } - } - - var spaceIndex = creators[i].lastIndexOf(" "); - var lastName = spaceIndex == -1 ? creators[i] : creators[i].substr(spaceIndex+1); - s.addCondition("lastName", "contains", lastName); - } - if (title) s.addCondition("title", "contains", title); - s.addCondition("date", "is", date); - var ids = await s.search(); - Zotero.debug("Mapped to " + ids); - citationItemIDs[citationString] = ids; - - if (!ids) { // no mapping found - let row = _generateItem(citationString, ""); - row.parent = unmappedRow; - this._insertRows(row, this._rowMap.ambiguous); - } - else { // some mapping found - var items = await Zotero.Items.getAsync(ids); - if (items.length > 1) { - // check to see how well the author list matches the citation - var matchedItems = []; - for (let item of items) { - await item.loadAllData(); - if (_matchesItemCreators(creators, item)) matchedItems.push(item); - } - - if (matchedItems.length != 0) items = matchedItems; - } - - if (items.length == 1) { // only one mapping - await items[0].loadAllData(); - let row = _generateItem(citationString, items[0].getField("title")); - row.parent = mappedRow; - this._insertRows(row, this._rows.length); - citationItemIDs[citationString] = [items[0].id]; - } - else { // ambiguous mapping - let row = _generateItem(citationString, ""); - row.parent = ambiguousRow; - this._insertRows(row, this._rowMap.mapped); - - // generate child items - let children = []; - for (let item of items) { - let childRow = _generateItem("", item.getField("title"), true); - childRow.parent = row; - children.push(childRow); - } - this._insertRows(children, this._rowMap[row.id] + 1); - } - } - } - tree.invalidate(); - - // when scanning is complete, go to citations page - document.documentElement.canAdvance = true; - document.documentElement.advance(); - }; - - function _generateItem(citationString, itemName, action) { - return { - rtf: citationString, - item: itemName, - action - }; - } - - function _matchesItemCreators(creators, item, etAl) { - var itemCreators = item.getCreators(); - var primaryCreators = []; - var primaryCreatorTypeID = Zotero.CreatorTypes.getPrimaryIDForType(item.itemTypeID); - - // use only primary creators if primary creators exist - for(var i=0; i creators.length)) { - var matched = true; - for(var i=0; i lowerLast.length) { - var firstName = Zotero.Utilities.trim(creator.substr(0, creator.length-lowerLast.length)); - if(firstName.length) { - // check to see whether the first name is all initials - const initialRe = /^(?:[A-Z]\.? ?)+$/; - var m = initialRe.exec(firstName); - if(m) { - var initials = firstName.replace(/[^A-Z]/g, ""); - var itemInitials = itemCreator.firstName.split(/ +/g) - .map(name => name[0].toUpperCase()) - .join(""); - if(initials != itemInitials) return false; - } else { - // not all initials; verify that the first name matches - var firstWord = firstName.substr(0, itemCreator.firstName).toLowerCase(); - var itemFirstWord = itemCreator.firstName.substr(0, itemCreator.firstName.indexOf(" ")).toLowerCase(); - if(firstWord != itemFirstWord) return false; - } - } - } - - return true; - } - - /** CITATIONS PAGE UI **/ - - /** - * Called when citations page is shown to determine whether user can immediately advance. - */ - this.citationsPageShowing = function() { - _refreshCanAdvance(); - } - - /** - * Called when the citations page is rewound. Removes all citations from the list, clears - * globals, and returns to intro page. - */ - this.citationsPageRewound = function () { - // skip back to intro page - document.documentElement.currentPage = document.getElementById('intro-page'); - - this._rows = [ - { id: 'unmapped', rtf: Zotero.getString('zotero.rtfScan.unmappedCitations.label'), collapsed: false }, - { id: 'ambiguous', rtf: Zotero.getString('zotero.rtfScan.ambiguousCitations.label'), collapsed: false }, - { id: 'mapped', rtf: Zotero.getString('zotero.rtfScan.mappedCitations.label'), collapsed: false }, - ]; - this._rowMap = {}; - this._rows.forEach((row, index) => this._rowMap[row.id] = index); - - return false; - } - - /** - * Called when a tree item is clicked to remap a citation, or accept a suggestion for an - * ambiguous citation - */ - this.treeClick = function(event) { - var tree = document.getElementById("tree"); - - // get clicked cell - var { row, col } = tree.getCellAt(event.clientX, event.clientY); - - // figure out which item this corresponds to - var level = tree.view.getLevel(row); - } - - /** - * Determines whether the button to advance the wizard should be enabled or not based on whether - * unmapped citations exist, and sets the status appropriately - */ - function _refreshCanAdvance() { - var canAdvance = true; - for (let i in citationItemIDs) { - let itemList = citationItemIDs[i]; - if(itemList.length != 1) { - canAdvance = false; - break; - } - } - - document.documentElement.canAdvance = canAdvance; - } - - /** STYLE PAGE UI **/ - - /** - * Called when style page is shown to add styles to listbox. - */ - this.stylePageShowing = async function() { - await Zotero.Styles.init(); - Zotero_File_Interface_Bibliography.init({ - supportedNotes: ['footnotes', 'endnotes'] - }); - } - - /** - * Called when style page is hidden to save preferences. - */ - this.stylePageAdvanced = function() { - Zotero.Prefs.set("export.lastStyle", document.getElementById("style-listbox").selectedItem.value); - } - - /** FORMAT PAGE UI **/ - - this.formatPageShowing = function() { - // can't advance - document.documentElement.canAdvance = false; - - // wait a ms so that UI thread gets updated - window.setTimeout(function() { _formatRTF() }, 1); - } - - function _formatRTF() { - // load style and create ItemSet with all items - var zStyle = Zotero.Styles.get(document.getElementById("style-listbox").value) - var locale = document.getElementById("locale-menu").value; - var cslEngine = zStyle.getCiteProc(locale, 'rtf'); - var isNote = zStyle.class == "note"; - - // create citations - var k = 0; - var cslCitations = []; - var itemIDs = {}; - var shouldBeSubsequent = {}; - for(var i=0; i { - const row = this._rows[index]; - if (!row.collapsed) { - // Store children rows on the parent when collapsing - row.children = []; - const depth = this._getRowLevel(index); - for (let childIndex = index + 1; childIndex < this._rows.length && this._getRowLevel(this._rows[childIndex]) > depth; childIndex++) { - row.children.push(this._rows[childIndex]); - } - // And then remove them - this._removeRows(row.children.map((_, childIndex) => index + 1 + childIndex)); - } - else { - // Insert children rows from the ones stored on the parent - this._insertRows(row.children, index + 1); - delete row.children; - } - row.collapsed = !row.collapsed; - tree.invalidate(); - }; - - this._onActionMouseUp = (event, index) => { - let row = this._rows[index]; - if (!row.parent) return; - let level = this._getRowLevel(row); - if (level == 2) { // ambiguous citation item - let parentIndex = this._rowMap[row.parent.id]; - // Update parent item - row.parent.item = row.item; - - // Remove children - let children = []; - for (let childIndex = parentIndex + 1; childIndex < this._rows.length && this._getRowLevel(this._rows[childIndex]) >= level; childIndex++) { - children.push(this._rows[childIndex]); - } - this._removeRows(children.map((_, childIndex) => parentIndex + 1 + childIndex)); - - // Move citation to mapped rows - row.parent.parent = this._rows[this._rowMap.mapped]; - this._removeRows(parentIndex); - this._insertRows(row.parent, this._rows.length); - - // update array - citationItemIDs[row.parent.rtf] = [citationItemIDs[row.parent.rtf][index-parentIndex-1]]; - } - else { // mapped or unmapped citation, or ambiguous citation parent - var citation = row.rtf; - var io = { singleSelection: true }; - if (citationItemIDs[citation] && citationItemIDs[citation].length == 1) { // mapped citation - // specify that item should be selected in window - io.select = citationItemIDs[citation][0]; - } - - window.openDialog('chrome://zotero/content/selectItemsDialog.xul', '', 'chrome,modal', io); - - if (io.dataOut && io.dataOut.length) { - var selectedItemID = io.dataOut[0]; - var selectedItem = Zotero.Items.get(selectedItemID); - // update item name - row.item = selectedItem.getField("title"); - - // Remove children - let children = []; - for (let childIndex = index + 1; childIndex < this._rows.length && this._getRowLevel(this._rows[childIndex]) > level; childIndex++) { - children.push(this._rows[childIndex]); - } - this._removeRows(children.map((_, childIndex) => index + 1 + childIndex)); - - if (row.parent.id != 'mapped') { - // Move citation to mapped rows - row.parent = this._rows[this._rowMap.mapped]; - this._removeRows(index); - this._insertRows(row, this._rows.length); - } - - // update array - citationItemIDs[citation] = [selectedItemID]; - } - } - tree.invalidate(); - _refreshCanAdvance(); - }; - - this._insertRows = (rows, beforeRow) => { - if (!Array.isArray(rows)) { - rows = [rows]; - } - this._rows.splice(beforeRow, 0, ...rows); - rows.forEach(row => row.id = ids++); - for (let row of rows) { - row.id = ids++; - } - // Refresh the row map - this._rowMap = {}; - this._rows.forEach((row, index) => this._rowMap[row.id] = index); - }; - - this._removeRows = (indices) => { - if (!Array.isArray(indices)) { - indices = [indices]; - } - // Reverse sort so we can safely splice out the entries from the rows array - indices.sort((a, b) => b - a); - for (const index of indices) { - this._rows.splice(index, 1); - } - // Refresh the row map - this._rowMap = {}; - this._rows.forEach((row, index) => this._rowMap[row.id] = index); - }; - - this._getRowLevel = (row, depth=0) => { - if (typeof row == 'number') { - row = this._rows[row]; - } - if (!row.parent) { - return depth; - } - return this._getRowLevel(row.parent, depth+1); - } - - this._renderItem = (index, selection, oldDiv=null, columns) => { - const row = this._rows[index]; - let div; - if (oldDiv) { - div = oldDiv; - div.innerHTML = ""; - } - else { - div = document.createElement('div'); - div.className = "row"; - } - - for (const column of columns) { - if (column.primary) { - let twisty; - if (row.children || (this._rows[index + 1] && this._rows[index + 1].parent == row)) { - twisty = getDOMElement("IconTwisty"); - twisty.classList.add('twisty'); - if (!row.collapsed) { - twisty.classList.add('open'); - } - twisty.style.pointerEvents = 'auto'; - twisty.addEventListener('mousedown', event => event.stopPropagation()); - twisty.addEventListener('mouseup', event => this._onTwistyMouseUp(event, index), - { passive: true }); - } - else { - twisty = document.createElement('span'); - twisty.classList.add("spacer-twisty"); - } - - let textSpan = document.createElement('span'); - textSpan.className = "cell-text"; - textSpan.innerText = row[column.dataKey] || ""; - - let span = document.createElement('span'); - span.className = `cell primary ${column.className}`; - span.appendChild(twisty); - span.appendChild(textSpan); - span.style.paddingLeft = (5 + 20 * this._getRowLevel(row)) + 'px'; - div.appendChild(span); - } - else if (column.dataKey == 'action') { - let span = document.createElement('span'); - span.className = `cell action ${column.className}`; - if (row.parent) { - if (row.action) { - span.appendChild(getDOMElement('IconRTFScanAccept')); - } - else { - span.appendChild(getDOMElement('IconRTFScanLink')); - } - span.addEventListener('mouseup', e => this._onActionMouseUp(e, index), { passive: true }); - span.style.pointerEvents = 'auto'; - } - - div.appendChild(span); - } - else { - let span = document.createElement('span'); - span.className = `cell ${column.className}`; - span.innerText = row[column.dataKey] || ""; - div.appendChild(span); - } - } - return div; - }; - - this._initCitationTree = function () { - const domEl = document.querySelector('#tree'); - const elem = ( - this._rows.length} - id="rtfScan-table" - ref={ref => tree = ref} - renderItem={this._renderItem} - showHeader={true} - columns={columns} - disableFontSizeScaling={true} - /> - ); - return new Promise(resolve => ReactDOM.render(elem, domEl, resolve)); - }; -} diff --git a/chrome/content/zotero/rtfScan.xhtml b/chrome/content/zotero/rtfScan.xhtml new file mode 100644 index 0000000000..9cfcb6527b --- /dev/null +++ b/chrome/content/zotero/rtfScan.xhtml @@ -0,0 +1,72 @@ + + + + + + + + + + + + +