diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 76e0415a22..31751fd2aa 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -23,7 +23,8 @@ ***** END LICENSE BLOCK ***** */ -Components.utils.import("resource://gre/modules/osfile.jsm") +Components.utils.import("resource://gre/modules/osfile.jsm"); +Components.utils.import("resource://gre/modules/Services.jsm"); import FilePicker from 'zotero/filePicker'; /****Zotero_File_Exporter**** @@ -266,7 +267,7 @@ var Zotero_File_Interface = new function() { }; - this.showImportWizard = function () { + this.showImportWizard = function (extraArgs = {}) { var libraryID = Zotero.Libraries.userLibraryID; try { let zp = Zotero.getActiveZoteroPane(); @@ -276,7 +277,8 @@ var Zotero_File_Interface = new function() { Zotero.logError(e); } var args = { - libraryID + libraryID, + ...extraArgs }; args.wrappedJSObject = args; @@ -329,30 +331,39 @@ var Zotero_File_Interface = new function() { var defaultNewCollectionPrefix = Zotero.getString("fileInterface.imported"); var translation; - // Check if the file is an SQLite database - var sample = yield Zotero.File.getSample(file.path); - if (file.path == Zotero.DataDirectory.getDatabase()) { - // Blacklist the current Zotero database, which would cause a hang - } - else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { - // Mendeley import doesn't use the real translation architecture, but we create a - // translation object with the same interface + + if (options.mendeleyOnlineToken) { translation = yield _getMendeleyTranslation(); translation.createNewCollection = createNewCollection; - defaultNewCollectionPrefix = Zotero.getString( - 'fileInterface.appImportCollection', 'Mendeley' - ); + translation.token = options.mendeleyOnlineToken; } - else if (file.path.endsWith('@www.mendeley.com.sqlite') - || file.path.endsWith('online.sqlite')) { - // Keep in sync with importWizard.js - throw new Error('Encrypted Mendeley database'); + else { + // Check if the file is an SQLite database + var sample = yield Zotero.File.getSample(file.path); + if (file.path == Zotero.DataDirectory.getDatabase()) { + // Blacklist the current Zotero database, which would cause a hang + } + else if (Zotero.MIME.sniffForMIMEType(sample) == 'application/x-sqlite3') { + // Mendeley import doesn't use the real translation architecture, but we create a + // translation object with the same interface + translation = yield _getMendeleyTranslation(); + translation.createNewCollection = createNewCollection; + defaultNewCollectionPrefix = Zotero.getString( + 'fileInterface.appImportCollection', 'Mendeley' + ); + } + else if (file.path.endsWith('@www.mendeley.com.sqlite') + || file.path.endsWith('online.sqlite')) { + // Keep in sync with importWizard.js + throw new Error('Encrypted Mendeley database'); + } + + if (!translation) { + translation = new Zotero.Translate.Import(); + } + translation.setLocation(file); } - - if (!translation) { - translation = new Zotero.Translate.Import(); - } - translation.setLocation(file); + return _finishImport({ translation, createNewCollection, @@ -592,7 +603,7 @@ var Zotero_File_Interface = new function() { eval(xmlhttp.response); } return new Zotero_Import_Mendeley(); - } + }; /** @@ -849,7 +860,57 @@ var Zotero_File_Interface = new function() { return false; } } -} + + this.authenticateMendeleyOnlinePoll = function (win) { + if (win && win[0] && win[0].location) { + const matchResult = win[0].location.toString().match(/access_token=(.*?)(?:&|$)/i); + if (matchResult) { + const mendeleyAccessToken = matchResult[1]; + Zotero.getMainWindow().setTimeout(() => this.showImportWizard({ mendeleyAccessToken }), 0); + win.close(); + return; + } + } + + if (win && !win.closed) { + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + } + }; + + this.authenticateMendeleyOnline = function () { + const uri = 'https://api.mendeley.com/oauth/authorize?client_id=5907&redirect_uri=https%3A%2F%2Fzotero-static.s3.amazonaws.com%2Fmendeley_oauth_redirect.html&response_type=token&state=&scope=all'; + + var win = Services.wm.getMostRecentWindow("zotero:basicViewer"); + if (win) { + win.loadURI(uri); + } + else { + const ww = Services.ww; + const arg = Components.classes["@mozilla.org/supports-string;1"] + .createInstance(Components.interfaces.nsISupportsString); + arg.data = uri; + win = ww.openWindow(null, "chrome://zotero/content/standalone/basicViewer.xul", + "basicViewer", "chrome,dialog=yes,resizable,centerscreen,menubar,scrollbars", arg); + } + + let browser; + let func = function () { + win.removeEventListener("load", func); + browser = win.document.documentElement.getElementsByTagName('browser')[0]; + browser.addEventListener("pageshow", innerFunc); + }; + let innerFunc = function () { + browser.removeEventListener("pageshow", innerFunc); + win.outerWidth = Math.max(640, Math.min(1000, win.screen.availHeight)); + win.outerHeight = Math.max(480, Math.min(800, win.screen.availWidth)); + }; + + win.addEventListener("load", func); + + // polling executed by the main window because current (wizard) window will be closed + Zotero.getMainWindow().setTimeout(this.authenticateMendeleyOnlinePoll.bind(this, win), 200); + }; +}; // Handles the display of a progress indicator Zotero_File_Interface.Progress = new function() { diff --git a/chrome/content/zotero/import/importWizard.js b/chrome/content/zotero/import/importWizard.js index 559d8e8a9e..66cab732db 100644 --- a/chrome/content/zotero/import/importWizard.js +++ b/chrome/content/zotero/import/importWizard.js @@ -5,11 +5,12 @@ var Zotero_Import_Wizard = { _dbs: null, _file: null, _translation: null, + _mendeleyOnlineRedirectURLWithCode: null, + _mendeleyAccessToken: null, init: async function () { this._wizard = document.getElementById('import-wizard'); - var dbs = await Zotero_File_Interface.findMendeleyDatabases(); if (dbs.length) { document.getElementById('radio-import-source-mendeley').hidden = false; @@ -32,8 +33,17 @@ var Zotero_Import_Wizard = { document.getElementById('create-collection-checkbox').removeAttribute('checked'); } } + + if (args && args.mendeleyAccessToken) { + this._mendeleyAccessToken = args.mendeleyAccessToken; + this._wizard.goTo('page-options'); + } // Update labels + document.getElementById('radio-import-source-mendeley-online').label + = `Mendeley Reference Manager (${Zotero.getString('import.onlineImport')})`; + document.getElementById('radio-import-source-mendeley').label + = `Mendeley Desktop (${Zotero.getString('import.localImport')})`; document.getElementById('file-handling-store').label = Zotero.getString( 'import.fileHandling.store', Zotero.appName @@ -57,6 +67,11 @@ var Zotero_Import_Wizard = { case 'radio-import-source-file': await this.chooseFile(); break; + + case 'radio-import-source-mendeley-online': + wizard.goTo('mendeley-online-explanation'); + wizard.canRewind = true; + break; case 'radio-import-source-mendeley': this._dbs = await Zotero_File_Interface.findMendeleyDatabases(); @@ -85,7 +100,19 @@ var Zotero_Import_Wizard = { throw e; } }, - + + onMendeleyOnlineShow: async function () { + document.getElementById('mendeley-online-description').textContent = Zotero.getString( + 'import.mendeleyOnline.intro', [Zotero.appName, 'Mendeley Reference Manager', 'Mendeley'] + ); + }, + + onMendeleyOnlineAdvance: function () { + if (!this._mendeleyOnlineRedirectURLWithCode) { + Zotero_File_Interface.authenticateMendeleyOnline(); + window.close(); + } + }, goToStart: function () { this._wizard.goTo('page-start'); @@ -165,7 +192,7 @@ var Zotero_Import_Wizard = { onOptionsShown: function () { - + document.getElementById('file-handling-options').hidden = !!this._mendeleyAccessToken; }, @@ -192,7 +219,7 @@ var Zotero_Import_Wizard = { onImportStart: async function () { - if (!this._file) { + if (!this._file && !this._mendeleyAccessToken) { let index = document.getElementById('file-list').selectedIndex; this._file = this._dbs[index].path; } @@ -206,7 +233,8 @@ var Zotero_Import_Wizard = { onBeforeImport: this.onBeforeImport.bind(this), addToLibraryRoot: !document.getElementById('create-collection-checkbox') .hasAttribute('checked'), - linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1 + linkFiles: document.getElementById('file-handling-radio').selectedIndex == 1, + mendeleyOnlineToken: this._mendeleyAccessToken }); // Cancelled by user or due to error @@ -224,13 +252,12 @@ var Zotero_Import_Wizard = { catch (e) { if (e.message == 'Encrypted Mendeley database') { let url = 'https://www.zotero.org/support/kb/mendeley_import'; - this._onDone( - Zotero.getString('general.error'), - // TODO: Localize - `The selected Mendeley database cannot be read, likely because it is encrypted. ` - + `See How do I import a Mendeley library ` - + `into Zotero? for more information.` - ); + let HTML_NS = 'http://www.w3.org/1999/xhtml' + let elem = document.createElementNS(HTML_NS, 'div'); + elem.innerHTML = `The selected Mendeley database cannot be read, likely because it ` + + `is encrypted. See How do I import a ` + + `Mendeley library into Zotero? for more information.` + this._onDone(Zotero.getString('general.error'), elem); } else { this._onDone( @@ -310,9 +337,9 @@ var Zotero_Import_Wizard = { var xulElem = document.getElementById('result-description'); var htmlElem = document.getElementById('result-description-html'); - if (description.includes('href')) { - htmlElem.innerHTML = description; - Zotero.Utilities.Internal.updateHTMLInXUL(htmlElem); + if (description instanceof HTMLElement) { + htmlElem.appendChild(description); + Zotero.Utilities.Internal.updateHTMLInXUL(htmlElem, { callback: () => window.close() }); xulElem.hidden = true; htmlElem.setAttribute('display', 'block'); } @@ -321,7 +348,7 @@ var Zotero_Import_Wizard = { xulElem.hidden = false; htmlElem.setAttribute('display', 'none'); } - document.getElementById('result-description') + document.getElementById('result-description'); if (showReportErrorButton) { let button = document.getElementById('result-report-error'); diff --git a/chrome/content/zotero/import/importWizard.xul b/chrome/content/zotero/import/importWizard.xul index 8721aed14f..d204df3633 100644 --- a/chrome/content/zotero/import/importWizard.xul +++ b/chrome/content/zotero/import/importWizard.xul @@ -1,6 +1,7 @@ + @@ -21,9 +22,20 @@ onpageadvanced="Zotero_Import_Wizard.onModeChosen(); return false;"> - + + + + - { + let next = null; + let links = response.getResponseHeader('link'); + if (links) { + const matches = links.match(/<(.*?)>;\s+rel="next"/i); + + if (matches && matches.length > 1) { + next = matches[1]; + } + } + return next; +}; + +const apiFetchUrl = async (token, url, headers = {}, options = {}) => { + headers = { ...headers, Authorization: `Bearer ${token}` }; + return Zotero.HTTP.request('GET', url, { ...options, headers }); +}; + +const apiFetch = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const stringParams = Object.entries(params).map(p => p.join('=')).join('&'); + const url = MENDELEY_API_URL + '/' + endPoint + '?' + stringParams; + return apiFetchUrl(token, url, headers, options); +}; + +const get = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const response = await apiFetch(token, endPoint, params, headers, options); + return JSON.parse(response.responseText); +}; + +const getAll = async (token, endPoint, params = {}, headers = {}, options = {}) => { + const PER_PAGE = endPoint === 'annotations' ? 200 : 500; + const response = await apiFetch(token, endPoint, { ...params, limit: PER_PAGE }, headers, options); + var next = getNextLinkFromResponse(response); + var data = JSON.parse(response.responseText); + + while (next) { + const response = await apiFetchUrl(token, next, headers, options); //eslint-disable-line no-await-in-loop + data = [...data, ...JSON.parse(response.responseText)]; + next = getNextLinkFromResponse(response); + } + + return data; +}; + + +return { getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll }; +})(); diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 890ccb2c7c..a999c516ef 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -1,19 +1,28 @@ +/* global map:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */ var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"]; Components.utils.import("resource://gre/modules/Services.jsm"); Components.utils.import("resource://gre/modules/osfile.jsm"); Services.scriptloader.loadSubScript("chrome://zotero/content/include.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyOnlineMappings.js"); +Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleyAPIUtils.js"); + +const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; +const { apiFetch, get, getAll } = mendeleyAPIUtils; var Zotero_Import_Mendeley = function () { this.createNewCollection = null; this.linkFiles = null; this.newItems = []; + this.token = null; this._db; this._file; + this._saveOptions = null; this._itemDone; this._progress = 0; this._progressMax; + this._tmpFilesToDelete = []; }; Zotero_Import_Mendeley.prototype.setLocation = function (file) { @@ -42,6 +51,11 @@ Zotero_Import_Mendeley.prototype.setTranslator = function () {}; Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { this._linkFiles = options.linkFiles; + this._saveOptions = { + skipSelect: true, + ...(options.saveOptions || {}) + }; + this.timestamp = Date.now(); if (true) { Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/mendeleySchemaMap.js"); @@ -77,39 +91,119 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { // Disable syncing while we're importing var resumeSync = Zotero.Sync.Runner.delayIndefinite(); - this._db = new Zotero.DBConnection(this._file); + if (this._file) { + this._db = new Zotero.DBConnection(this._file); + } try { - if (!await this._isValidDatabase()) { + if (this._file && !await this._isValidDatabase()) { throw new Error("Not a valid Mendeley database"); } - - // Collections - let folders = await this._getFolders(mendeleyGroupID); - let collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); - let folderKeys = this._getFolderKeys(collectionJSON); + + if (!this._file && !this.token) { + throw new Error("Missing import token"); + } + + const folders = this.token + ? await this._getFoldersAPI(mendeleyGroupID) + : await this._getFoldersDB(mendeleyGroupID); + + const collectionJSON = this._foldersToAPIJSON(folders, rootCollectionKey); + const folderKeys = this._getFolderKeys(collectionJSON); await this._saveCollections(libraryID, collectionJSON, folderKeys); // // Items // - let documents = await this._getDocuments(mendeleyGroupID); + let documents = this.token + ? await this._getDocumentsAPI(mendeleyGroupID) + : await this._getDocumentsDB(mendeleyGroupID); + this._progressMax = documents.length; // Get various attributes mapped to document ids - let urls = await this._getDocumentURLs(mendeleyGroupID); - let creators = await this._getDocumentCreators(mendeleyGroupID, map.creatorTypes); - let tags = await this._getDocumentTags(mendeleyGroupID); - let collections = await this._getDocumentCollections( - mendeleyGroupID, - documents, - rootCollectionKey, - folderKeys - ); - let files = await this._getDocumentFiles(mendeleyGroupID); - let annotations = await this._getDocumentAnnotations(mendeleyGroupID); + let urls = this.token + ? await this._getDocumentURLsAPI(documents) + : await this._getDocumentURLsDB(mendeleyGroupID); + + let creators = this.token + ? await this._getDocumentCreatorsAPI(documents) + : await this._getDocumentCreatorsDB(mendeleyGroupID, map.creatorTypes); + + let tags = this.token + ? await this._getDocumentTagsAPI(documents) + : await this._getDocumentTagsDB(mendeleyGroupID); + + let collections = this.token + ? await this._getDocumentCollectionsAPI(documents, rootCollectionKey, folderKeys) + : await this._getDocumentCollectionsDB(mendeleyGroupID, documents, rootCollectionKey, folderKeys); + + let files = this.token + ? await this._getDocumentFilesAPI(documents) + : await this._getDocumentFilesDB(mendeleyGroupID); + + let annotations = this.token + ? await this._getDocumentAnnotationsAPI(mendeleyGroupID) + : await this._getDocumentAnnotationsDB(mendeleyGroupID); + + let profile = this.token + ? await this._getProfileAPI() + : await this._getProfileDB(); + + let groups = this.token + ? await this._getGroupsAPI() + : await this._getGroupsDB(); + + const fileHashLookup = new Map(); + + for (let [documentID, fileEntries] of files) { + for (let fileEntry of fileEntries) { + fileHashLookup.set(fileEntry.hash, documentID); + } + } + + + for (let group of groups) { + let groupAnnotations = this.token + ? await this._getDocumentAnnotationsAPI(group.id, profile.id) + : await this._getDocumentAnnotationsDB(group.id, profile.id); + + for (let groupAnnotationsList of groupAnnotations.values()) { + for (let groupAnnotation of groupAnnotationsList) { + if (fileHashLookup.has(groupAnnotation.hash)) { + const targetDocumentID = fileHashLookup.get(groupAnnotation.hash); + if (!annotations.has(targetDocumentID)) { + annotations.set(targetDocumentID, []); + } + annotations.get(targetDocumentID).push(groupAnnotation); + } + } + } + } + for (let document of documents) { let docURLs = urls.get(document.id); let docFiles = files.get(document.id); + + if (this.token) { + // extract identifiers + ['arxiv', 'doi', 'isbn', 'issn', 'pmid', 'scopus', 'pui', 'pii', 'sgr'].forEach( + i => document[i] = (document.identifiers || {})[i] + ); + + // normalise item type from the API to match Mendeley DB + document.type = apiTypeToDBType[document.type] || document.type; + + // normalise field names from the API to match Mendeley DB + Object.keys(apiFieldToDBField).forEach((key) => { + if (key in document) { + const newKey = apiFieldToDBField[key]; + if (newKey) { + document[newKey] = document[key]; + } + delete document[key]; + } + }); + } // If there's a single PDF file, use "PDF" for the attachment title if (docFiles && docFiles.length == 1 && docFiles[0].fileURL.endsWith('.pdf')) { @@ -173,7 +267,14 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } finally { try { - await this._db.closeDatabase(); + if (this._file) { + await this._db.closeDatabase(); + } + if (this.token) { + await Promise.all( + this._tmpFilesToDelete.map(f => this._removeTemporaryFile(f)) + ); + } } catch (e) { Zotero.logError(e); @@ -183,6 +284,18 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } }; +Zotero_Import_Mendeley.prototype._removeTemporaryFile = async function (file) { + const containingDir = OS.Path.dirname(file); + try { + await Zotero.File.removeIfExists(file); + await OS.File.removeEmptyDir(containingDir); + } + catch (e) { + Zotero.logError(e); + } +}; + + Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { var tables = [ 'DocumentContributors', @@ -208,7 +321,7 @@ Zotero_Import_Mendeley.prototype._isValidDatabase = async function () { // // Collections // -Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { +Zotero_Import_Mendeley.prototype._getFoldersDB = async function (groupID) { return this._db.queryAsync( `SELECT F.id, F.uuid, F.name, ` // Top-level folders can have a parentId of 0 instead of -1 (by mistake?) @@ -221,6 +334,23 @@ Zotero_Import_Mendeley.prototype._getFolders = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getFoldersAPI = async function (groupID) { + const params = {}; + const headers = { Accept: 'application/vnd.mendeley-folder.1+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + return (await getAll(this.token, 'folders', params, headers)).map(f => ({ + id: f.id, + uuid: f.id, + name: f.name, + parentId: f.parent_id || -1, + remoteUuid: f.id + })); +}; + /** * Get flat array of collection API JSON with parentCollection set * @@ -306,9 +436,7 @@ Zotero_Import_Mendeley.prototype._saveCollections = async function (libraryID, j delete toSave.remoteUUID; collection.fromJSON(toSave); - await collection.saveTx({ - skipSelect: true - }); + await collection.saveTx(this._saveOptions); } }; @@ -337,13 +465,13 @@ Zotero_Import_Mendeley.prototype._findExistingCollection = async function (libra Zotero.debug(`Found existing collection ${collections[0].libraryKey} for ` + `${predicate} ${collectionJSON.relations[predicate]}`); return collections[0]; -} +}; // // Items // -Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentsDB = async function (groupID) { return this._db.queryAsync( `SELECT D.*, RD.remoteUuid FROM Documents D ` + `JOIN RemoteDocuments RD ON (D.id=RD.documentId) ` @@ -352,12 +480,28 @@ Zotero_Import_Mendeley.prototype._getDocuments = async function (groupID) { ); }; +Zotero_Import_Mendeley.prototype._getDocumentsAPI = async function (groupID) { + const params = { view: 'all' }; + const headers = { Accept: 'application/vnd.mendeley-document-with-files-list+json' }; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + + return (await getAll(this.token, 'documents', params, headers)).map(d => ({ + ...d, + uuid: d.id, + remoteUuid: d.id + })); +}; + /** * Get a Map of document ids to arrays of URLs * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentURLsDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, CAST(url AS TEXT) AS url FROM DocumentUrls DU ` + `JOIN RemoteDocuments USING (documentId) ` @@ -374,13 +518,17 @@ Zotero_Import_Mendeley.prototype._getDocumentURLs = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentURLsAPI = async function (documents) { + return new Map(documents.map(d => ([d.id, d.websites]))); +}; + /** * Get a Map of document ids to arrays of creator API JSON * * @param {Integer} groupID * @param {Object} creatorTypeMap - Mapping of Mendeley creator types to Zotero creator types */ -Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, creatorTypeMap) { +Zotero_Import_Mendeley.prototype._getDocumentCreatorsDB = async function (groupID, creatorTypeMap) { var rows = await this._db.queryAsync( `SELECT * FROM DocumentContributors ` + `JOIN RemoteDocuments USING (documentId) ` @@ -401,10 +549,21 @@ Zotero_Import_Mendeley.prototype._getDocumentCreators = async function (groupID, return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCreatorsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const authors = (doc.authors || []).map(c => this._makeCreator('author', c.first_name, c.last_name)); + const editors = (doc.editors || []).map(c => this._makeCreator('editor', c.first_name, c.last_name)); + const translators = (doc.translators || []).map(c => this._makeCreator('translator', c.first_name, c.last_name)); + map.set(doc.id, [...authors, ...editors, ...translators]); + } + return map; +}; + /** * Get a Map of document ids to arrays of tag API JSON */ -Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentTagsDB = async function (groupID) { var rows = await this._db.queryAsync( // Manual tags `SELECT documentId, tag, 0 AS type FROM DocumentTags ` @@ -432,10 +591,19 @@ Zotero_Import_Mendeley.prototype._getDocumentTags = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._getDocumentTagsAPI = async function (documents) { + var map = new Map(); + for (let doc of documents) { + const tags = [...(doc.tags || []).map(tag => ({ tag, type: 0 })), ...(doc.keywords || []).map(tag => ({ tag, type: 1 }))]; + map.set(doc.id, tags); + } + return map; +}; + /** * Get a Map of document ids to arrays of collection keys */ -Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (groupID, documents, rootCollectionKey, folderKeys) { +Zotero_Import_Mendeley.prototype._getDocumentCollectionsDB = async function (groupID, documents, rootCollectionKey, folderKeys) { var rows = await this._db.queryAsync( `SELECT documentId, folderId FROM DocumentFolders DF ` + `JOIN RemoteDocuments USING (documentId) ` @@ -460,12 +628,28 @@ Zotero_Import_Mendeley.prototype._getDocumentCollections = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentCollectionsAPI = async function (documents, rootCollectionKey, folderKeys) { + return new Map( + documents.map((d) => { + const keys = (d.folder_uuids || []).map((fuuid) => { + const key = folderKeys.get(fuuid); + if (!key) { + Zotero.debug(`Document folder ${fuuid} not found -- skipping`, 2); + } + return key; + }).filter(Boolean); + // Add all documents to root collection if specified + return [d.id, [...keys, ...(rootCollectionKey ? [rootCollectionKey] : [])]]; + }) + ); +}; + /** * Get a Map of document ids to arrays of file metadata * * @return {Map} */ -Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentFilesDB = async function (groupID) { var rows = await this._db.queryAsync( `SELECT documentId, hash, localUrl FROM DocumentFiles ` + `JOIN Files USING (hash) ` @@ -490,10 +674,59 @@ Zotero_Import_Mendeley.prototype._getDocumentFiles = async function (groupID) { return map; }; +Zotero_Import_Mendeley.prototype._fetchFile = async function (fileID, filePath) { + const fileDir = OS.Path.dirname(filePath); + await Zotero.File.createDirectoryIfMissingAsync(fileDir); + const xhr = await apiFetch(this.token, `files/${fileID}`, {}, {}, { responseType: 'blob', followRedirects: false }); + const uri = xhr.getResponseHeader('location'); + await Zotero.File.download(uri, filePath); + + this._progress += 1; + if (this._itemDone) { + this._itemDone(); + } +}; + +Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (documents) { + const map = new Map(); + + let totalSize = 0; + + Components.utils.import("resource://zotero/concurrentCaller.js"); + var caller = new ConcurrentCaller({ + numConcurrent: 6, + onError: e => Zotero.logError(e), + Promise: Zotero.Promise + }); + + for (let doc of documents) { + const files = []; + for (let file of (doc.files || [])) { + const fileName = file.file_name || 'file'; + const tmpFile = OS.Path.join(Zotero.getTempDirectory().path, `mendeley-online-import-${this.timestamp}-${file.id}`, fileName); + this._tmpFilesToDelete.push(tmpFile); + caller.add(this._fetchFile.bind(this, file.id, tmpFile)); + files.push({ + fileURL: OS.Path.toFileURI(tmpFile), + title: file.file_name || '', + contentType: file.mime_type || '', + hash: file.filehash, + }); + totalSize += file.size; + this._progressMax += 1; + } + map.set(doc.id, files); + } + // check if enough space available totalSize + await caller.runAll(); + return map; +}; + + /** * Get a Map of document ids to arrays of annotations */ -Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (groupID) { +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsDB = async function (groupID, profileID = null) { var map = new Map(); // Highlights @@ -504,8 +737,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group + `JOIN RemoteDocuments USING (documentId) ` + `JOIN FileHighlightRects FHR ON (FH.id=FHR.highlightId) ` + `WHERE groupId=? ` + + (profileID !== null ? `AND profileUuid=? ` : ``) + `ORDER BY FH.id, page, y1 DESC, x1`, - groupID + profileID !== null ? [groupID, profileID] : groupID ); var currentHighlight = null; for (let i = 0; i < rows.length; i++) { @@ -548,8 +782,9 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group + `FROM FileNotes ` + `JOIN RemoteDocuments USING (documentId) ` + `WHERE groupId=? ` + + (profileID !== null ? `AND profileUuid=? ` : ``) + `ORDER BY page, y, x`, - groupID + profileID !== null ? [groupID, profileID] : groupID ); for (let row of rows) { let docAnnotations = map.get(row.documentId); @@ -572,6 +807,100 @@ Zotero_Import_Mendeley.prototype._getDocumentAnnotations = async function (group return map; }; +Zotero_Import_Mendeley.prototype._getDocumentAnnotationsAPI = async function (groupID, profileID = null) { + const params = {}; + + if (groupID && groupID !== 0) { + params.group_id = groupID; //eslint-disable-line camelcase + } + + const map = new Map(); + (await getAll(this.token, 'annotations', params, { Accept: 'application/vnd.mendeley-annotation.1+json' })) + .forEach((a) => { + if (profileID !== null && a.profile_id !== profileID) { + // optionally filter annotations by profile id + return; + } + + if (a.type === 'note') { + // This is a "general note" in Mendeley. It appears to be the same thing as + // document.note thus not an annotations and can be discarded + return; + } + + const rects = (a.positions || []).map(position => ({ + x1: (position.top_left || {}).x || 0, + y1: (position.top_left || {}).y || 0, + x2: (position.bottom_right || {}).x || 0, + y2: (position.bottom_right || {}).y || 0, + })); + let page = 1; + try { + // const page = ((a.positions || [])[0] || {}).page; // ??? + page = a.positions[0].page; + } + catch (e) { } + + const annotation = { + id: a.id, + color: a.color ? `#${a.color.r.toString(16)}${a.color.g.toString(16)}${a.color.b.toString(16)}` : null, + dateAdded: a.created, + dateModified: a.last_modified, + hash: a.filehash, + uuid: a.id, + page, + }; + + if (a.type === 'highlight') { + annotation.type = 'highlight'; + annotation.rects = rects; + } + + if (a.type === 'sticky_note' && rects.length > 0) { + annotation.type = 'note'; + annotation.note = a.text; + annotation.x = rects[0].x1; + annotation.y = rects[0].y1; + } + + if (!map.has(a.document_id)) { + map.set(a.document_id, []); + } + map.get(a.document_id).push(annotation); + }); + return map; +}; + +Zotero_Import_Mendeley.prototype._getGroupsAPI = async function () { + const params = { type: 'all' }; + const headers = { Accept: 'application/vnd.mendeley-group-list+json' }; + + return getAll(this.token, 'groups/v2', params, headers); +}; + +Zotero_Import_Mendeley.prototype._getGroupsDB = async function () { + const rows = await this._db.queryAsync( + "SELECT id, remoteUUid, name, isOwner FROM Groups WHERE remoteUuID != ?", [''] + ); + return rows; +}; + + +Zotero_Import_Mendeley.prototype._getProfileAPI = async function () { + const params = { }; + const headers = { Accept: 'application/vnd.mendeley-profiles.2+json' }; + + return get(this.token, 'profiles/v2/me', params, headers); +}; + +Zotero_Import_Mendeley.prototype._getProfileDB = async function () { + const rows = await this._db.queryAsync( + "SELECT uuid as id, firstName, lastName, displayName FROM Profiles ORDER BY ROWID LIMIT 1" + ); + + return rows[0]; +}; + /** * Create API JSON array with item and any child attachments or notes */ @@ -923,8 +1252,8 @@ Zotero_Import_Mendeley.prototype._saveItems = async function (libraryID, json) { item.fromJSON(toSave); await item.saveTx({ - skipSelect: true, - skipDateModifiedUpdate: true + skipDateModifiedUpdate: true, + ...this._saveOptions }); if (itemJSON.documentID) { idMap.set(itemJSON.documentID, item.id); @@ -1076,9 +1405,7 @@ Zotero_Import_Mendeley.prototype._saveFilesAndAnnotations = async function (file attachment.setRelations({ 'mendeleyDB:fileHash': file.hash }); - await attachment.saveTx({ - skipSelect: true - }); + await attachment.saveTx(this._saveOptions); } } else { @@ -1124,7 +1451,8 @@ Zotero_Import_Mendeley.prototype._isDownloadedFile = function (path) { return parentDir.endsWith(OS.Path.join('Application Support', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd', 'Mendeley Desktop', 'Downloaded')) || parentDir.endsWith(OS.Path.join('Local', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) - || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')); + || parentDir.endsWith(OS.Path.join('data', 'Mendeley Ltd.', 'Mendeley Desktop', 'Downloaded')) + || parentDir.startsWith(OS.Path.join(Zotero.getTempDirectory().path, 'mendeley-online-import')); // Mendeley Online Importer } /** @@ -1170,7 +1498,7 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations, let type = 'application/pdf'; if (Zotero.MIME.sniffForMIMEType(await Zotero.File.getSample(file)) == type) { attachmentItem.attachmentContentType = type; - await attachmentItem.saveTx(); + await attachmentItem.saveTx(this._saveOptions); } } @@ -1269,9 +1597,7 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations, }); } note.setNote('

' + Zotero.getString('extractedAnnotations') + '

\n' + noteStrings.join('\n')); - return note.saveTx({ - skipSelect: true - }); + return note.saveTx(this._saveOptions); }; diff --git a/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js new file mode 100644 index 0000000000..1d81ffeabe --- /dev/null +++ b/chrome/content/zotero/import/mendeley/mendeleyOnlineMappings.js @@ -0,0 +1,57 @@ +/* eslint-disable camelcase, no-unused-vars */ + +var mendeleyOnlineMappings = { + // lookup to normalise from item type presented by API to item type as stored in DB + apiTypeToDBType: { + bill: 'Bill', + book: 'Book', + book_section: 'BookSection', + case: 'Case', + computer_program: 'ComputerProgram', + conference_proceedings: 'ConferenceProceedings', + encyclopedia_article: 'EncyclopediaArticle', + film: 'Film', + generic: 'Generic', + hearing: 'Hearing', + journal: 'JournalArticle', + magazine_article: 'MagazineArticle', + newspaper_article: 'NewspaperArticle', + patent: 'Patent', + report: 'Report', + statute: 'Statute', + television_broadcast: 'TelevisionBroadcast', + thesis: 'Thesis', + web_page: 'WebPage', + working_paper: 'WorkingPaper' + }, + apiFieldToDBField: { + accessed: 'dateAccessed', + authors: false, // all author types handled separately + citation_key: 'citationKey', + created: 'added', + edition: 'edition', + editors: false, // all author types handled separately + file_attached: false, + folder_uuids: false, // collections handled separately + group_id: 'groupID', + identifiers: false, // identifiers are separately copied directly into document + keywords: false, // tags handled separately + last_modified: 'modified', + notes: 'note', + patent_application_number: 'patentApplicationNumber', + patent_legal_status: 'patentLegalStatus', + patent_owner: 'patentOwner', + private_publication: 'privatePublication', + profile_id: 'profileID', + reprint_edition: 'reprintEdition', + revision: 'revisionNumber', + series_editor: 'seriesEditor', + series_number: 'seriesNumber', + short_title: 'shortTitle', + source_type: 'sourceType', + tags: false, // tags handled separately + translators: false, // all author types handled separately + user_context: 'userContext', + websites: false // URLs handled separately + } +}; diff --git a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js index af8e929e33..51ca57f8ff 100644 --- a/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js +++ b/chrome/content/zotero/import/mendeley/mendeleySchemaMap.js @@ -10,6 +10,7 @@ var map = { EncyclopediaArticle: "encyclopediaArticle", Film: "film", Generic: "document", + Hearing: "hearing", JournalArticle: "journalArticle", MagazineArticle: "magazineArticle", NewspaperArticle: "newspaperArticle", diff --git a/chrome/content/zotero/standalone/basicViewer.xul b/chrome/content/zotero/standalone/basicViewer.xul index 3db5e5642f..67c7f3f8a4 100644 --- a/chrome/content/zotero/standalone/basicViewer.xul +++ b/chrome/content/zotero/standalone/basicViewer.xul @@ -44,7 +44,7 @@ onload="window.sizeToContent()" windowtype="zotero:basicViewer" title="&brandShortName;" - width="1000" height="500" + width="1000" height="700" persist="screenX screenY width height sizemode">