diff --git a/chrome/content/zotero/actors/ActorManager.jsm b/chrome/content/zotero/actors/ActorManager.jsm index 9c8f81de10..1a8812e886 100644 --- a/chrome/content/zotero/actors/ActorManager.jsm +++ b/chrome/content/zotero/actors/ActorManager.jsm @@ -77,3 +77,12 @@ if (AppConstants.platform === "macosx") { includeChrome: true }); } + +ChromeUtils.registerWindowActor("MendeleyAuth", { + parent: { + moduleURI: "chrome://zotero/content/actors/MendeleyAuthParent.jsm" + }, + child: { + moduleURI: "chrome://zotero/content/actors/MendeleyAuthChild.jsm" + } +}); diff --git a/chrome/content/zotero/actors/MendeleyAuthChild.jsm b/chrome/content/zotero/actors/MendeleyAuthChild.jsm new file mode 100644 index 0000000000..576d9cdc7a --- /dev/null +++ b/chrome/content/zotero/actors/MendeleyAuthChild.jsm @@ -0,0 +1,69 @@ +/* global JSWindowActorChild:false */ + +var EXPORTED_SYMBOLS = ["MendeleyAuthChild"]; // eslint-disable-line no-unused-vars + +class MendeleyAuthChild extends JSWindowActorChild { // eslint-disable-line no-unused-vars + async receiveMessage(message) { + let window = this.contentWindow; + let document = window.document; + + await this.documentIsReady(); + + switch (message.name) { + case "login": + try { + document.querySelector('input[name="pf.username"]').value = message.data.login; + document.querySelector("button[value=emailContinue]").removeAttribute("disabled"); + document.querySelector("button[value=emailContinue]").click(); + return true; + } + catch (e) { + this.sendAsyncMessage('debug', { kind: 'error', message: 'Failed to enter login', error: e.message }); + } + break; + case "password": + try { + document.querySelector('input[name="password"]').value = message.data.password; + document.querySelector("button[type=submit][value=signin]").removeAttribute("disabled"); + document.querySelector("button[type=submit][value=signin]").click(); + return true; + } + catch (e) { + this.sendAsyncMessage('debug', { kind: 'error', message: 'Failed to enter password', error: e.message }); + } + break; + } + + return false; + } + + // From Mozilla's ScreenshotsComponentChild.jsm + documentIsReady() { + const contentWindow = this.contentWindow; + const document = this.document; + + function readyEnough() { + return document.readyState === "complete"; + } + + if (readyEnough()) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + function onChange(event) { + if (event.type === "pagehide") { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + reject(new Error("document unloaded before it was ready")); + } + else if (readyEnough()) { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + resolve(); + } + } + document.addEventListener("readystatechange", onChange); + contentWindow.addEventListener("pagehide", onChange, { once: true }); + }); + } +} diff --git a/chrome/content/zotero/actors/MendeleyAuthParent.jsm b/chrome/content/zotero/actors/MendeleyAuthParent.jsm new file mode 100644 index 0000000000..70d2b63d6e --- /dev/null +++ b/chrome/content/zotero/actors/MendeleyAuthParent.jsm @@ -0,0 +1,22 @@ +/* global JSWindowActorParent:false */ + +var EXPORTED_SYMBOLS = ["MendeleyAuthParent"]; // eslint-disable-line no-unused-vars + +ChromeUtils.defineESModuleGetters(this, { + Zotero: "chrome://zotero/content/zotero.mjs" +}); + +class MendeleyAuthParent extends JSWindowActorParent { // eslint-disable-line no-unused-vars + async receiveMessage({ name, data }) { + switch (name) { + case "debug": { + if (data.kind === "log") { + Zotero.debug(`MendeleyAuth actor: ${data.message}`); + } + else if (data.kind === "error") { + Zotero.debug(`MendeleyAuth actor: ${data.message}. Error: ${data.error}`); + } + } + } + } +} diff --git a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js index 176c780f19..772892b936 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js +++ b/chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js @@ -6,6 +6,9 @@ const MENDELEY_API_URL = 'https://api.mendeley.com'; const CLIENT_ID = '6'; const CLIENT_NOT_VERY_SECRET = 'JtSAMzFdwC6RAED3RMZU'; const USER_AGENT = 'Mendeley Desktop/1.18'; +const API_DATA_TIMEOUT = 60000; +const API_TOKEN_TIMEOUT = 30000; +const ACCESS_TOKEN_TIMEOUT = 15000; const getTokens = async (url, bodyProps, headers = {}, options = {}) => { const body = Object.entries(bodyProps) @@ -18,9 +21,15 @@ const getTokens = async (url, bodyProps, headers = {}, options = {}) => { headers['User-Agent'] = USER_AGENT; } - options = { ...options, body, headers, timeout: 30000 }; + options = { ...options, body, headers, timeout: API_TOKEN_TIMEOUT }; const response = await Zotero.HTTP.request('POST', url, options); - return JSON.parse(response.responseText); + const parsedResponse = JSON.parse(response.responseText); + + return { + kind: Zotero.Prefs.get('import.mendeleyUseOAuth') ? 'oauth' : 'direct', + accessToken: parsedResponse.access_token, // eslint-disable-line camelcase + refreshToken: parsedResponse.refresh_token // eslint-disable-line camelcase + }; }; const directAuth = async (username, password, headers = {}, options = {}) => { @@ -33,7 +42,8 @@ const directAuth = async (username, password, headers = {}, options = {}) => { username }; - return getTokens(OAUTH_URL, bodyProps, headers, options); + const tokens = await getTokens(OAUTH_URL, bodyProps, headers, options); + return { username, password, tokens }; }; const codeAuth = async (code, headers = {}, options = {}) => { @@ -77,8 +87,8 @@ const getNextLinkFromResponse = (response) => { const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => { - headers = { ...headers, Authorization: `Bearer ${tokens.access_token}` }; - options = { ...options, headers, timeout: 60000 }; + headers = { ...headers, Authorization: `Bearer ${tokens.accessToken}` }; + options = { ...options, headers, timeout: API_DATA_TIMEOUT }; const method = 'GET'; // Run the request. If we see 401 or 403, try to refresh tokens and run the request again @@ -87,11 +97,18 @@ const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => { } catch (e) { if (e.status === 401 || e.status === 403) { - const newTokens = await refreshAuth(tokens.refresh_token); - // update tokens in the tokens object and in the header for next request - tokens.access_token = newTokens.access_token; // eslint-disable-line camelcase - tokens.refresh_token = newTokens.refresh_token; // eslint-disable-line camelcase - headers.Authorization = `Bearer ${tokens.access_token}`; + if (tokens.kind === 'referenceManager') { + const newToken = await obtainReferenceManagerTokenWithRetry(tokens.username, tokens.password); + tokens.accessToken = newToken; + headers.Authorization = `Bearer ${tokens.accessToken}`; + } + else { + const newTokens = await refreshAuth(tokens.refreshToken); + // update tokens in the tokens object and in the header for next request + tokens.accessToken = newTokens.accessToken; + tokens.refreshToken = newTokens.refreshToken; + headers.Authorization = `Bearer ${tokens.accessToken}`; + } } } @@ -126,5 +143,100 @@ const getAll = async (tokens, endPoint, params = {}, headers = {}, options = {}, return data; }; -return { codeAuth, directAuth, getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll }; +/** + * Obtain Reference Manager access token + * + * This function automates the process of logging into Mendeley Reference Manager and retrieving an + * access token. It uses a hidden browser to navigate through the login process and extract the + * access token from the cookies once logged in. This token enables access to some features + * not available through the normal API token (e.g. notebooks). + * + * @param {string} login - The login email for the Mendeley account. + * @param {string} password - The password for the Mendeley account. + * @returns {Promise} - A promise that resolves to the Mendeley access token. + * @throws {Error} - Throws an error if login fails, or if the access token cannot be obtained. + */ +const obtainReferenceManagerToken = async (login, password) => { + let { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + let cookieSandbox = new Zotero.CookieSandbox({}); + let browser = new HiddenBrowser({ + cookieSandbox, + docShell: { + allowMetaRedirects: true, + allowAuth: true, + } + }); + await browser._createdPromise; + + return new Promise((resolve, reject) => { + let hasEnteredLogin = false; + let hasEnteredPassword = false; + + browser.webProgress.addProgressListener({ + QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), + async onLocationChange() { + let url = browser.currentURI.spec; + Zotero.debug(`Obtain Mendeley access token, visiting "${url}"`, 5); + if (url.startsWith("https://id.elsevier.com/as/authorization.oauth2")) { + Zotero.debug("Logging in to Mendeley Reference Manager"); + if (!hasEnteredLogin) { + hasEnteredLogin = await browser.browsingContext.currentWindowGlobal + .getActor("MendeleyAuth") + .sendQuery("login", { login }); + Zotero.debug(`hasEnteredLogin: ${hasEnteredLogin}`); + if (!hasEnteredLogin) { + reject(new Error("Failed to enter login")); + } + } + } + else if (url.match(/https:\/\/id.elsevier.com\/as\/(.*?)\/resume\/as/)) { + Zotero.debug("Entering password to the Mendeley Reference Manager"); + if (!hasEnteredPassword) { + hasEnteredPassword = await browser.browsingContext.currentWindowGlobal + .getActor("MendeleyAuth") + .sendQuery("password", { password }); + Zotero.debug(`hasEnteredPassword: ${hasEnteredPassword}`); + if (!hasEnteredPassword) { + reject(new Error("Failed to enter password")); + } + } + } + else if (url.startsWith("https://www.mendeley.com/reference-manager/library")) { + const cookies = cookieSandbox.getCookiesForURI( + Services.io.newURI("https://www.mendeley.com/reference-manager/library") + ); + if (!cookies.accessToken) { + reject(new Error("Failed to obtain Mendeley access token")); + } + resolve(cookies.accessToken); + } + else { + Zotero.debug(`Ignoring unexpected URL while obtaining Mendeley access token: ${url}`); + } + } + }, Ci.nsIWebProgress.NOTIFY_LOCATION); + + browser.load("https://www.mendeley.com/sign-in?routeTo=https://www.mendeley.com/reference-manager/library/"); + Zotero.Promise.delay(ACCESS_TOKEN_TIMEOUT).then(() => { + reject(new Error("Timed out while obtaining Mendeley access token")); + }); + }); +}; + +const obtainReferenceManagerTokenWithRetry = async (login, password, tries = 3) => { + for (let i = 0; i < tries; i++) { + try { + return await obtainReferenceManagerToken(login, password); + } + catch (e) { + if (i === tries - 1) { + throw e; + } + Zotero.debug(`Failed to obtain Reference Manager token on attempt ${i + 1}. Retrying...`); + } + } + return null; +}; + +return { codeAuth, directAuth, getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll, obtainReferenceManagerToken, obtainReferenceManagerTokenWithRetry }; })(); diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 669b4c23a4..d6a9fe3ea6 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -1,5 +1,5 @@ /* eslint-disable no-await-in-loop, camelcase */ -/* global mendeleyDBMaps:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */ +/* global mendeleyDBMaps:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false, PathUtils: false */ var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"]; //eslint-disable-line no-unused-vars Components.utils.import("resource://gre/modules/Services.jsm"); @@ -11,7 +11,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men const importerVersion = 1; const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings; -const { apiFetch, codeAuth, get, getAll } = mendeleyAPIUtils; +const { apiFetch, codeAuth, get, getAll, obtainReferenceManagerTokenWithRetry } = mendeleyAPIUtils; const colorMap = new Map(); colorMap.set('rgb(255, 245, 173)', '#ffd400'); @@ -34,8 +34,10 @@ var Zotero_Import_Mendeley = function () { this.newItemsOnly = false; this.relinkOnly = false; this.numRelinked = 0; + this.skipNotebooks = false; this._tokens = null; + this._credentials = null; this._db = null; this._file = null; this._saveOptions = null; @@ -123,7 +125,8 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } if (this.mendeleyAuth) { - this._tokens = this.mendeleyAuth; + this._tokens = this.mendeleyAuth.tokens; + this._credentials = { username: this.mendeleyAuth.username, password: this.mendeleyAuth.password }; } else if (this.mendeleyCode) { this._tokens = await codeAuth(this.mendeleyCode); @@ -160,9 +163,11 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { : await this._getDocumentsDB(mendeleyGroupID); // Update progress to reflect items to import and remaining meta data stages - // We arbitrary set progress at approx 4%. We then add 8, one "tick" for each remaining meta data download. + // We arbitrary set progress at approx 4%. We then add 8, one "tick" for each remaining meta data download + // Finally we arbitrary add 5 "ticks" to represent steps required to import notebooks. This will be adjust + // later to account for the number of notebooks to import this._progress = Math.max(Math.floor(0.04 * documents.length), 2); - this._progressMax = documents.length + this._progress + 8; + this._progressMax = documents.length + this._progress + 8 + 5; // Get various attributes mapped to document ids let urls = this._tokens ? await this._getDocumentURLsAPI(documents) @@ -320,6 +325,53 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) { } this._interruptChecker(true); } + + if (this._credentials && !this.skipNotebooks) { + const token = await obtainReferenceManagerTokenWithRetry(this._credentials.username, this._credentials.password); + this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 4 more left + if (token) { + this._refManagerToken = { + kind: 'referenceManager', + accessToken: token, + username: this._credentials.username, + password: this._credentials.password + }; + const notebooks = await this._getNotebooksAPI(); + const notesContent = await Promise.all(notebooks.map(notebook => this._translateNotebookToNoteContent(libraryID, notebook))); + this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 1 more left + for (let i = 0; i < notebooks.length; i++) { + const notebook = notebooks[i]; + const predicate = 'mendeleyDB:notebookUUID'; + const uuid = notebook.id; + const noteContent = notesContent[i]; + let existingItem = await this._getItemByRelation(libraryID, predicate, uuid); + + if (this.newItemsOnly && existingItem) { + Zotero.debug(`Skipping import of notebook "${uuid}" as it already exists in Zotero as "${existingItem.key}" and newItemsOnly is set`, 5); + continue; + } + + const isMappedToExisting = !!existingItem; + Zotero.debug(isMappedToExisting ? `Updating existing notebook "${uuid}" -> "${existingItem.key}"` : `Importing new notebook "${uuid}"`, 5); + let item = existingItem ?? new Zotero.Item('note'); + item.libraryID = libraryID; + item.setNote(noteContent); + item.addRelation(predicate, uuid); + if (rootCollectionKey) { + item.addToCollection(rootCollectionKey); + } + await item.saveTx(this._saveOptions); + this.newItems.push(item); + this._progress += 1; + } + this._progress += 1; // progress one last arbitrary "tick" assigned to importing notebooks task + } + } + else { + Zotero.debug(`Skipping import of Mendeley notebooks: ${this.skipNotebooks ? `skipNotebooks = ${this.skipNotebooks}` : 'No reference manager credentials provided'}`); + this._progress += 5; // we've assigned 5 arbitrary "ticks" for importing notebooks task, advance progress since we cannot import notebooks + } + if (this.newItemsOnly && rootCollectionKey && this.newItems.length === 0) { Zotero.debug(`Mendeley Import detected no new items, removing import collection containing ${this.newCollections.length} collections created during the import`); const rootCollection = await Zotero.Collections.getAsync(options.collections[0]); @@ -1001,6 +1053,59 @@ Zotero_Import_Mendeley.prototype._getProfileDB = async function () { return rows[0]; }; +Zotero_Import_Mendeley.prototype._getNotebooksAPI = async function () { + let params = { }; + let headers = { Accept: 'application/json' }; + this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 3 more left + let notebooksMeta = await getAll(this._refManagerToken, 'notes/v1', params, headers, {}, this._interruptChecker); + this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 2 more left + this._progressMax += notebooksMeta.length * 2; // extend progress bar to account for the number of notebook we've discovered. One tick for fetching the notebook, one for processing and adding as a note + let notebookPromises = notebooksMeta.map(async (notebookEntryMeta) => { + const id = notebookEntryMeta.id; + const noteBookEntry = get(this._refManagerToken, `notes/v1/${id}`, params, headers); + this._progress += 1; + return noteBookEntry; + }); + return Promise.all(notebookPromises); +}; + +Zotero_Import_Mendeley.prototype._translateNotebookToNoteContent = async function (libraryID, mendeleyNotebook) { + let zoteroNoteBlocks = await Promise.all(mendeleyNotebook.blocks.map(async (block) => { + switch (block.type) { + case 'freetext': + return block.freetext?.value?.text ? `

${block.freetext?.value?.text}

` : ''; + case 'annotation': { + const idURI = block.annotation?.id; + if (idURI) { + const match = idURI.match(/https:\/\/api.mendeley.com\/annotations\/v2\/([0-9a-fA-F-]+)/); + if (match) { + const annotationUUID = match[1]; + let annotation = await this._getItemByRelation( + libraryID, + 'mendeleyDB:annotationUUID', + annotationUUID + ); + + let attachmentItem = Zotero.Items.get(annotation.parentID); + let jsonAnnotation = await Zotero.Annotations.toJSON(annotation); + jsonAnnotation.attachmentItemID = attachmentItem.id; + jsonAnnotation.id = annotation.key; + + const { html } = Zotero.EditorInstanceUtilities.serializeAnnotations([jsonAnnotation]); + return html; + } + } + } + } + return ''; + })); + if (mendeleyNotebook.title) { + zoteroNoteBlocks.unshift(`

${mendeleyNotebook.title}

`); + } + + return zoteroNoteBlocks.join("\n"); +}; + /** * Create API JSON array with item and any child attachments or notes */ diff --git a/test/tests/data/mendeleyMock/notebook.json b/test/tests/data/mendeleyMock/notebook.json new file mode 100644 index 0000000000..bd9664e016 --- /dev/null +++ b/test/tests/data/mendeleyMock/notebook.json @@ -0,0 +1,78 @@ +{ + "id": "8041a43b-f740-41ec-be42-a24cc5106d68", + "created": "2024-07-09T09:56:04.349Z", + "modified": "2024-11-04T13:54:23.050Z", + "title": "TEST", + "blocks": [ + { + "annotation": { + "stylesheet": { + "type": "CssStylesheet", + "value": ".annotation-color-rgb-250-244-209 { background-color: rgb(250,244,209); }" + }, + "creator": [ + { + "id": "https://www.mendeley.com/", + "type": "Software" + } + ], + "created": "2024-11-04T13:49:03.274Z", + "modified": "2024-11-04T13:49:03.274Z", + "id": "https://api.mendeley.com/annotations/v2/84f12446-3b49-4052-bbdc-832d28e1e072", + "body": [ + { + "purpose": "highlighting", + "format": "application/json", + "type": "TextualBody", + "value": { + "textAttributes": { + "entityRanges": [ + { + "offset": 0, + "length": 142, + "key": 0 + } + ] + }, + "text": "Highlighted text" + } + } + ], + "type": "Annotation", + "@context": "http://www.w3.org/ns/anno.jsonld", + "target": [ + { + "format": "application/pdf", + "selector": [ + { + "exact": "Highlighted text", + "type": "TextQuoteSelector" + }, + { + "conformsTo": "http://tools.ietf.org/rfc/rfc3778", + "type": "FragmentSelector", + "value": "page=1&viewrect=47.61420000000004,524.3954,200.42259999999993,-18.536&page=1&viewrect=47.61420000000004,499.6994000000001,249.05580000000003,-17.639999999999986&page=1&viewrect=47.61420000000004,479.6934000000001,228.01519999999994,-17.640000000000043&page=1&viewrect=47.61420000000004,459.68740000000014,133.57120000000003,-17.640000000000043" + } + ], + "source": "https://api.mendeley.com/documents/e4a6a49b-8622-3064-abb5-f1f1dac5e02e/files/099d8443-dbc9-6925-78ee-e462db2a28f3/", + "styleClass": "annotation-color-rgb-250-244-209" + } + ] + }, + "type": "annotation" + }, + { + "freetext": { + "format": "application/json", + "value": { + "textAttributes": { + "inlineStyleRanges": [], + "entityRanges": [] + }, + "text": "Lorem Ipsum" + } + }, + "type": "freetext" + } + ] +} \ No newline at end of file diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index caf66f5f73..dda70d8e97 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -1,11 +1,12 @@ -/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false */ +/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false, createAnnotation: false */ describe('Zotero_Import_Mendeley', function () { var server, httpd, httpdURL, importers; const getImporter = () => { const importer = new Zotero_Import_Mendeley(); - importer.mendeleyAuth = { access_token: 'access_token', refresh_token: 'refresh_token' };// eslint-disable-line camelcase + importer.mendeleyAuth = { kind: 'direct', tokens: { accessToken: 'access_token', refreshToken: 'refresh_token' } }; + importer.skipNotebooks = true; importers.push(importer); return importer; }; @@ -511,5 +512,37 @@ describe('Zotero_Import_Mendeley', function () { assert.equal(journalEmptyTags.getField('title'), 'This one has empty tags and keywords'); assert.equal(journalEmptyTags.getTags().length, 0); }); + + it('should translate a notebook content into Zotero note content', async () => { + let item = await createDataObject('item'); + item.itemType = 'journalArticle'; + item.title = 'Journal Article'; + await item.saveTx(); + + let attachment = await importFileAttachment('test.pdf', { parentID: item.id }); + await attachment.saveTx(); + + var annotation = await createAnnotation('highlight', attachment); + annotation.annotationText = 'Highlight text'; + annotation.annotationComment = 'Highlight comment'; + annotation.annotationPageLabel = '57'; + annotation.addRelation('mendeleyDB:annotationUUID', '84f12446-3b49-4052-bbdc-832d28e1e072'); + await annotation.saveTx(); + + let mendeleyNotebook = JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/notebook.json') + ); + const importer = getImporter(); + const noteContent = await importer._translateNotebookToNoteContent(Zotero.Libraries.userLibraryID, mendeleyNotebook); + + assert.match( + noteContent, + /^

TEST<\/h1>\n

“Highlight text”<\/span> \(, p. 57<\/span>\)<\/span> Highlight comment<\/p>\n

Lorem Ipsum<\/p>$/i + ); + + await annotation.eraseTx(); + await attachment.eraseTx(); + await item.eraseTx(); + }); }); });