scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-09-09 22:31:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Import Mendeley notebooks (#4829)

Browse source
This commit is contained in:
Tom Najdek 2024-11-12 06:13:17 +01:00 • committed by GitHub
parent a4e89e62f2
commit 055fbb68fc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 446 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
chrome/content/zotero/actors/ActorManager.jsm
View file

@ -77,3 +77,12 @@ if (AppConstants.platform === "macosx") {
includeChrome: true
});
}
ChromeUtils.registerWindowActor("MendeleyAuth", {
parent: {
moduleURI: "chrome://zotero/content/actors/MendeleyAuthParent.jsm"
},
child: {
moduleURI: "chrome://zotero/content/actors/MendeleyAuthChild.jsm"
}
});

69
chrome/content/zotero/actors/MendeleyAuthChild.jsm Normal file
View file

@ -0,0 +1,69 @@
/* global JSWindowActorChild:false */
var EXPORTED_SYMBOLS = ["MendeleyAuthChild"]; // eslint-disable-line no-unused-vars
class MendeleyAuthChild extends JSWindowActorChild { // eslint-disable-line no-unused-vars
async receiveMessage(message) {
let window = this.contentWindow;
let document = window.document;
await this.documentIsReady();
switch (message.name) {
case "login":
try {
document.querySelector('input[name="pf.username"]').value = message.data.login;
document.querySelector("button[value=emailContinue]").removeAttribute("disabled");
document.querySelector("button[value=emailContinue]").click();
return true;
}
catch (e) {
this.sendAsyncMessage('debug', { kind: 'error', message: 'Failed to enter login', error: e.message });
}
break;
case "password":
try {
document.querySelector('input[name="password"]').value = message.data.password;
document.querySelector("button[type=submit][value=signin]").removeAttribute("disabled");
document.querySelector("button[type=submit][value=signin]").click();
return true;
}
catch (e) {
this.sendAsyncMessage('debug', { kind: 'error', message: 'Failed to enter password', error: e.message });
}
break;
}
return false;
}
// From Mozilla's ScreenshotsComponentChild.jsm
documentIsReady() {
const contentWindow = this.contentWindow;
const document = this.document;
function readyEnough() {
return document.readyState === "complete";
}
if (readyEnough()) {
return Promise.resolve();
}
return new Promise((resolve, reject) => {
function onChange(event) {
if (event.type === "pagehide") {
document.removeEventListener("readystatechange", onChange);
contentWindow.removeEventListener("pagehide", onChange);
reject(new Error("document unloaded before it was ready"));
}
else if (readyEnough()) {
document.removeEventListener("readystatechange", onChange);
contentWindow.removeEventListener("pagehide", onChange);
resolve();
}
}
document.addEventListener("readystatechange", onChange);
contentWindow.addEventListener("pagehide", onChange, { once: true });
});
}
}

22
chrome/content/zotero/actors/MendeleyAuthParent.jsm Normal file
View file

@ -0,0 +1,22 @@
/* global JSWindowActorParent:false */
var EXPORTED_SYMBOLS = ["MendeleyAuthParent"]; // eslint-disable-line no-unused-vars
ChromeUtils.defineESModuleGetters(this, {
Zotero: "chrome://zotero/content/zotero.mjs"
});
class MendeleyAuthParent extends JSWindowActorParent { // eslint-disable-line no-unused-vars
async receiveMessage({ name, data }) {
switch (name) {
case "debug": {
if (data.kind === "log") {
Zotero.debug(`MendeleyAuth actor: ${data.message}`);
}
else if (data.kind === "error") {
Zotero.debug(`MendeleyAuth actor: ${data.message}. Error: ${data.error}`);
}
}
}
}
}

134
chrome/content/zotero/import/mendeley/mendeleyAPIUtils.js
View file

@ -6,6 +6,9 @@ const MENDELEY_API_URL = 'https://api.mendeley.com';
const CLIENT_ID = '6';
const CLIENT_NOT_VERY_SECRET = 'JtSAMzFdwC6RAED3RMZU';
const USER_AGENT = 'Mendeley Desktop/1.18';
const API_DATA_TIMEOUT = 60000;
const API_TOKEN_TIMEOUT = 30000;
const ACCESS_TOKEN_TIMEOUT = 15000;
const getTokens = async (url, bodyProps, headers = {}, options = {}) => {
const body = Object.entries(bodyProps)
@ -18,9 +21,15 @@ const getTokens = async (url, bodyProps, headers = {}, options = {}) => {
headers['User-Agent'] = USER_AGENT;
}
options = { ...options, body, headers, timeout: 30000 };
options = { ...options, body, headers, timeout: API_TOKEN_TIMEOUT };
const response = await Zotero.HTTP.request('POST', url, options);
return JSON.parse(response.responseText);
const parsedResponse = JSON.parse(response.responseText);
return {
kind: Zotero.Prefs.get('import.mendeleyUseOAuth') ? 'oauth' : 'direct',
accessToken: parsedResponse.access_token, // eslint-disable-line camelcase
refreshToken: parsedResponse.refresh_token // eslint-disable-line camelcase
};
};
const directAuth = async (username, password, headers = {}, options = {}) => {
@ -33,7 +42,8 @@ const directAuth = async (username, password, headers = {}, options = {}) => {
username
};
return getTokens(OAUTH_URL, bodyProps, headers, options);
const tokens = await getTokens(OAUTH_URL, bodyProps, headers, options);
return { username, password, tokens };
};
const codeAuth = async (code, headers = {}, options = {}) => {
@ -77,8 +87,8 @@ const getNextLinkFromResponse = (response) => {
const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => {
headers = { ...headers, Authorization: `Bearer ${tokens.access_token}` };
options = { ...options, headers, timeout: 60000 };
headers = { ...headers, Authorization: `Bearer ${tokens.accessToken}` };
options = { ...options, headers, timeout: API_DATA_TIMEOUT };
const method = 'GET';
// Run the request. If we see 401 or 403, try to refresh tokens and run the request again
@ -87,11 +97,18 @@ const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => {
}
catch (e) {
if (e.status === 401 || e.status === 403) {
const newTokens = await refreshAuth(tokens.refresh_token);
// update tokens in the tokens object and in the header for next request
tokens.access_token = newTokens.access_token; // eslint-disable-line camelcase
tokens.refresh_token = newTokens.refresh_token; // eslint-disable-line camelcase
headers.Authorization = `Bearer ${tokens.access_token}`;
if (tokens.kind === 'referenceManager') {
const newToken = await obtainReferenceManagerTokenWithRetry(tokens.username, tokens.password);
tokens.accessToken = newToken;
headers.Authorization = `Bearer ${tokens.accessToken}`;
}
else {
const newTokens = await refreshAuth(tokens.refreshToken);
// update tokens in the tokens object and in the header for next request
tokens.accessToken = newTokens.accessToken;
tokens.refreshToken = newTokens.refreshToken;
headers.Authorization = `Bearer ${tokens.accessToken}`;
}
}
}
@ -126,5 +143,100 @@ const getAll = async (tokens, endPoint, params = {}, headers = {}, options = {},
return data;
};
return { codeAuth, directAuth, getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll };
/**
* Obtain Reference Manager access token
*
* This function automates the process of logging into Mendeley Reference Manager and retrieving an
* access token. It uses a hidden browser to navigate through the login process and extract the
* access token from the cookies once logged in. This token enables access to some features
* not available through the normal API token (e.g. notebooks).
*
* @param {string} login - The login email for the Mendeley account.
* @param {string} password - The password for the Mendeley account.
* @returns {Promise<string>} - A promise that resolves to the Mendeley access token.
* @throws {Error} - Throws an error if login fails, or if the access token cannot be obtained.
*/
const obtainReferenceManagerToken = async (login, password) => {
let { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm");
let cookieSandbox = new Zotero.CookieSandbox({});
let browser = new HiddenBrowser({
cookieSandbox,
docShell: {
allowMetaRedirects: true,
allowAuth: true,
}
});
await browser._createdPromise;
return new Promise((resolve, reject) => {
let hasEnteredLogin = false;
let hasEnteredPassword = false;
browser.webProgress.addProgressListener({
QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]),
async onLocationChange() {
let url = browser.currentURI.spec;
Zotero.debug(`Obtain Mendeley access token, visiting "${url}"`, 5);
if (url.startsWith("https://id.elsevier.com/as/authorization.oauth2")) {
Zotero.debug("Logging in to Mendeley Reference Manager");
if (!hasEnteredLogin) {
hasEnteredLogin = await browser.browsingContext.currentWindowGlobal
.getActor("MendeleyAuth")
.sendQuery("login", { login });
Zotero.debug(`hasEnteredLogin: ${hasEnteredLogin}`);
if (!hasEnteredLogin) {
reject(new Error("Failed to enter login"));
}
}
}
else if (url.match(/https:\/\/id.elsevier.com\/as\/(.*?)\/resume\/as/)) {
Zotero.debug("Entering password to the Mendeley Reference Manager");
if (!hasEnteredPassword) {
hasEnteredPassword = await browser.browsingContext.currentWindowGlobal
.getActor("MendeleyAuth")
.sendQuery("password", { password });
Zotero.debug(`hasEnteredPassword: ${hasEnteredPassword}`);
if (!hasEnteredPassword) {
reject(new Error("Failed to enter password"));
}
}
}
else if (url.startsWith("https://www.mendeley.com/reference-manager/library")) {
const cookies = cookieSandbox.getCookiesForURI(
Services.io.newURI("https://www.mendeley.com/reference-manager/library")
);
if (!cookies.accessToken) {
reject(new Error("Failed to obtain Mendeley access token"));
}
resolve(cookies.accessToken);
}
else {
Zotero.debug(`Ignoring unexpected URL while obtaining Mendeley access token: ${url}`);
}
}
}, Ci.nsIWebProgress.NOTIFY_LOCATION);
browser.load("https://www.mendeley.com/sign-in?routeTo=https://www.mendeley.com/reference-manager/library/");
Zotero.Promise.delay(ACCESS_TOKEN_TIMEOUT).then(() => {
reject(new Error("Timed out while obtaining Mendeley access token"));
});
});
};
const obtainReferenceManagerTokenWithRetry = async (login, password, tries = 3) => {
for (let i = 0; i < tries; i++) {
try {
return await obtainReferenceManagerToken(login, password);
}
catch (e) {
if (i === tries - 1) {
throw e;
}
Zotero.debug(`Failed to obtain Reference Manager token on attempt ${i + 1}. Retrying...`);
}
}
return null;
};
return { codeAuth, directAuth, getNextLinkFromResponse, apiFetch, apiFetchUrl, get, getAll, obtainReferenceManagerToken, obtainReferenceManagerTokenWithRetry };
})();

115
chrome/content/zotero/import/mendeley/mendeleyImport.js
View file

@ -1,5 +1,5 @@
/* eslint-disable no-await-in-loop, camelcase */
/* global mendeleyDBMaps:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false */
/* global mendeleyDBMaps:false, mendeleyOnlineMappings:false, mendeleyAPIUtils:false, PathUtils: false */
var EXPORTED_SYMBOLS = ["Zotero_Import_Mendeley"]; //eslint-disable-line no-unused-vars
Components.utils.import("resource://gre/modules/Services.jsm");
@ -11,7 +11,7 @@ Services.scriptloader.loadSubScript("chrome://zotero/content/import/mendeley/men
const importerVersion = 1;
const { apiTypeToDBType, apiFieldToDBField } = mendeleyOnlineMappings;
const { apiFetch, codeAuth, get, getAll } = mendeleyAPIUtils;
const { apiFetch, codeAuth, get, getAll, obtainReferenceManagerTokenWithRetry } = mendeleyAPIUtils;
const colorMap = new Map();
colorMap.set('rgb(255, 245, 173)', '#ffd400');
@ -34,8 +34,10 @@ var Zotero_Import_Mendeley = function () {
this.newItemsOnly = false;
this.relinkOnly = false;
this.numRelinked = 0;
this.skipNotebooks = false;
this._tokens = null;
this._credentials = null;
this._db = null;
this._file = null;
this._saveOptions = null;
@ -123,7 +125,8 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
}
if (this.mendeleyAuth) {
this._tokens = this.mendeleyAuth;
this._tokens = this.mendeleyAuth.tokens;
this._credentials = { username: this.mendeleyAuth.username, password: this.mendeleyAuth.password };
}
else if (this.mendeleyCode) {
this._tokens = await codeAuth(this.mendeleyCode);
@ -160,9 +163,11 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
: await this._getDocumentsDB(mendeleyGroupID);
// Update progress to reflect items to import and remaining meta data stages
// We arbitrary set progress at approx 4%. We then add 8, one "tick" for each remaining meta data download.
// We arbitrary set progress at approx 4%. We then add 8, one "tick" for each remaining meta data download
// Finally we arbitrary add 5 "ticks" to represent steps required to import notebooks. This will be adjust
// later to account for the number of notebooks to import
this._progress = Math.max(Math.floor(0.04 * documents.length), 2);
this._progressMax = documents.length + this._progress + 8;
this._progressMax = documents.length + this._progress + 8 + 5;
// Get various attributes mapped to document ids
let urls = this._tokens
? await this._getDocumentURLsAPI(documents)
@ -320,6 +325,53 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
}
this._interruptChecker(true);
}
if (this._credentials && !this.skipNotebooks) {
const token = await obtainReferenceManagerTokenWithRetry(this._credentials.username, this._credentials.password);
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 4 more left
if (token) {
this._refManagerToken = {
kind: 'referenceManager',
accessToken: token,
username: this._credentials.username,
password: this._credentials.password
};
const notebooks = await this._getNotebooksAPI();
const notesContent = await Promise.all(notebooks.map(notebook => this._translateNotebookToNoteContent(libraryID, notebook)));
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 1 more left
for (let i = 0; i < notebooks.length; i++) {
const notebook = notebooks[i];
const predicate = 'mendeleyDB:notebookUUID';
const uuid = notebook.id;
const noteContent = notesContent[i];
let existingItem = await this._getItemByRelation(libraryID, predicate, uuid);
if (this.newItemsOnly && existingItem) {
Zotero.debug(`Skipping import of notebook "${uuid}" as it already exists in Zotero as "${existingItem.key}" and newItemsOnly is set`, 5);
continue;
}
const isMappedToExisting = !!existingItem;
Zotero.debug(isMappedToExisting ? `Updating existing notebook "${uuid}" -> "${existingItem.key}"` : `Importing new notebook "${uuid}"`, 5);
let item = existingItem ?? new Zotero.Item('note');
item.libraryID = libraryID;
item.setNote(noteContent);
item.addRelation(predicate, uuid);
if (rootCollectionKey) {
item.addToCollection(rootCollectionKey);
}
await item.saveTx(this._saveOptions);
this.newItems.push(item);
this._progress += 1;
}
this._progress += 1; // progress one last arbitrary "tick" assigned to importing notebooks task
}
}
else {
Zotero.debug(`Skipping import of Mendeley notebooks: ${this.skipNotebooks ? `skipNotebooks = ${this.skipNotebooks}` : 'No reference manager credentials provided'}`);
this._progress += 5; // we've assigned 5 arbitrary "ticks" for importing notebooks task, advance progress since we cannot import notebooks
}
if (this.newItemsOnly && rootCollectionKey && this.newItems.length === 0) {
Zotero.debug(`Mendeley Import detected no new items, removing import collection containing ${this.newCollections.length} collections created during the import`);
const rootCollection = await Zotero.Collections.getAsync(options.collections[0]);
@ -1001,6 +1053,59 @@ Zotero_Import_Mendeley.prototype._getProfileDB = async function () {
return rows[0];
};
Zotero_Import_Mendeley.prototype._getNotebooksAPI = async function () {
let params = { };
let headers = { Accept: 'application/json' };
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 3 more left
let notebooksMeta = await getAll(this._refManagerToken, 'notes/v1', params, headers, {}, this._interruptChecker);
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 2 more left
this._progressMax += notebooksMeta.length * 2; // extend progress bar to account for the number of notebook we've discovered. One tick for fetching the notebook, one for processing and adding as a note
let notebookPromises = notebooksMeta.map(async (notebookEntryMeta) => {
const id = notebookEntryMeta.id;
const noteBookEntry = get(this._refManagerToken, `notes/v1/${id}`, params, headers);
this._progress += 1;
return noteBookEntry;
});
return Promise.all(notebookPromises);
};
Zotero_Import_Mendeley.prototype._translateNotebookToNoteContent = async function (libraryID, mendeleyNotebook) {
let zoteroNoteBlocks = await Promise.all(mendeleyNotebook.blocks.map(async (block) => {
switch (block.type) {
case 'freetext':
return block.freetext?.value?.text ? `<p>${block.freetext?.value?.text}</p>` : '';
case 'annotation': {
const idURI = block.annotation?.id;
if (idURI) {
const match = idURI.match(/https:\/\/api.mendeley.com\/annotations\/v2\/([0-9a-fA-F-]+)/);
if (match) {
const annotationUUID = match[1];
let annotation = await this._getItemByRelation(
libraryID,
'mendeleyDB:annotationUUID',
annotationUUID
);
let attachmentItem = Zotero.Items.get(annotation.parentID);
let jsonAnnotation = await Zotero.Annotations.toJSON(annotation);
jsonAnnotation.attachmentItemID = attachmentItem.id;
jsonAnnotation.id = annotation.key;
const { html } = Zotero.EditorInstanceUtilities.serializeAnnotations([jsonAnnotation]);
return html;
}
}
}
}
return '';
}));
if (mendeleyNotebook.title) {
zoteroNoteBlocks.unshift(`<h1>${mendeleyNotebook.title}</h1>`);
}
return zoteroNoteBlocks.join("\n");
};
/**
* Create API JSON array with item and any child attachments or notes
*/

78
test/tests/data/mendeleyMock/notebook.json Normal file
View file

@ -0,0 +1,78 @@
{
"id": "8041a43b-f740-41ec-be42-a24cc5106d68",
"created": "2024-07-09T09:56:04.349Z",
"modified": "2024-11-04T13:54:23.050Z",
"title": "TEST",
"blocks": [
{
"annotation": {
"stylesheet": {
"type": "CssStylesheet",
"value": ".annotation-color-rgb-250-244-209 { background-color: rgb(250,244,209); }"
},
"creator": [
{
"id": "https://www.mendeley.com/",
"type": "Software"
}
],
"created": "2024-11-04T13:49:03.274Z",
"modified": "2024-11-04T13:49:03.274Z",
"id": "https://api.mendeley.com/annotations/v2/84f12446-3b49-4052-bbdc-832d28e1e072",
"body": [
{
"purpose": "highlighting",
"format": "application/json",
"type": "TextualBody",
"value": {
"textAttributes": {
"entityRanges": [
{
"offset": 0,
"length": 142,
"key": 0
}
]
},
"text": "Highlighted text"
}
}
],
"type": "Annotation",
"@context": "http://www.w3.org/ns/anno.jsonld",
"target": [
{
"format": "application/pdf",
"selector": [
{
"exact": "Highlighted text",
"type": "TextQuoteSelector"
},
{
"conformsTo": "http://tools.ietf.org/rfc/rfc3778",
"type": "FragmentSelector",
"value": "page=1&viewrect=47.61420000000004,524.3954,200.42259999999993,-18.536&page=1&viewrect=47.61420000000004,499.6994000000001,249.05580000000003,-17.639999999999986&page=1&viewrect=47.61420000000004,479.6934000000001,228.01519999999994,-17.640000000000043&page=1&viewrect=47.61420000000004,459.68740000000014,133.57120000000003,-17.640000000000043"
}
],
"source": "https://api.mendeley.com/documents/e4a6a49b-8622-3064-abb5-f1f1dac5e02e/files/099d8443-dbc9-6925-78ee-e462db2a28f3/",
"styleClass": "annotation-color-rgb-250-244-209"
}
]
},
"type": "annotation"
},
{
"freetext": {
"format": "application/json",
"value": {
"textAttributes": {
"inlineStyleRanges": [],
"entityRanges": []
},
"text": "Lorem Ipsum"
}
},
"type": "freetext"
}
]
}

37
test/tests/mendeleyImportTest.js
View file

@ -1,11 +1,12 @@
/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false */
/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false, createAnnotation: false */
describe('Zotero_Import_Mendeley', function () {
var server, httpd, httpdURL, importers;
const getImporter = () => {
const importer = new Zotero_Import_Mendeley();
importer.mendeleyAuth = { access_token: 'access_token', refresh_token: 'refresh_token' };// eslint-disable-line camelcase
importer.mendeleyAuth = { kind: 'direct', tokens: { accessToken: 'access_token', refreshToken: 'refresh_token' } };
importer.skipNotebooks = true;
importers.push(importer);
return importer;
};
@ -511,5 +512,37 @@ describe('Zotero_Import_Mendeley', function () {
assert.equal(journalEmptyTags.getField('title'), 'This one has empty tags and keywords');
assert.equal(journalEmptyTags.getTags().length, 0);
});
it('should translate a notebook content into Zotero note content', async () => {
let item = await createDataObject('item');
item.itemType = 'journalArticle';
item.title = 'Journal Article';
await item.saveTx();
let attachment = await importFileAttachment('test.pdf', { parentID: item.id });
await attachment.saveTx();
var annotation = await createAnnotation('highlight', attachment);
annotation.annotationText = 'Highlight text';
annotation.annotationComment = 'Highlight comment';
annotation.annotationPageLabel = '57';
annotation.addRelation('mendeleyDB:annotationUUID', '84f12446-3b49-4052-bbdc-832d28e1e072');
await annotation.saveTx();
let mendeleyNotebook = JSON.parse(
await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/notebook.json')
);
const importer = getImporter();
const noteContent = await importer._translateNotebookToNoteContent(Zotero.Libraries.userLibraryID, mendeleyNotebook);
assert.match(
noteContent,
/^<h1>TEST<\/h1>\n<p><span class="highlight" data-annotation="((?:(?:%[0-9A-F]{2}|[^<>'" %])+))">“Highlight text”<\/span> <span class="citation" data-citation="((?:(?:%[0-9A-F]{2}|[^<>'" %])+))">\(<span class="citation-item">, p. 57<\/span>\)<\/span> Highlight comment<\/p>\n<p>Lorem Ipsum<\/p>$/i
);
await annotation.eraseTx();
await attachment.eraseTx();
await item.eraseTx();
});
});
});