mirror of
https://github.com/zotero/zotero.git
synced 2026-10-08 03:08:19 +00:00
Improve importing from Mendeley
- Make the annotations importer more resilient to missing data - Fix notebooks not being imported in some cases - Fix an instance of a hidden browser leaked by the Mendeley Importer
This commit is contained in:
Tom Najdek
parent
6c437e3df2
commit
d3ef6cfebd
6 changed files with 161 additions and 34 deletions
|
|
@ -117,7 +117,7 @@ const apiFetchUrl = async (tokens, url, headers = {}, options = {}) => {
|
|||
|
||||
const apiFetch = async (tokens, endPoint, params = {}, headers = {}, options = {}) => {
|
||||
const stringParams = Object.entries(params).map(p => p.join('=')).join('&');
|
||||
const url = MENDELEY_API_URL + '/' + endPoint + '?' + stringParams;
|
||||
const url = `${MENDELEY_API_URL}/${endPoint}${stringParams.length ? '?' + stringParams : ''}`;
|
||||
return apiFetchUrl(tokens, url, headers, options);
|
||||
};
|
||||
|
||||
|
|
@ -183,8 +183,8 @@ const obtainReferenceManagerToken = async (login, password) => {
|
|||
hasEnteredLogin = await browser.browsingContext.currentWindowGlobal
|
||||
.getActor("MendeleyAuth")
|
||||
.sendQuery("login", { login });
|
||||
Zotero.debug(`hasEnteredLogin: ${hasEnteredLogin}`);
|
||||
if (!hasEnteredLogin) {
|
||||
browser.destroy();
|
||||
reject(new Error("Failed to enter login"));
|
||||
}
|
||||
}
|
||||
|
|
@ -195,8 +195,8 @@ const obtainReferenceManagerToken = async (login, password) => {
|
|||
hasEnteredPassword = await browser.browsingContext.currentWindowGlobal
|
||||
.getActor("MendeleyAuth")
|
||||
.sendQuery("password", { password });
|
||||
Zotero.debug(`hasEnteredPassword: ${hasEnteredPassword}`);
|
||||
if (!hasEnteredPassword) {
|
||||
browser.destroy();
|
||||
reject(new Error("Failed to enter password"));
|
||||
}
|
||||
}
|
||||
|
|
@ -205,6 +205,7 @@ const obtainReferenceManagerToken = async (login, password) => {
|
|||
const cookies = cookieSandbox.getCookiesForURI(
|
||||
Services.io.newURI("https://www.mendeley.com/reference-manager/library")
|
||||
);
|
||||
browser.destroy();
|
||||
if (!cookies.accessToken) {
|
||||
reject(new Error("Failed to obtain Mendeley access token"));
|
||||
}
|
||||
|
|
@ -218,6 +219,7 @@ const obtainReferenceManagerToken = async (login, password) => {
|
|||
|
||||
browser.load("https://www.mendeley.com/sign-in?routeTo=https://www.mendeley.com/reference-manager/library/");
|
||||
Zotero.Promise.delay(ACCESS_TOKEN_TIMEOUT).then(() => {
|
||||
browser.destroy();
|
||||
reject(new Error("Timed out while obtaining Mendeley access token"));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ var Zotero_Import_Mendeley = function () {
|
|||
|
||||
this._tokens = null;
|
||||
this._credentials = null;
|
||||
this._refManagerToken = null;
|
||||
this._db = null;
|
||||
this._file = null;
|
||||
this._saveOptions = null;
|
||||
|
|
@ -327,15 +328,18 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
|
|||
}
|
||||
|
||||
if (this._credentials && !this.skipNotebooks) {
|
||||
const token = await obtainReferenceManagerTokenWithRetry(this._credentials.username, this._credentials.password);
|
||||
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 4 more left
|
||||
if (token) {
|
||||
if (!this._refManagerToken) {
|
||||
const token = await obtainReferenceManagerTokenWithRetry(this._credentials.username, this._credentials.password);
|
||||
this._refManagerToken = {
|
||||
kind: 'referenceManager',
|
||||
accessToken: token,
|
||||
username: this._credentials.username,
|
||||
password: this._credentials.password
|
||||
};
|
||||
}
|
||||
|
||||
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 4 more left
|
||||
if (this._refManagerToken) {
|
||||
const notebooks = await this._getNotebooksAPI();
|
||||
const notesContent = await Promise.all(notebooks.map(notebook => this._translateNotebookToNoteContent(libraryID, notebook)));
|
||||
this._progress += 1; // progress one arbitrary "tick" assigned to importing notebooks task, we have 1 more left
|
||||
|
|
@ -353,7 +357,7 @@ Zotero_Import_Mendeley.prototype.translate = async function (options = {}) {
|
|||
|
||||
const isMappedToExisting = !!existingItem;
|
||||
Zotero.debug(isMappedToExisting ? `Updating existing notebook "${uuid}" -> "${existingItem.key}"` : `Importing new notebook "${uuid}"`, 5);
|
||||
let item = existingItem ?? new Zotero.Item('note');
|
||||
let item = isMappedToExisting ? existingItem : new Zotero.Item('note');
|
||||
item.libraryID = libraryID;
|
||||
item.setNote(noteContent);
|
||||
item.addRelation(predicate, uuid);
|
||||
|
|
@ -1085,14 +1089,15 @@ Zotero_Import_Mendeley.prototype._translateNotebookToNoteContent = async functio
|
|||
'mendeleyDB:annotationUUID',
|
||||
annotationUUID
|
||||
);
|
||||
|
||||
let attachmentItem = Zotero.Items.get(annotation.parentID);
|
||||
let jsonAnnotation = await Zotero.Annotations.toJSON(annotation);
|
||||
jsonAnnotation.attachmentItemID = attachmentItem.id;
|
||||
jsonAnnotation.id = annotation.key;
|
||||
if (annotation) {
|
||||
let attachmentItem = Zotero.Items.get(annotation.parentID);
|
||||
let jsonAnnotation = await Zotero.Annotations.toJSON(annotation);
|
||||
jsonAnnotation.attachmentItemID = attachmentItem.id;
|
||||
jsonAnnotation.id = annotation.key;
|
||||
|
||||
const { html } = Zotero.EditorInstanceUtilities.serializeAnnotations([jsonAnnotation]);
|
||||
return html;
|
||||
const { html } = Zotero.EditorInstanceUtilities.serializeAnnotations([jsonAnnotation]);
|
||||
return html;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1744,6 +1749,11 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations,
|
|||
// do not attempt to import annotations for non-PDF files
|
||||
return;
|
||||
}
|
||||
|
||||
// Mendeley produces empty, type-less annotations that confuse the PDF worker
|
||||
annotations = annotations.filter(
|
||||
annotation => ['highlight', 'note', 'image', 'ink', 'underline', 'text'].includes(annotation.type)
|
||||
);
|
||||
|
||||
let annotationMap = new Map();
|
||||
for (let annotation of annotations) {
|
||||
|
|
@ -1766,6 +1776,7 @@ Zotero_Import_Mendeley.prototype._saveAnnotations = async function (annotations,
|
|||
for (let annotation of annotations) {
|
||||
// Ignore empty highlights
|
||||
if (annotation.type == 'highlight' && !annotation.text) {
|
||||
Zotero.debug(`Skipping empty highlight with uuid ${annotation.uuid}`, 5);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,25 +2,43 @@
|
|||
{
|
||||
"id": "339d0202-d99f-48a2-aa0d-9b0c5631af26",
|
||||
"type": "highlight",
|
||||
"color":
|
||||
{
|
||||
"color": {
|
||||
"r": 255,
|
||||
"g": 245,
|
||||
"b": 173
|
||||
},
|
||||
"profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a",
|
||||
"positions":
|
||||
[
|
||||
"positions": [
|
||||
{
|
||||
"top_left":
|
||||
{
|
||||
"x": 429.74257247464135,
|
||||
"y": 770.1238821199208
|
||||
"top_left": {
|
||||
"x": 191.5049,
|
||||
"y": 588.5418
|
||||
},
|
||||
"bottom_right":
|
||||
{
|
||||
"x": 572.5401236420382,
|
||||
"y": 680.1292839565452
|
||||
"bottom_right": {
|
||||
"x": 461.6047999999994,
|
||||
"y": 576.1542
|
||||
},
|
||||
"page": 1
|
||||
},
|
||||
{
|
||||
"top_left": {
|
||||
"x": 146.3,
|
||||
"y": 576.8651
|
||||
},
|
||||
"bottom_right": {
|
||||
"x": 461.3467999999992,
|
||||
"y": 564.6542
|
||||
},
|
||||
"page": 1
|
||||
},
|
||||
{
|
||||
"top_left": {
|
||||
"x": 146.3,
|
||||
"y": 565.3651
|
||||
},
|
||||
"bottom_right": {
|
||||
"x": 220.06760000000003,
|
||||
"y": 553.1542
|
||||
},
|
||||
"page": 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
],
|
||||
"created": "2024-11-04T13:49:03.274Z",
|
||||
"modified": "2024-11-04T13:49:03.274Z",
|
||||
"id": "https://api.mendeley.com/annotations/v2/84f12446-3b49-4052-bbdc-832d28e1e072",
|
||||
"id": "https://api.mendeley.com/annotations/v2/339d0202-d99f-48a2-aa0d-9b0c5631af26",
|
||||
"body": [
|
||||
{
|
||||
"purpose": "highlighting",
|
||||
|
|
|
|||
8
test/tests/data/mendeleyMock/notebooks.json
Normal file
8
test/tests/data/mendeleyMock/notebooks.json
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
[
|
||||
{
|
||||
"id": "8041a43b-f740-41ec-be42-a24cc5106d68",
|
||||
"created": "2024-07-09T09:56:04.349Z",
|
||||
"modified": "2024-12-06T17:41:37.200Z",
|
||||
"title": "TEST"
|
||||
}
|
||||
]
|
||||
|
|
@ -3,10 +3,18 @@
|
|||
describe('Zotero_Import_Mendeley', function () {
|
||||
var server, httpd, httpdURL, importers;
|
||||
|
||||
const getImporter = () => {
|
||||
const getImporter = ({ importNotes = false } = {}) => {
|
||||
const importer = new Zotero_Import_Mendeley();
|
||||
importer.mendeleyAuth = { kind: 'direct', tokens: { accessToken: 'access_token', refreshToken: 'refresh_token' } };
|
||||
importer.skipNotebooks = true;
|
||||
importer.skipNotebooks = !importNotes;
|
||||
if (importNotes) {
|
||||
importer._refManagerToken = {
|
||||
kind: 'referenceManager',
|
||||
accessToken: 'refman_token',
|
||||
username: 'username',
|
||||
password: 'password'
|
||||
};
|
||||
}
|
||||
importers.push(importer);
|
||||
return importer;
|
||||
};
|
||||
|
|
@ -14,7 +22,7 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
before(async () => {
|
||||
Components.utils.import('chrome://zotero/content/import/mendeley/mendeleyImport.js');
|
||||
|
||||
// real http server is used to deliver an empty pdf so that annotations can be processed during import
|
||||
// A real HTTP server is used to deliver a Bitcoin PDF so that annotations can be processed during import.
|
||||
Components.utils.import("resource://zotero-unit/httpd.js");
|
||||
const port = 16213;
|
||||
httpd = new HttpServer();
|
||||
|
|
@ -22,7 +30,7 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
httpd.start(port);
|
||||
httpd.registerFile(
|
||||
'/file1.pdf',
|
||||
Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'empty.pdf'))
|
||||
Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'recognizePDF_test_title.pdf'))
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -84,7 +92,7 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
|
||||
setHTTPResponse(server, 'https://api.mendeley.com/', {
|
||||
method: 'GET',
|
||||
url: `profiles/v2/me?`,
|
||||
url: `profiles/v2/me`,
|
||||
status: 200,
|
||||
headers: {},
|
||||
json: JSON.parse(
|
||||
|
|
@ -104,7 +112,7 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
|
||||
setHTTPResponse(server, 'https://api.mendeley.com/', {
|
||||
method: 'GET',
|
||||
url: `files/19fb5e5b-1a39-4851-b513-d48441a670e1?`,
|
||||
url: `files/19fb5e5b-1a39-4851-b513-d48441a670e1`,
|
||||
status: 200, // ideally would be 303 but mock http doesn't like it
|
||||
headers: {
|
||||
Location: `${httpdURL}/file1.pdf`
|
||||
|
|
@ -526,7 +534,7 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
annotation.annotationText = 'Highlight text';
|
||||
annotation.annotationComment = 'Highlight comment';
|
||||
annotation.annotationPageLabel = '57';
|
||||
annotation.addRelation('mendeleyDB:annotationUUID', '84f12446-3b49-4052-bbdc-832d28e1e072');
|
||||
annotation.addRelation('mendeleyDB:annotationUUID', '339d0202-d99f-48a2-aa0d-9b0c5631af26');
|
||||
await annotation.saveTx();
|
||||
|
||||
let mendeleyNotebook = JSON.parse(
|
||||
|
|
@ -544,5 +552,85 @@ describe('Zotero_Import_Mendeley', function () {
|
|||
await attachment.eraseTx();
|
||||
await item.eraseTx();
|
||||
});
|
||||
|
||||
it('should import a notebook from Mendeley', async () => {
|
||||
const importer = getImporter({ importNotes: true });
|
||||
|
||||
setHTTPResponse(server, 'https://api.mendeley.com/', {
|
||||
method: 'GET',
|
||||
url: `notes/v1?limit=500`,
|
||||
status: 200,
|
||||
headers: {},
|
||||
json: JSON.parse(
|
||||
await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/notebooks.json')
|
||||
)
|
||||
});
|
||||
|
||||
setHTTPResponse(server, 'https://api.mendeley.com/', {
|
||||
method: 'GET',
|
||||
url: 'notes/v1/8041a43b-f740-41ec-be42-a24cc5106d68',
|
||||
status: 200,
|
||||
headers: {},
|
||||
json: JSON.parse(
|
||||
await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/notebook.json')
|
||||
)
|
||||
});
|
||||
|
||||
await importer.translate({
|
||||
libraryID: Zotero.Libraries.userLibraryID,
|
||||
collections: null,
|
||||
linkFiles: false,
|
||||
});
|
||||
|
||||
const note = (await Zotero.Relations
|
||||
.getByPredicateAndObject('item', 'mendeleyDB:notebookUUID', '8041a43b-f740-41ec-be42-a24cc5106d68'))
|
||||
.filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted)
|
||||
.shift();
|
||||
|
||||
assert.match(
|
||||
note.note,
|
||||
/^<h1>TEST<\/h1>\n<p><span class="highlight" data-annotation="((?:(?:%[0-9A-F]{2}|[^<>'" %])+))">“A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution”<\/span> <span class="citation" data-citation="((?:(?:%[0-9A-F]{2}|[^<>'" %])+))">\(<span class="citation-item">“Item with PDF”, 2005, p. 1<\/span>\)<\/span><\/p>\n<p>Lorem Ipsum<\/p>$/i
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle incorrect annotations', async () => {
|
||||
let annotationsResponse = JSON.parse(
|
||||
await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/annotations.json')
|
||||
);
|
||||
delete annotationsResponse.find(a => a.id == '885615a7-170e-4613-af80-0227ea76ae55').type;
|
||||
delete annotationsResponse.find(a => a.id == 'bfbdb972-171d-4b21-8ae6-f156ac9a2b41').color;
|
||||
annotationsResponse.push({ type: 'note', id: 'invalid' });
|
||||
setHTTPResponse(server, 'https://api.mendeley.com/', {
|
||||
method: 'GET',
|
||||
url: `annotations?limit=200`,
|
||||
status: 200,
|
||||
headers: {},
|
||||
json: annotationsResponse
|
||||
});
|
||||
|
||||
const importer = getImporter();
|
||||
await importer.translate({
|
||||
libraryID: Zotero.Libraries.userLibraryID,
|
||||
collections: null,
|
||||
linkFiles: false,
|
||||
});
|
||||
|
||||
const pdf = (await Zotero.Relations
|
||||
.getByPredicateAndObject('item', 'mendeleyDB:fileHash', 'cc22c6611277df346ff8dc7386ba3880b2bafa15'))
|
||||
.filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted)
|
||||
.shift();
|
||||
|
||||
const annotations = await pdf.getAnnotations();
|
||||
// Typeless notes and invalid annotations should be discarded.
|
||||
assert.lengthOf(annotations, 5);
|
||||
assert.lengthOf(await Zotero.Relations.getByPredicateAndObject('item', 'mendeleyDB:annotationUUID', '885615a7-170e-4613-af80-0227ea76ae55'), 0);
|
||||
assert.lengthOf(await Zotero.Relations.getByPredicateAndObject('item', 'mendeleyDB:annotationUUID', 'invalid'), 0);
|
||||
// Colorless note annotations should have a default colour.
|
||||
const colorlessNote = (await Zotero.Relations
|
||||
.getByPredicateAndObject('item', 'mendeleyDB:annotationUUID', 'bfbdb972-171d-4b21-8ae6-f156ac9a2b41'))
|
||||
.filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted)
|
||||
.shift();
|
||||
assert.equal(colorlessNote.annotationColor, '#ffd400');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue