From f10649483e064af98f94a1d77e45659987644b3c Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Tue, 9 Nov 2021 17:16:28 +0100 Subject: [PATCH 1/3] Mendeley Import: Add more tests for the importer Also rephrased a comment in the importer code and renamed tests file to mendeleyImportTest.js for consistency. --- .../zotero/import/mendeley/mendeleyImport.js | 2 +- test/tests/data/mendeleyMock/annotations.json | 132 ++++++++++++++++++ .../tests/data/mendeleyMock/items-simple.json | 45 ++++++ ...endeleyImport.js => mendeleyImportTest.js} | 63 ++++++++- 4 files changed, 235 insertions(+), 7 deletions(-) create mode 100644 test/tests/data/mendeleyMock/annotations.json rename test/tests/{mendeleyImport.js => mendeleyImportTest.js} (70%) diff --git a/chrome/content/zotero/import/mendeley/mendeleyImport.js b/chrome/content/zotero/import/mendeley/mendeleyImport.js index 504706b424..d8ff6303dd 100644 --- a/chrome/content/zotero/import/mendeley/mendeleyImport.js +++ b/chrome/content/zotero/import/mendeley/mendeleyImport.js @@ -725,7 +725,7 @@ Zotero_Import_Mendeley.prototype._getDocumentFilesAPI = async function (document } map.set(doc.id, files); } - // check if enough space available totalSize + // TODO: check if enough space available totalSize await caller.runAll(); return map; }; diff --git a/test/tests/data/mendeleyMock/annotations.json b/test/tests/data/mendeleyMock/annotations.json new file mode 100644 index 0000000000..db436b5fcf --- /dev/null +++ b/test/tests/data/mendeleyMock/annotations.json @@ -0,0 +1,132 @@ +[ + { + "id": "339d0202-d99f-48a2-aa0d-9b0c5631af26", + "type": "highlight", + "color": + { + "r": 255, + "g": 181, + "b": 182 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 429.74257247464135, + "y": 770.1238821199208 + }, + "bottom_right": + { + "x": 572.5401236420382, + "y": 680.1292839565452 + }, + "page": 1 + } + ], + "created": "2021-11-09T10:26:16.728Z", + "last_modified": "2021-11-09T10:26:16.728Z", + "privacy_level": "private", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862" + }, + { + "id": "885615a7-170e-4613-af80-0227ea76ae55", + "type": "highlight", + "color": + { + "r": 255, + "g": 196, + "b": 251 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 69.97863215, + "y": 682.3314708379199 + }, + "bottom_right": + { + "x": 210.78493472000002, + "y": 718.03235188976 + }, + "page": 1 + } + ], + "created": "2021-11-09T10:26:16.713Z", + "last_modified": "2021-11-09T10:26:16.713Z", + "privacy_level": "private", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862" + }, + { + "id": "bfbdb972-171d-4b21-8ae6-f156ac9a2b41", + "type": "sticky_note", + "color": + { + "r": 186, + "g": 226, + "b": 255 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 309.44367084808835, + "y": 662.2221955464859 + }, + "bottom_right": + { + "x": 309.44367084808835, + "y": 662.2221955464859 + }, + "page": 1 + } + ], + "text": "blue note 2\n", + "created": "2021-11-09T10:26:16.643Z", + "last_modified": "2021-11-09T10:26:16.643Z", + "privacy_level": "private", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862" + }, + { + "id": "c436932f-b14b-4580-a649-4587a5cdc2c3", + "type": "sticky_note", + "color": + { + "r": 255, + "g": 222, + "b": 180 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 287.4041774203229, + "y": 662.2221955464859 + }, + "bottom_right": + { + "x": 287.4041774203229, + "y": 662.2221955464859 + }, + "page": 1 + } + ], + "text": "orange note1", + "created": "2021-11-09T10:26:16.640Z", + "last_modified": "2021-11-09T10:26:16.640Z", + "privacy_level": "private", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862" + } +] \ No newline at end of file diff --git a/test/tests/data/mendeleyMock/items-simple.json b/test/tests/data/mendeleyMock/items-simple.json index 79921d0def..997edbe792 100644 --- a/test/tests/data/mendeleyMock/items-simple.json +++ b/test/tests/data/mendeleyMock/items-simple.json @@ -19,6 +19,17 @@ }, "starred": false, "title": "Foo Bar", + "authors": + [ + { + "first_name": "Tom", + "last_name": "Najdek" + }, + { + "first_name": "Lorem", + "last_name": "Ipsum" + } + ], "type": "journal", "folder_uuids": [ "8d2f262d-49b3-4dfc-8968-0bb71bcd92ea" @@ -39,5 +50,39 @@ "title": "Sample Report", "type": "report", "year": 2002 + }, { + "title": "Item with PDF", + "type": "journal", + "year": 2005, + "source": "Zotero", + "pages": "1-11", + "websites": + [ + "https://zotero.org" + ], + "id": "c54b0c6f-c4ce-4706-8742-bc7d032df862", + "created": "2021-11-09T10:26:15.201Z", + "file_attached": true, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "last_modified": "2021-11-09T10:26:16.303Z", + "read": false, + "starred": false, + "authored": false, + "confirmed": true, + "hidden": false, + "private_publication": false, + "abstract": "Lorem Ipsum. Nostrud elit ullamco laborum cillum.", + "files": + [ + { + "id": "19fb5e5b-1a39-4851-b513-d48441a670e1", + "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862", + "mime_type": "application/pdf", + "file_name": "item.pdf", + "size": 123456, + "created": "2021-11-09T10:26:16.292Z", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15" + } + ] } ] \ No newline at end of file diff --git a/test/tests/mendeleyImport.js b/test/tests/mendeleyImportTest.js similarity index 70% rename from test/tests/mendeleyImport.js rename to test/tests/mendeleyImportTest.js index 6563899812..c92798f04e 100644 --- a/test/tests/mendeleyImport.js +++ b/test/tests/mendeleyImportTest.js @@ -1,12 +1,27 @@ -/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false */ +/* global setHTTPResponse:false, sinon: false, Zotero_Import_Mendeley: false, HttpServer: false */ describe('Zotero_Import_Mendeley', function () { - var server, importer; + var server, importer, httpd, httpdURL; before(async () => { Components.utils.import('chrome://zotero/content/import/mendeley/mendeleyImport.js'); importer = new Zotero_Import_Mendeley(); importer.mendeleyCode = 'CODE'; + + // real http server is used to deliver an empty pdf so that annotations can be processed during import + Components.utils.import("resource://zotero-unit/httpd.js"); + const port = 16213; + httpd = new HttpServer(); + httpdURL = `http://127.0.0.1:${port}`; + httpd.start(port); + httpd.registerFile( + '/file1.pdf', + Zotero.File.pathToFile(OS.Path.join(getTestDataDirectory().path, 'empty.pdf')) + ); + }); + + after(async () => { + await new Zotero.Promise(resolve => httpd.stop(resolve)); }); beforeEach(async () => { @@ -43,7 +58,9 @@ describe('Zotero_Import_Mendeley', function () { url: `annotations?limit=200`, status: 200, headers: {}, - json: [] + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/annotations.json') + ) }); setHTTPResponse(server, 'https://api.mendeley.com/', { @@ -73,6 +90,16 @@ describe('Zotero_Import_Mendeley', function () { headers: {}, json: [] }); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `files/19fb5e5b-1a39-4851-b513-d48441a670e1?`, + status: 200, // ideally would be 303 but mock http doesn't like it + headers: { + Location: `${httpdURL}/file1.pdf` + }, + text: '' + }); }); afterEach(() => { @@ -81,7 +108,6 @@ describe('Zotero_Import_Mendeley', function () { describe('#import', () => { it("should import items & collections", async () => { - await importer.translate({ libraryID: Zotero.Libraries.userLibraryID, collections: null, @@ -98,16 +124,41 @@ describe('Zotero_Import_Mendeley', function () { .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) .shift(); + const withpdf = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:documentUUID', 'c54b0c6f-c4ce-4706-8742-bc7d032df862')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + + const pdf = (await Zotero.Relations + .getByPredicateAndObject('item', 'mendeleyDB:fileHash', 'cc22c6611277df346ff8dc7386ba3880b2bafa15')) + .filter(item => item.libraryID == Zotero.Libraries.userLibraryID && !item.deleted) + .shift(); + assert.equal(journal.getField('title'), 'Foo Bar'); assert.equal(journal.itemTypeID, Zotero.ItemTypes.getID('journalArticle')); assert.equal(report.getField('title'), 'Sample Report'); assert.equal(report.itemTypeID, Zotero.ItemTypes.getID('report')); + assert.equal(withpdf.getField('title'), 'Item with PDF'); + assert.equal(withpdf.itemTypeID, Zotero.ItemTypes.getID('journalArticle')); + + // creators + const creators = journal.getCreators(); + assert.lengthOf(creators, 2); + assert.sameMembers(creators.map(c => c.firstName), ["Tom", "Lorem"]); + assert.sameMembers(creators.map(c => c.lastName), ["Najdek", "Ipsum"]); - // test identifiers + // identifiers assert.equal(journal.getField('DOI'), '10.1111'); assert.include(journal.getField('extra'), 'PMID: 11111111'); assert.include(journal.getField('extra'), 'arXiv: 1111.2222'); - + + // attachment & annotations + assert.lengthOf(withpdf.getAttachments(), 1); + assert.equal(pdf.parentID, withpdf.id); + const annotations = await pdf.getAnnotations(); + assert.lengthOf(annotations, 4); + + // collection const parentCollection = await Zotero.Collections.getAsync( journal.getCollections().pop() ); From 7664fedf7002486066036699d23a221825db1cec Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Wed, 10 Nov 2021 13:46:02 +0100 Subject: [PATCH 2/3] Mendeley Import: Test skipping mismatched annotations --- test/tests/data/mendeleyMock/annotations.json | 33 +++++++++++++++++++ test/tests/mendeleyImportTest.js | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/test/tests/data/mendeleyMock/annotations.json b/test/tests/data/mendeleyMock/annotations.json index db436b5fcf..0c5b518df5 100644 --- a/test/tests/data/mendeleyMock/annotations.json +++ b/test/tests/data/mendeleyMock/annotations.json @@ -128,5 +128,38 @@ "privacy_level": "private", "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", "document_id": "c54b0c6f-c4ce-4706-8742-bc7d032df862" + }, + { + "id": "e88bd089-2023-4544-a0bb-a21cb3990037", + "type": "sticky_note", + "color": + { + "r": 255, + "g": 222, + "b": 180 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 287.4041774203229, + "y": 662.2221955464859 + }, + "bottom_right": + { + "x": 287.4041774203229, + "y": 662.2221955464859 + }, + "page": 1 + } + ], + "text": "mismatched note", + "created": "2021-11-09T10:26:16.640Z", + "last_modified": "2021-11-09T10:26:16.640Z", + "privacy_level": "private", + "filehash": "thishashdoesntmatchanythingthishashdoesn", + "document_id": "d0e01b30-ea60-4eae-818a-34ae7cd8bfb8" } ] \ No newline at end of file diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index c92798f04e..dd6e9b9777 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -156,7 +156,7 @@ describe('Zotero_Import_Mendeley', function () { assert.lengthOf(withpdf.getAttachments(), 1); assert.equal(pdf.parentID, withpdf.id); const annotations = await pdf.getAnnotations(); - assert.lengthOf(annotations, 4); + assert.equal(annotations.length, 4); // collection const parentCollection = await Zotero.Collections.getAsync( From 092459dbfc41c23948f2bc653d63ed48feb69a1b Mon Sep 17 00:00:00 2001 From: Tom Najdek Date: Wed, 10 Nov 2021 14:59:37 +0100 Subject: [PATCH 3/3] Mendeley Import: Tests for group annotations Extended Mendeley Import test to include a scenario where other users attached an annotation to an item in a group library that also exists in user's library. --- .../data/mendeleyMock/group-annotations.json | 68 +++++++++++++++++ test/tests/data/mendeleyMock/groups.json | 74 +++++++++++++++++++ test/tests/data/mendeleyMock/user.json | 2 +- test/tests/mendeleyImportTest.js | 36 ++++++++- 4 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 test/tests/data/mendeleyMock/group-annotations.json create mode 100644 test/tests/data/mendeleyMock/groups.json diff --git a/test/tests/data/mendeleyMock/group-annotations.json b/test/tests/data/mendeleyMock/group-annotations.json new file mode 100644 index 0000000000..0037c95610 --- /dev/null +++ b/test/tests/data/mendeleyMock/group-annotations.json @@ -0,0 +1,68 @@ +[ + { + "id": "656fd591-451a-4bb0-8d5f-30c36c135fc9", + "type": "sticky_note", + "color": + { + "r": 211, + "g": 194, + "b": 255 + }, + "profile_id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", + "positions": + [ + { + "top_left": + { + "x": 232.5055339011597, + "y": 744.3609065973726 + }, + "bottom_right": + { + "x": 232.5055339011597, + "y": 744.3609065973726 + }, + "page": 1 + } + ], + "text": "note by me", + "created": "2021-04-22T12:09:51.125Z", + "last_modified": "2021-04-22T12:09:51.125Z", + "privacy_level": "group", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "15edb87e-8526-4e27-af5e-03ada85e447a" + }, + { + "id": "c781467c-31eb-4522-8676-489aea8975e9", + "type": "sticky_note", + "color": + { + "r": 255, + "g": 245, + "b": 173 + }, + "profile_id": "1beef079-1488-3b20-ba55-12b1ae854506", + "positions": + [ + { + "top_left": + { + "x": 167.22474856840384, + "y": 678.5074827967856 + }, + "bottom_right": + { + "x": 167.22474856840384, + "y": 678.5074827967856 + }, + "page": 1 + } + ], + "text": "note by other", + "created": "2021-04-22T12:03:07.293Z", + "last_modified": "2021-04-27T08:51:05.373Z", + "privacy_level": "group", + "filehash": "cc22c6611277df346ff8dc7386ba3880b2bafa15", + "document_id": "15edb87e-8526-4e27-af5e-03ada85e447a" + } +] \ No newline at end of file diff --git a/test/tests/data/mendeleyMock/groups.json b/test/tests/data/mendeleyMock/groups.json new file mode 100644 index 0000000000..7cbbb88fbf --- /dev/null +++ b/test/tests/data/mendeleyMock/groups.json @@ -0,0 +1,74 @@ +[ + { + "name": "group 1", + "description": "group 1 is owned by me", + "disciplines": + [], + "tags": + [], + "photo": + { + "standard": "", + "square": "" + }, + "id": "ec66aee6-455c-300c-b601-ba4d6a34a95e", + "created": "2021-04-22T11:41:43.000Z", + "owning_profile_id": "97d3193e-2d2a-4700-abd6-d82335ef551d", + "link": "", + "role": "owner", + "access_level": "private", + "used_space": 1234, + "member_count": 3, + "member_limit": 25, + "url_slug": "", + "pending_invitation_count": 0 + }, + { + "name": "group 2", + "description": "group 2 is also owned by me", + "disciplines": + [], + "tags": + [], + "photo": + { + "standard": "", + "square": "" + }, + "id": "cc697d28-054c-37d2-afa3-74fa4cf8a727", + "created": "2021-04-22T13:40:54.000Z", + "owning_profile_id": "97d3193e-2d2a-4700-abd6-d82335ef551d", + "link": "", + "role": "owner", + "access_level": "private", + "used_space": 0, + "member_count": 1, + "member_limit": 25, + "url_slug": "", + "pending_invitation_count": 0 + }, + { + "name": "group 3", + "description": "group 3 is owned by someone else", + "disciplines": + [], + "tags": + [], + "photo": + { + "standard": "", + "square": "" + }, + "id": "6a15e9d6-c7e6-3716-8834-7a67d6f5f91f", + "created": "2021-04-22T13:38:17.000Z", + "owning_profile_id": "18b8c872-0e57-36e2-912a-40325fe8c542", + "link": "", + "role": "normal", + "access_level": "private", + "used_space": 0, + "member_count": 2, + "member_limit": 25, + "url_slug": "", + "pending_invitation_count": 0 + } +] \ No newline at end of file diff --git a/test/tests/data/mendeleyMock/user.json b/test/tests/data/mendeleyMock/user.json index e0d488b85f..881bf9c54c 100644 --- a/test/tests/data/mendeleyMock/user.json +++ b/test/tests/data/mendeleyMock/user.json @@ -12,7 +12,7 @@ "display_name": "some name", "email": "zotero@zotero.com", "first_name": "some", - "id": "97d3193e-2d2a-4700-abd6-d82335ef551d", + "id": "8dbf0832-8723-4c48-b532-20c0b7f6e01a", "last_name": "name", "marketing": true, "member_type": "free", diff --git a/test/tests/mendeleyImportTest.js b/test/tests/mendeleyImportTest.js index dd6e9b9777..796a34cd2e 100644 --- a/test/tests/mendeleyImportTest.js +++ b/test/tests/mendeleyImportTest.js @@ -88,7 +88,9 @@ describe('Zotero_Import_Mendeley', function () { url: `groups/v2?type=all&limit=500`, status: 200, headers: {}, - json: [] + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/groups.json') + ) }); setHTTPResponse(server, 'https://api.mendeley.com/', { @@ -100,6 +102,29 @@ describe('Zotero_Import_Mendeley', function () { }, text: '' }); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `annotations?group_id=ec66aee6-455c-300c-b601-ba4d6a34a95e&limit=200`, + status: 200, + json: JSON.parse( + await Zotero.File.getContentsFromURLAsync('resource://zotero-unit-tests/data/mendeleyMock/group-annotations.json') + ) + }); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `annotations?group_id=cc697d28-054c-37d2-afa3-74fa4cf8a727&limit=200`, + status: 200, + json: [] + }); + + setHTTPResponse(server, 'https://api.mendeley.com/', { + method: 'GET', + url: `annotations?group_id=6a15e9d6-c7e6-3716-8834-7a67d6f5f91f&limit=200`, + status: 200, + json: [] + }); }); afterEach(() => { @@ -107,7 +132,7 @@ describe('Zotero_Import_Mendeley', function () { }); describe('#import', () => { - it("should import items & collections", async () => { + it("should import collections, items, attachments & annotations", async () => { await importer.translate({ libraryID: Zotero.Libraries.userLibraryID, collections: null, @@ -156,7 +181,12 @@ describe('Zotero_Import_Mendeley', function () { assert.lengthOf(withpdf.getAttachments(), 1); assert.equal(pdf.parentID, withpdf.id); const annotations = await pdf.getAnnotations(); - assert.equal(annotations.length, 4); + assert.equal(annotations.length, 5); + assert.isTrue(annotations.some(a => a.annotationComment === 'blue note 2')); + assert.isTrue(annotations.some(a => a.annotationComment === 'orange note1')); + assert.isTrue(annotations.some(a => a.annotationComment === 'note by me')); + assert.isFalse(annotations.some(a => a.annotationComment === 'note by other')); + assert.isFalse(annotations.some(a => a.annotationComment === 'mismatched note')); // collection const parentCollection = await Zotero.Collections.getAsync(