From 1d9fbaa0a22a36ab4120b84f9e5a6519b3af9cf4 Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 19:12:01 +0800 Subject: [PATCH 01/19] Add connector duplicate lookup endpoint --- .../zotero/xpcom/server/server_connector.js | 141 ++++++++++++++++++ test/tests/server_connectorTest.js | 50 +++++++ 2 files changed, 191 insertions(+) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 19783ecb0e..17897275f7 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -287,6 +287,147 @@ Zotero.Server.Connector.Detect.prototype = { }, } +Zotero.Server.Connector.FindExistingItems = function () {}; +Zotero.Server.Endpoints["/connector/findExistingItems"] = Zotero.Server.Connector.FindExistingItems; +Zotero.Server.Connector.FindExistingItems.prototype = { + supportedMethods: ["POST"], + supportedDataTypes: ["application/json"], + permitBookmarklet: true, + + init: async function (requestData) { + let data = requestData.data || {}; + let identifiers = Zotero.Server.Connector._getItemIdentifiers(data); + if (!identifiers.doi.length && !identifiers.url.length) { + return [400, "application/json", JSON.stringify({ error: "IDENTIFIERS_NOT_PROVIDED" })]; + } + + let { library } = Zotero.Server.Connector.getSaveTarget(); + let matches = await Zotero.Server.Connector.findExistingItemsByIdentifiers( + identifiers, + library.libraryID + ); + return [200, "application/json", JSON.stringify({ matches })]; + } +}; + +Zotero.Server.Connector._getItemIdentifiers = function (data) { + let identifiers = { + doi: new Set(), + url: new Set() + }; + + let addDOI = (doi) => { + doi = doi && Zotero.Utilities.cleanDOI(doi); + if (doi) { + identifiers.doi.add(doi.toLowerCase()); + } + }; + let addURL = (url) => { + if (url && typeof url == 'string') { + identifiers.url.add(url); + } + }; + + if (data.identifiers) { + for (let doi of data.identifiers.doi || []) { + addDOI(doi); + } + for (let url of data.identifiers.url || []) { + addURL(url); + } + } + + for (let item of data.items || []) { + addDOI(item.DOI); + addURL(item.url); + } + + return { + doi: Array.from(identifiers.doi), + url: Array.from(identifiers.url) + }; +}; + +Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identifiers, libraryID) { + let itemIDs = new Set(); + let doiSet = new Set(identifiers.doi); + let urlSet = new Set(identifiers.url); + let matches = []; + + if (doiSet.size) { + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + + "JOIN itemDataValues USING (valueID) " + + "WHERE libraryID=? AND fieldID=? AND value LIKE ? " + + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", + [ + libraryID, + Zotero.ItemFields.getID('DOI'), + '10.%' + ] + ); + for (let row of rows) { + let doi = Zotero.Utilities.cleanDOI(row.value); + if (doi && doiSet.has(doi.toLowerCase())) { + itemIDs.add(row.itemID); + } + } + } + if (urlSet.size) { + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + + "JOIN itemDataValues USING (valueID) " + + "WHERE libraryID=? AND fieldID=? " + + "AND value IN (" + identifiers.url.map(() => '?').join(',') + ") " + + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", + [ + libraryID, + Zotero.ItemFields.getID('url'), + ...identifiers.url + ] + ); + for (let row of rows) { + if (urlSet.has(row.value)) { + itemIDs.add(row.itemID); + } + } + } + + for (let itemID of itemIDs) { + let item = Zotero.Items.get(itemID); + if (!item || !item.isTopLevelItem()) { + continue; + } + + let matchedFields = []; + let matchedIdentifiers = {}; + let itemDOI = Zotero.Utilities.cleanDOI(item.getField('DOI')); + if (itemDOI && doiSet.has(itemDOI.toLowerCase())) { + matchedFields.push('DOI'); + matchedIdentifiers.doi = itemDOI; + } + let itemURL = item.getField('url'); + if (itemURL && urlSet.has(itemURL)) { + matchedFields.push('url'); + matchedIdentifiers.url = itemURL; + } + if (!matchedFields.length) { + continue; + } + + matches.push({ + id: item.id, + key: item.key, + libraryID: item.libraryID, + title: item.getField('title'), + matchedFields, + matchedIdentifiers + }); + } + + return matches; +}; + /** * Saves items to DB * diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 54d80d02b0..9f48e8281c 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -320,6 +320,56 @@ describe("Connector Server", function () { }); }); + describe("/connector/findExistingItems", function () { + it("should return matching top-level items for DOI and URL in the selected target library", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Article"); + item.setField("DOI", "10.1234/Example.1"); + item.setField("url", "https://example.com/article"); + await item.saveTx(); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + doi: ["10.1234/example.1"], + url: ["https://example.com/article"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].libraryID, Zotero.Libraries.userLibraryID); + assert.sameMembers(data.matches[0].matchedFields, ["DOI", "url"]); + }); + + it("should reject requests without identifiers", async function () { + let error = await getPromiseError(httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({}) + } + )); + assert.instanceOf(error, Zotero.HTTP.UnexpectedStatusException); + assert.include(error.message, "400"); + }); + }); + describe("/connector/saveSingleFile", function () { it("should save a webpage item with /saveSnapshot", async function () { var collection = await createDataObject('collection'); From ae2db1bef58a6d80d019061b3095e6fc7accd04d Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 19:55:31 +0800 Subject: [PATCH 02/19] Respect requested target in duplicate lookup --- .../zotero/xpcom/server/server_connector.js | 4 +- test/tests/server_connectorTest.js | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 17897275f7..6a36732eb3 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -301,7 +301,9 @@ Zotero.Server.Connector.FindExistingItems.prototype = { return [400, "application/json", JSON.stringify({ error: "IDENTIFIERS_NOT_PROVIDED" })]; } - let { library } = Zotero.Server.Connector.getSaveTarget(); + let { library } = data.target + ? Zotero.Server.Connector.resolveTarget(data.target) + : Zotero.Server.Connector.getSaveTarget(); let matches = await Zotero.Server.Connector.findExistingItemsByIdentifiers( identifiers, library.libraryID diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 9f48e8281c..9fea3f506f 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -354,6 +354,44 @@ describe("Connector Server", function () { assert.sameMembers(data.matches[0].matchedFields, ["DOI", "url"]); }); + it("should use the requested target library for duplicate lookup", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let userItem = new Zotero.Item("journalArticle"); + userItem.setField("title", "Existing User Article"); + userItem.setField("DOI", "10.1234/targeted"); + await userItem.saveTx(); + + let group = await createGroup({ editable: true }); + let groupItem = new Zotero.Item("journalArticle"); + groupItem.libraryID = group.libraryID; + groupItem.setField("title", "Existing Group Article"); + groupItem.setField("DOI", "10.1234/targeted"); + await groupItem.saveTx(); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + target: group.treeViewID, + identifiers: { + doi: ["10.1234/targeted"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, groupItem.id); + assert.equal(data.matches[0].libraryID, group.libraryID); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 02fc179c71208b649b2766b690b1b915945215eb Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:01:42 +0800 Subject: [PATCH 03/19] Load duplicate lookup matches asynchronously --- chrome/content/zotero/xpcom/server/server_connector.js | 2 +- test/tests/server_connectorTest.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 6a36732eb3..8b5adf2e17 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -396,7 +396,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif } for (let itemID of itemIDs) { - let item = Zotero.Items.get(itemID); + let item = await Zotero.Items.getAsync(itemID); if (!item || !item.isTopLevelItem()) { continue; } diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 9fea3f506f..c8fd41daf4 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -330,6 +330,7 @@ describe("Connector Server", function () { item.setField("DOI", "10.1234/Example.1"); item.setField("url", "https://example.com/article"); await item.saveTx(); + Zotero.Items.unload(item.id); let response = await httpRequest( "POST", @@ -369,6 +370,7 @@ describe("Connector Server", function () { groupItem.setField("title", "Existing Group Article"); groupItem.setField("DOI", "10.1234/targeted"); await groupItem.saveTx(); + Zotero.Items.unload([userItem.id, groupItem.id]); let response = await httpRequest( "POST", From cdf7690f3c2787c22562db7e0e6750e10af3bc51 Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:08:02 +0800 Subject: [PATCH 04/19] Deproxify URLs for duplicate lookup --- .../zotero/xpcom/server/server_connector.js | 4 +++ test/tests/server_connectorTest.js | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 8b5adf2e17..63fac9d859 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -317,6 +317,7 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { doi: new Set(), url: new Set() }; + let proxy = data.proxy && new Zotero.Proxy(data.proxy); let addDOI = (doi) => { doi = doi && Zotero.Utilities.cleanDOI(doi); @@ -325,6 +326,9 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { } }; let addURL = (url) => { + if (proxy && url) { + url = proxy.toProper(url); + } if (url && typeof url == 'string') { identifiers.url.add(url); } diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index c8fd41daf4..c528566c01 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -394,6 +394,42 @@ describe("Connector Server", function () { assert.equal(data.matches[0].libraryID, group.libraryID); }); + it("should deproxify item URLs before duplicate lookup", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("webpage"); + item.setField("title", "Existing Proxied Page"); + item.setField("url", "https://www.example.com/path"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + proxy: { + scheme: "https://%h.proxy.example.com/%p" + }, + items: [{ + itemType: "webpage", + title: "New Proxied Page", + url: "https://www-example-com.proxy.example.com/path" + }] + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedIdentifiers.url, "https://www.example.com/path"); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 8f3726d16d32ebf7c353bd1a83981071929bb0b1 Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:19:48 +0800 Subject: [PATCH 05/19] Chunk duplicate URL lookup queries --- .../zotero/xpcom/server/server_connector.js | 38 +++++++++++-------- test/tests/server_connectorTest.js | 34 +++++++++++++++++ 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 63fac9d859..8ea2e04290 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -380,23 +380,29 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif } } if (urlSet.size) { - let rows = await Zotero.DB.queryAsync( - "SELECT itemID, value FROM items JOIN itemData USING (itemID) " - + "JOIN itemDataValues USING (valueID) " - + "WHERE libraryID=? AND fieldID=? " - + "AND value IN (" + identifiers.url.map(() => '?').join(',') + ") " - + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", - [ - libraryID, - Zotero.ItemFields.getID('url'), - ...identifiers.url - ] - ); - for (let row of rows) { - if (urlSet.has(row.value)) { - itemIDs.add(row.itemID); + await Zotero.Utilities.Internal.forEachChunkAsync( + identifiers.url, + Zotero.DB.MAX_BOUND_PARAMETERS - 2, + async function (chunk) { + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + + "JOIN itemDataValues USING (valueID) " + + "WHERE libraryID=? AND fieldID=? " + + "AND value IN (" + chunk.map(() => '?').join(',') + ") " + + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", + [ + libraryID, + Zotero.ItemFields.getID('url'), + ...chunk + ] + ); + for (let row of rows) { + if (urlSet.has(row.value)) { + itemIDs.add(row.itemID); + } + } } - } + ); } for (let itemID of itemIDs) { diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index c528566c01..d3f6e312d9 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -430,6 +430,40 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.url, "https://www.example.com/path"); }); + it("should chunk large URL duplicate lookups", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("webpage"); + item.setField("title", "Existing Large Lookup Page"); + item.setField("url", "https://example.com/target-large-lookup"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let urls = []; + for (let i = 0; i < Zotero.DB.MAX_BOUND_PARAMETERS + 5; i++) { + urls.push(`https://example.com/not-${i}`); + } + urls.push("https://example.com/target-large-lookup"); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { url: urls } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 7b04079322199a1984672528fd4cfee8fd940731 Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:24:27 +0800 Subject: [PATCH 06/19] Extract duplicate DOI values from Extra --- .../zotero/xpcom/server/server_connector.js | 4 +++ test/tests/server_connectorTest.js | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 8ea2e04290..7cec553f16 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -345,6 +345,10 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { for (let item of data.items || []) { addDOI(item.DOI); + if (item.extra) { + let { fields } = Zotero.Utilities.Internal.extractExtraFields(item.extra); + addDOI(fields.get('DOI')); + } addURL(item.url); } diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index d3f6e312d9..c9c495ba5a 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -464,6 +464,39 @@ describe("Connector Server", function () { assert.equal(data.matches[0].id, item.id); }); + it("should extract DOI from translated item Extra before duplicate lookup", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Extra DOI Article"); + item.setField("DOI", "10.1234/extra-doi"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + items: [{ + itemType: "journalArticle", + title: "New Extra DOI Article", + extra: "DOI: 10.1234/extra-doi" + }] + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/extra-doi"); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 0e044b0ea8f978a9a4601a000ff114dc8f6609ea Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:31:20 +0800 Subject: [PATCH 07/19] Handle prefixed DOI values in duplicate lookup --- .../zotero/xpcom/server/server_connector.js | 5 ++- test/tests/server_connectorTest.js | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 7cec553f16..66a5002294 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -368,12 +368,11 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let rows = await Zotero.DB.queryAsync( "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + "JOIN itemDataValues USING (valueID) " - + "WHERE libraryID=? AND fieldID=? AND value LIKE ? " + + "WHERE libraryID=? AND fieldID=? " + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", [ libraryID, - Zotero.ItemFields.getID('DOI'), - '10.%' + Zotero.ItemFields.getID('DOI') ] ); for (let row of rows) { diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index c9c495ba5a..05d42dcf1e 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -497,6 +497,37 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/extra-doi"); }); + it("should match saved DOI values with resolver prefixes", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Prefixed DOI Article"); + item.setField("DOI", "https://doi.org/10.1234/prefixed-doi"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + doi: ["10.1234/prefixed-doi"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/prefixed-doi"); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 3c22aef89ca6933d7bb4a21ce47dc85abe80a9e8 Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:44:27 +0800 Subject: [PATCH 08/19] Match duplicate DOI values stored in Extra --- .../zotero/xpcom/server/server_connector.js | 26 ++++++++++++++-- test/tests/server_connectorTest.js | 31 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 66a5002294..54e3f47b35 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -365,7 +365,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let matches = []; if (doiSet.size) { - let rows = await Zotero.DB.queryAsync( + let doiRows = await Zotero.DB.queryAsync( "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + "JOIN itemDataValues USING (valueID) " + "WHERE libraryID=? AND fieldID=? " @@ -375,12 +375,29 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif Zotero.ItemFields.getID('DOI') ] ); - for (let row of rows) { + for (let row of doiRows) { let doi = Zotero.Utilities.cleanDOI(row.value); if (doi && doiSet.has(doi.toLowerCase())) { itemIDs.add(row.itemID); } } + let extraRows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + + "JOIN itemDataValues USING (valueID) " + + "WHERE libraryID=? AND fieldID=? " + + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", + [ + libraryID, + Zotero.ItemFields.getID('extra') + ] + ); + for (let row of extraRows) { + let { fields } = Zotero.Utilities.Internal.extractExtraFields(row.value); + let doi = Zotero.Utilities.cleanDOI(fields.get('DOI')); + if (doi && doiSet.has(doi.toLowerCase())) { + itemIDs.add(row.itemID); + } + } } if (urlSet.size) { await Zotero.Utilities.Internal.forEachChunkAsync( @@ -416,7 +433,10 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let matchedFields = []; let matchedIdentifiers = {}; - let itemDOI = Zotero.Utilities.cleanDOI(item.getField('DOI')); + let itemDOI = [ + item.getField('DOI'), + item.getExtraField('DOI') + ].map(doi => Zotero.Utilities.cleanDOI(doi)).find(doi => doi && doiSet.has(doi.toLowerCase())); if (itemDOI && doiSet.has(itemDOI.toLowerCase())) { matchedFields.push('DOI'); matchedIdentifiers.doi = itemDOI; diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 05d42dcf1e..110ba283da 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -528,6 +528,37 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/prefixed-doi"); }); + it("should match existing item DOI values stored in Extra", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("webpage"); + item.setField("title", "Existing Extra DOI Page"); + item.setField("extra", "DOI: 10.1234/existing-extra-doi"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + doi: ["10.1234/existing-extra-doi"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/existing-extra-doi"); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From 110cf3156e3ff71a6901dd0d4139d3bb2046284a Mon Sep 17 00:00:00 2001 From: Shallow Date: Mon, 29 Jun 2026 20:53:26 +0800 Subject: [PATCH 09/19] Return display titles for duplicate matches --- .../zotero/xpcom/server/server_connector.js | 2 +- test/tests/server_connectorTest.js | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 54e3f47b35..8d95755d14 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -454,7 +454,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif id: item.id, key: item.key, libraryID: item.libraryID, - title: item.getField('title'), + title: item.getDisplayTitle(), matchedFields, matchedIdentifiers }); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 110ba283da..ef922741d9 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -559,6 +559,37 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/existing-extra-doi"); }); + it("should return display titles for base-mapped title fields", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("legalCase"); + item.setField("caseName", "Existing Case Name"); + item.setField("url", "https://example.com/case"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + url: ["https://example.com/case"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].title, "Existing Case Name"); + }); + it("should reject requests without identifiers", async function () { let error = await getPromiseError(httpRequest( "POST", From c09f11d9ab6db58ad90a87c73574480c3468528d Mon Sep 17 00:00:00 2001 From: Shallow Date: Tue, 30 Jun 2026 13:08:06 +0800 Subject: [PATCH 10/19] Load item data for duplicate lookup matches --- chrome/content/zotero/xpcom/server/server_connector.js | 1 + test/tests/server_connectorTest.js | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 8d95755d14..9d1112208a 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -430,6 +430,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif if (!item || !item.isTopLevelItem()) { continue; } + await item.loadDataType('itemData'); let matchedFields = []; let matchedIdentifiers = {}; diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index ef922741d9..203dc9840b 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -400,7 +400,7 @@ describe("Connector Server", function () { let item = new Zotero.Item("webpage"); item.setField("title", "Existing Proxied Page"); - item.setField("url", "https://www.example.com/path"); + item.setField("url", "https://duplicate.example.com/path"); await item.saveTx(); Zotero.Items.unload(item.id); @@ -418,7 +418,7 @@ describe("Connector Server", function () { items: [{ itemType: "webpage", title: "New Proxied Page", - url: "https://www-example-com.proxy.example.com/path" + url: "https://duplicate-example-com.proxy.example.com/path" }] }) } @@ -427,7 +427,7 @@ describe("Connector Server", function () { let data = JSON.parse(response.response); assert.lengthOf(data.matches, 1); assert.equal(data.matches[0].id, item.id); - assert.equal(data.matches[0].matchedIdentifiers.url, "https://www.example.com/path"); + assert.equal(data.matches[0].matchedIdentifiers.url, "https://duplicate.example.com/path"); }); it("should chunk large URL duplicate lookups", async function () { @@ -563,7 +563,7 @@ describe("Connector Server", function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); - let item = new Zotero.Item("legalCase"); + let item = new Zotero.Item("case"); item.setField("caseName", "Existing Case Name"); item.setField("url", "https://example.com/case"); await item.saveTx(); From a4115bcc86475d562b7cafdf7d96e3bf25e1313f Mon Sep 17 00:00:00 2001 From: Shallow Date: Tue, 30 Jun 2026 14:21:02 +0800 Subject: [PATCH 11/19] Filter DOI duplicate lookup candidates --- .../zotero/xpcom/server/server_connector.js | 46 ++++++------ test/tests/server_connectorTest.js | 72 +++++++++++++++++++ 2 files changed, 98 insertions(+), 20 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 9d1112208a..a5375e2eec 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -365,32 +365,38 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let matches = []; if (doiSet.size) { - let doiRows = await Zotero.DB.queryAsync( - "SELECT itemID, value FROM items JOIN itemData USING (itemID) " - + "JOIN itemDataValues USING (valueID) " - + "WHERE libraryID=? AND fieldID=? " - + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", - [ - libraryID, - Zotero.ItemFields.getID('DOI') - ] - ); + let getDOICandidateRows = async function (fieldID) { + let allRows = []; + await Zotero.Utilities.Internal.forEachChunkAsync( + identifiers.doi, + Zotero.DB.MAX_BOUND_PARAMETERS - 2, + async function (chunk) { + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM items JOIN itemData USING (itemID) " + + "JOIN itemDataValues USING (valueID) " + + "WHERE libraryID=? AND fieldID=? " + + "AND (" + chunk.map(() => "LOWER(value) LIKE ?").join(" OR ") + ") " + + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", + [ + libraryID, + fieldID, + ...chunk.map(doi => `%${doi}%`) + ] + ); + allRows.push(...rows); + } + ); + return allRows; + }; + + let doiRows = await getDOICandidateRows(Zotero.ItemFields.getID('DOI')); for (let row of doiRows) { let doi = Zotero.Utilities.cleanDOI(row.value); if (doi && doiSet.has(doi.toLowerCase())) { itemIDs.add(row.itemID); } } - let extraRows = await Zotero.DB.queryAsync( - "SELECT itemID, value FROM items JOIN itemData USING (itemID) " - + "JOIN itemDataValues USING (valueID) " - + "WHERE libraryID=? AND fieldID=? " - + "AND itemID NOT IN (SELECT itemID FROM deletedItems)", - [ - libraryID, - Zotero.ItemFields.getID('extra') - ] - ); + let extraRows = await getDOICandidateRows(Zotero.ItemFields.getID('extra')); for (let row of extraRows) { let { fields } = Zotero.Utilities.Internal.extractExtraFields(row.value); let doi = Zotero.Utilities.cleanDOI(fields.get('DOI')); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 203dc9840b..298235282a 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -559,6 +559,78 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/existing-extra-doi"); }); + it("should filter DOI lookup candidates before parsing values", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let unrelatedDOIItem = new Zotero.Item("journalArticle"); + unrelatedDOIItem.setField("title", "Unrelated DOI Article"); + unrelatedDOIItem.setField("DOI", "10.9999/unrelated-doi-candidate"); + await unrelatedDOIItem.saveTx(); + + let unrelatedExtraItem = new Zotero.Item("webpage"); + unrelatedExtraItem.setField("title", "Unrelated Extra Page"); + unrelatedExtraItem.setField("extra", "UNRELATED_EXTRA_SHOULD_NOT_PARSE"); + await unrelatedExtraItem.saveTx(); + + let doiItem = new Zotero.Item("journalArticle"); + doiItem.setField("title", "Existing Filtered DOI Article"); + doiItem.setField("DOI", "https://doi.org/10.1234/filtered-doi-candidate"); + await doiItem.saveTx(); + + let extraItem = new Zotero.Item("webpage"); + extraItem.setField("title", "Existing Filtered Extra Page"); + extraItem.setField("extra", "DOI: 10.1234/filtered-extra-candidate"); + await extraItem.saveTx(); + Zotero.Items.unload([unrelatedDOIItem.id, unrelatedExtraItem.id, doiItem.id, extraItem.id]); + + let originalCleanDOI = Zotero.Utilities.cleanDOI; + let originalExtractExtraFields = Zotero.Utilities.Internal.extractExtraFields; + let cleanedUnrelatedDOI = false; + let parsedUnrelatedExtra = false; + sinon.stub(Zotero.Utilities, "cleanDOI").callsFake(function (doi) { + if (doi && doi.includes("unrelated-doi-candidate")) { + cleanedUnrelatedDOI = true; + } + return originalCleanDOI.apply(this, arguments); + }); + sinon.stub(Zotero.Utilities.Internal, "extractExtraFields").callsFake(function (extra) { + if (extra && extra.includes("UNRELATED_EXTRA_SHOULD_NOT_PARSE")) { + parsedUnrelatedExtra = true; + } + return originalExtractExtraFields.apply(this, arguments); + }); + + try { + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + doi: [ + "10.1234/filtered-doi-candidate", + "10.1234/filtered-extra-candidate" + ] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.sameMembers(data.matches.map(match => match.id), [doiItem.id, extraItem.id]); + assert.isFalse(cleanedUnrelatedDOI); + assert.isFalse(parsedUnrelatedExtra); + } + finally { + Zotero.Utilities.cleanDOI.restore(); + Zotero.Utilities.Internal.extractExtraFields.restore(); + } + }); + it("should return display titles for base-mapped title fields", async function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); From f79f79c82c8627c22f459c327fbdafd0f798b05c Mon Sep 17 00:00:00 2001 From: Shallow Date: Thu, 2 Jul 2026 01:53:55 +0800 Subject: [PATCH 12/19] Return matched item indexes for duplicate lookup --- .../zotero/xpcom/server/server_connector.js | 62 ++++++++++++++++--- test/tests/server_connectorTest.js | 1 + 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index a5375e2eec..fe40410eee 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -317,12 +317,15 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { doi: new Set(), url: new Set() }; + let itemIdentifiers = []; let proxy = data.proxy && new Zotero.Proxy(data.proxy); let addDOI = (doi) => { doi = doi && Zotero.Utilities.cleanDOI(doi); if (doi) { - identifiers.doi.add(doi.toLowerCase()); + doi = doi.toLowerCase(); + identifiers.doi.add(doi); + return doi; } }; let addURL = (url) => { @@ -331,6 +334,7 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { } if (url && typeof url == 'string') { identifiers.url.add(url); + return url; } }; @@ -344,17 +348,38 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { } for (let item of data.items || []) { - addDOI(item.DOI); + let itemIdentifier = { + doi: new Set(), + url: new Set() + }; + let addItemDOI = (doi) => { + doi = addDOI(doi); + if (doi) { + itemIdentifier.doi.add(doi); + } + }; + let addItemURL = (url) => { + url = addURL(url); + if (url) { + itemIdentifier.url.add(url); + } + }; + addItemDOI(item.DOI); if (item.extra) { let { fields } = Zotero.Utilities.Internal.extractExtraFields(item.extra); - addDOI(fields.get('DOI')); + addItemDOI(fields.get('DOI')); } - addURL(item.url); + addItemURL(item.url); + itemIdentifiers.push({ + doi: Array.from(itemIdentifier.doi), + url: Array.from(itemIdentifier.url) + }); } return { doi: Array.from(identifiers.doi), - url: Array.from(identifiers.url) + url: Array.from(identifiers.url), + itemIdentifiers }; }; @@ -362,7 +387,25 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let itemIDs = new Set(); let doiSet = new Set(identifiers.doi); let urlSet = new Set(identifiers.url); + let requestItemIdentifiers = identifiers.itemIdentifiers || []; let matches = []; + let getMatchedItemIndex = function (matchedIdentifiers) { + if (!requestItemIdentifiers.length) { + return undefined; + } + let doi = matchedIdentifiers.doi && Zotero.Utilities.cleanDOI(matchedIdentifiers.doi); + doi = doi && doi.toLowerCase(); + for (let i = 0; i < requestItemIdentifiers.length; i++) { + let itemIdentifiers = requestItemIdentifiers[i]; + if (doi && itemIdentifiers.doi.includes(doi)) { + return i; + } + if (matchedIdentifiers.url && itemIdentifiers.url.includes(matchedIdentifiers.url)) { + return i; + } + } + return undefined; + }; if (doiSet.size) { let getDOICandidateRows = async function (fieldID) { @@ -457,14 +500,19 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif continue; } - matches.push({ + let match = { id: item.id, key: item.key, libraryID: item.libraryID, title: item.getDisplayTitle(), matchedFields, matchedIdentifiers - }); + }; + let matchedItemIndex = getMatchedItemIndex(matchedIdentifiers); + if (matchedItemIndex !== undefined) { + match.matchedItemIndex = matchedItemIndex; + } + matches.push(match); } return matches; diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 298235282a..57775294a1 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -494,6 +494,7 @@ describe("Connector Server", function () { let data = JSON.parse(response.response); assert.lengthOf(data.matches, 1); assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedItemIndex, 0); assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/extra-doi"); }); From 6df77258c0651e784f201679cc207b7dbfe984da Mon Sep 17 00:00:00 2001 From: Shallow Date: Fri, 3 Jul 2026 20:36:52 +0800 Subject: [PATCH 13/19] Guard duplicate lookup edge cases --- .../zotero/xpcom/server/server_connector.js | 24 +++- test/tests/server_connectorTest.js | 106 ++++++++++++++++++ 2 files changed, 126 insertions(+), 4 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index fe40410eee..0ebfcd7fae 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -301,8 +301,17 @@ Zotero.Server.Connector.FindExistingItems.prototype = { return [400, "application/json", JSON.stringify({ error: "IDENTIFIERS_NOT_PROVIDED" })]; } - let { library } = data.target - ? Zotero.Server.Connector.resolveTarget(data.target) + let target; + if (data.target) { + try { + target = Zotero.Server.Connector.resolveTarget(data.target); + } + catch (e) { + Zotero.debug(`Could not resolve duplicate lookup target '${data.target}': ${e.message}`); + } + } + let { library } = target && target.library + ? target : Zotero.Server.Connector.getSaveTarget(); let matches = await Zotero.Server.Connector.findExistingItemsByIdentifiers( identifiers, @@ -330,7 +339,13 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { }; let addURL = (url) => { if (proxy && url) { - url = proxy.toProper(url); + try { + url = proxy.toProper(url); + } + catch (e) { + Zotero.debug(`Could not deproxify item URL for duplicate lookup: ${e.message}`); + return; + } } if (url && typeof url == 'string') { identifiers.url.add(url); @@ -442,7 +457,8 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let extraRows = await getDOICandidateRows(Zotero.ItemFields.getID('extra')); for (let row of extraRows) { let { fields } = Zotero.Utilities.Internal.extractExtraFields(row.value); - let doi = Zotero.Utilities.cleanDOI(fields.get('DOI')); + let extraDOI = fields.get('DOI'); + let doi = extraDOI && Zotero.Utilities.cleanDOI(extraDOI); if (doi && doiSet.has(doi.toLowerCase())) { itemIDs.add(row.itemID); } diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 57775294a1..1acbaf1909 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -632,6 +632,112 @@ describe("Connector Server", function () { } }); + it("should ignore free-text Extra DOI candidates without failing the lookup", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let freeTextExtraItem = new Zotero.Item("webpage"); + freeTextExtraItem.setField("title", "Free Text Extra Page"); + freeTextExtraItem.setField("extra", "See https://doi.org/10.1234/free-text-extra for related work"); + await freeTextExtraItem.saveTx(); + + let doiItem = new Zotero.Item("journalArticle"); + doiItem.setField("title", "Existing DOI Article"); + doiItem.setField("DOI", "10.1234/free-text-extra"); + await doiItem.saveTx(); + Zotero.Items.unload([freeTextExtraItem.id, doiItem.id]); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { + doi: ["10.1234/free-text-extra"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.sameMembers(data.matches.map(match => match.id), [doiItem.id]); + }); + + it("should keep DOI lookup working when proxied item URL normalization fails", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing DOI With Bad URL Payload"); + item.setField("DOI", "10.1234/bad-proxy-url"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + proxy: { + scheme: "https://%h.proxy.example.com/%p" + }, + items: [{ + itemType: "journalArticle", + title: "New DOI With Bad URL Payload", + DOI: "10.1234/bad-proxy-url", + url: "/relative/path" + }] + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + }); + + it("should fall back to the current save target when requested collection no longer exists", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let collection = await createDataObject('collection'); + let targetID = collection.treeViewID; + await collection.eraseTx(); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing User Library Article"); + item.setField("DOI", "10.1234/deleted-target"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + target: targetID, + identifiers: { + doi: ["10.1234/deleted-target"] + } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].libraryID, Zotero.Libraries.userLibraryID); + }); + it("should return display titles for base-mapped title fields", async function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); From bc733a14a86120de0cb0a85e387bbecdd3840005 Mon Sep 17 00:00:00 2001 From: Shallow Date: Fri, 3 Jul 2026 20:48:48 +0800 Subject: [PATCH 14/19] Chunk DOI duplicate lookup predicates --- .../zotero/xpcom/server/server_connector.js | 3 +- test/tests/server_connectorTest.js | 34 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 0ebfcd7fae..20a4d8d405 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -423,11 +423,12 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif }; if (doiSet.size) { + let doiCandidateChunkSize = Math.min(100, Zotero.DB.MAX_BOUND_PARAMETERS - 2); let getDOICandidateRows = async function (fieldID) { let allRows = []; await Zotero.Utilities.Internal.forEachChunkAsync( identifiers.doi, - Zotero.DB.MAX_BOUND_PARAMETERS - 2, + doiCandidateChunkSize, async function (chunk) { let rows = await Zotero.DB.queryAsync( "SELECT itemID, value FROM items JOIN itemData USING (itemID) " diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index 1acbaf1909..bc059c626a 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -464,6 +464,40 @@ describe("Connector Server", function () { assert.equal(data.matches[0].id, item.id); }); + it("should chunk large DOI duplicate lookups", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Large DOI Lookup Article"); + item.setField("DOI", "10.1234/target-large-doi-lookup"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let dois = []; + for (let i = 0; i < Zotero.DB.MAX_BOUND_PARAMETERS + 5; i++) { + dois.push(`10.1234/not-large-doi-${i}`); + } + dois.push("10.1234/target-large-doi-lookup"); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + identifiers: { doi: dois } + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + }); + it("should extract DOI from translated item Extra before duplicate lookup", async function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); From 5d65c64ea13550cfec0e63c0dedfab9b5f51a711 Mon Sep 17 00:00:00 2001 From: Shallow Date: Sat, 4 Jul 2026 07:09:56 +0800 Subject: [PATCH 15/19] Simplify duplicate lookup DOI match guard The Array.prototype.find() predicate already guarantees a non-empty DOI present in doiSet, so the extra doiSet.has() check in the following if-condition is redundant. --- chrome/content/zotero/xpcom/server/server_connector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 20a4d8d405..019f2aa721 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -504,7 +504,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif item.getField('DOI'), item.getExtraField('DOI') ].map(doi => Zotero.Utilities.cleanDOI(doi)).find(doi => doi && doiSet.has(doi.toLowerCase())); - if (itemDOI && doiSet.has(itemDOI.toLowerCase())) { + if (itemDOI) { matchedFields.push('DOI'); matchedIdentifiers.doi = itemDOI; } From c2da119dc0808ebcc80c4bc7f21ede4a309ef786 Mon Sep 17 00:00:00 2001 From: Shallow Date: Sat, 4 Jul 2026 07:30:33 +0800 Subject: [PATCH 16/19] Only deproxify duplicate lookup URLs when proxy matches Proxy.toProper() normalizes non-matching URLs through new URL().href, which appends a trailing slash to bare host URLs (https://example.com -> https://example.com/). Since the duplicate lookup matches stored URLs by exact equality, a translator-supplied canonical URL that does not actually match the proxy scheme would be looked up under the normalized form and silently miss the existing item. Only call toProper() when the proxy regexp actually matches the URL; otherwise keep the URL as the translator supplied it. --- chrome/content/zotero/xpcom/server/server_connector.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 019f2aa721..322207d6ea 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -340,7 +340,14 @@ Zotero.Server.Connector._getItemIdentifiers = function (data) { let addURL = (url) => { if (proxy && url) { try { - url = proxy.toProper(url); + // Only deproxify when the proxy regexp actually matches. + // toProper() normalizes non-matching URLs through new URL().href, + // which can break exact-match lookups against values stored from + // the translator (e.g. https://example.com vs https://example.com/). + let normalized = new URL(url).href; + if (proxy.regexp.test(normalized)) { + url = proxy.toProper(normalized); + } } catch (e) { Zotero.debug(`Could not deproxify item URL for duplicate lookup: ${e.message}`); From 90165fad01689ab3322412ce6b8595ae1e8e424a Mon Sep 17 00:00:00 2001 From: Shallow Date: Tue, 28 Jul 2026 23:02:37 +0800 Subject: [PATCH 17/19] Return all matched translated item indexes --- .../zotero/xpcom/server/server_connector.js | 20 ++++++---- test/tests/server_connectorTest.js | 40 +++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 322207d6ea..c58c30dae4 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -411,22 +411,24 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let urlSet = new Set(identifiers.url); let requestItemIdentifiers = identifiers.itemIdentifiers || []; let matches = []; - let getMatchedItemIndex = function (matchedIdentifiers) { + let getMatchedItemIndexes = function (matchedIdentifiers) { if (!requestItemIdentifiers.length) { - return undefined; + return []; } let doi = matchedIdentifiers.doi && Zotero.Utilities.cleanDOI(matchedIdentifiers.doi); doi = doi && doi.toLowerCase(); + let indexes = []; for (let i = 0; i < requestItemIdentifiers.length; i++) { let itemIdentifiers = requestItemIdentifiers[i]; if (doi && itemIdentifiers.doi.includes(doi)) { - return i; + indexes.push(i); + continue; } if (matchedIdentifiers.url && itemIdentifiers.url.includes(matchedIdentifiers.url)) { - return i; + indexes.push(i); } } - return undefined; + return indexes; }; if (doiSet.size) { @@ -532,9 +534,11 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif matchedFields, matchedIdentifiers }; - let matchedItemIndex = getMatchedItemIndex(matchedIdentifiers); - if (matchedItemIndex !== undefined) { - match.matchedItemIndex = matchedItemIndex; + let matchedItemIndexes = getMatchedItemIndexes(matchedIdentifiers); + if (matchedItemIndexes.length) { + // Keep the singular field for older Connector versions. + match.matchedItemIndex = matchedItemIndexes[0]; + match.matchedItemIndexes = matchedItemIndexes; } matches.push(match); } diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index bc059c626a..e85bf1175a 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -532,6 +532,46 @@ describe("Connector Server", function () { assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/extra-doi"); }); + it("should return every translated item index sharing an Extra DOI match", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Shared Extra DOI Article"); + item.setField("DOI", "10.1234/shared-extra-doi"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + items: [ + { + itemType: "webpage", + title: "First Shared Extra DOI Page", + extra: "DOI: 10.1234/shared-extra-doi" + }, + { + itemType: "webpage", + title: "Second Shared Extra DOI Page", + extra: "DOI: 10.1234/shared-extra-doi" + } + ] + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].matchedItemIndex, 0); + assert.deepEqual(data.matches[0].matchedItemIndexes, [0, 1]); + }); + it("should match saved DOI values with resolver prefixes", async function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); From 417e8a597ae94f7e358f5e174b47b8d7273e521b Mon Sep 17 00:00:00 2001 From: Shallow Date: Tue, 28 Jul 2026 23:12:54 +0800 Subject: [PATCH 18/19] Include indexes for every matching DOI --- .../zotero/xpcom/server/server_connector.js | 21 +++++----- test/tests/server_connectorTest.js | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index c58c30dae4..6632e352e0 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -411,20 +411,18 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let urlSet = new Set(identifiers.url); let requestItemIdentifiers = identifiers.itemIdentifiers || []; let matches = []; - let getMatchedItemIndexes = function (matchedIdentifiers) { + let getMatchedItemIndexes = function (matchedDOIs, matchedURL) { if (!requestItemIdentifiers.length) { return []; } - let doi = matchedIdentifiers.doi && Zotero.Utilities.cleanDOI(matchedIdentifiers.doi); - doi = doi && doi.toLowerCase(); let indexes = []; for (let i = 0; i < requestItemIdentifiers.length; i++) { let itemIdentifiers = requestItemIdentifiers[i]; - if (doi && itemIdentifiers.doi.includes(doi)) { + if (matchedDOIs.some(doi => itemIdentifiers.doi.includes(doi))) { indexes.push(i); continue; } - if (matchedIdentifiers.url && itemIdentifiers.url.includes(matchedIdentifiers.url)) { + if (matchedURL && itemIdentifiers.url.includes(matchedURL)) { indexes.push(i); } } @@ -509,13 +507,16 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif let matchedFields = []; let matchedIdentifiers = {}; - let itemDOI = [ + let itemDOIs = [ item.getField('DOI'), item.getExtraField('DOI') - ].map(doi => Zotero.Utilities.cleanDOI(doi)).find(doi => doi && doiSet.has(doi.toLowerCase())); - if (itemDOI) { + ] + .map(doi => Zotero.Utilities.cleanDOI(doi)) + .filter(doi => doi && doiSet.has(doi.toLowerCase())); + let matchedDOIs = Array.from(new Set(itemDOIs.map(doi => doi.toLowerCase()))); + if (itemDOIs.length) { matchedFields.push('DOI'); - matchedIdentifiers.doi = itemDOI; + matchedIdentifiers.doi = itemDOIs[0]; } let itemURL = item.getField('url'); if (itemURL && urlSet.has(itemURL)) { @@ -534,7 +535,7 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif matchedFields, matchedIdentifiers }; - let matchedItemIndexes = getMatchedItemIndexes(matchedIdentifiers); + let matchedItemIndexes = getMatchedItemIndexes(matchedDOIs, matchedIdentifiers.url); if (matchedItemIndexes.length) { // Keep the singular field for older Connector versions. match.matchedItemIndex = matchedItemIndexes[0]; diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index e85bf1175a..b1ef3bb5a4 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -572,6 +572,48 @@ describe("Connector Server", function () { assert.deepEqual(data.matches[0].matchedItemIndexes, [0, 1]); }); + it("should return item indexes for every matching DOI stored on one item", async function () { + await selectLibrary(win, Zotero.Libraries.userLibraryID); + await waitForItemsLoad(win); + + let item = new Zotero.Item("journalArticle"); + item.setField("title", "Existing Article with Two DOI Values"); + item.setField("DOI", "10.1234/primary-doi"); + item.setField("extra", "DOI: 10.1234/extra-doi"); + await item.saveTx(); + Zotero.Items.unload(item.id); + + let response = await httpRequest( + "POST", + connectorServerPath + "/connector/findExistingItems", + { + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + items: [ + { + itemType: "journalArticle", + title: "Translated Primary DOI Article", + DOI: "10.1234/primary-doi" + }, + { + itemType: "webpage", + title: "Translated Extra DOI Page", + extra: "DOI: 10.1234/extra-doi" + } + ] + }) + } + ); + + let data = JSON.parse(response.response); + assert.lengthOf(data.matches, 1); + assert.equal(data.matches[0].id, item.id); + assert.equal(data.matches[0].matchedItemIndex, 0); + assert.deepEqual(data.matches[0].matchedItemIndexes, [0, 1]); + }); + it("should match saved DOI values with resolver prefixes", async function () { await selectLibrary(win, Zotero.Libraries.userLibraryID); await waitForItemsLoad(win); From e8a9ce351b45a92ee7931ffc7bf04c3283e1003b Mon Sep 17 00:00:00 2001 From: Shallow Date: Tue, 28 Jul 2026 23:26:09 +0800 Subject: [PATCH 19/19] Keep legacy duplicate index aligned --- .../zotero/xpcom/server/server_connector.js | 14 ++++++++++++-- test/tests/server_connectorTest.js | 13 +++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/chrome/content/zotero/xpcom/server/server_connector.js b/chrome/content/zotero/xpcom/server/server_connector.js index 6632e352e0..46254422ed 100644 --- a/chrome/content/zotero/xpcom/server/server_connector.js +++ b/chrome/content/zotero/xpcom/server/server_connector.js @@ -537,8 +537,18 @@ Zotero.Server.Connector.findExistingItemsByIdentifiers = async function (identif }; let matchedItemIndexes = getMatchedItemIndexes(matchedDOIs, matchedIdentifiers.url); if (matchedItemIndexes.length) { - // Keep the singular field for older Connector versions. - match.matchedItemIndex = matchedItemIndexes[0]; + // Keep the singular field aligned with the identifiers older + // Connector versions receive. + let compatibilityDOIs = matchedIdentifiers.doi + ? [matchedIdentifiers.doi.toLowerCase()] + : []; + let matchedItemIndex = getMatchedItemIndexes( + compatibilityDOIs, + matchedIdentifiers.url + )[0]; + if (matchedItemIndex !== undefined) { + match.matchedItemIndex = matchedItemIndex; + } match.matchedItemIndexes = matchedItemIndexes; } matches.push(match); diff --git a/test/tests/server_connectorTest.js b/test/tests/server_connectorTest.js index b1ef3bb5a4..737afa460e 100644 --- a/test/tests/server_connectorTest.js +++ b/test/tests/server_connectorTest.js @@ -592,15 +592,15 @@ describe("Connector Server", function () { }, body: JSON.stringify({ items: [ - { - itemType: "journalArticle", - title: "Translated Primary DOI Article", - DOI: "10.1234/primary-doi" - }, { itemType: "webpage", title: "Translated Extra DOI Page", extra: "DOI: 10.1234/extra-doi" + }, + { + itemType: "journalArticle", + title: "Translated Primary DOI Article", + DOI: "10.1234/primary-doi" } ] }) @@ -610,7 +610,8 @@ describe("Connector Server", function () { let data = JSON.parse(response.response); assert.lengthOf(data.matches, 1); assert.equal(data.matches[0].id, item.id); - assert.equal(data.matches[0].matchedItemIndex, 0); + assert.equal(data.matches[0].matchedIdentifiers.doi, "10.1234/primary-doi"); + assert.equal(data.matches[0].matchedItemIndex, 1); assert.deepEqual(data.matches[0].matchedItemIndexes, [0, 1]); });