From 09c3c8cd3d0070691ba0579bc0157f6fb7d77755 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Tue, 6 Oct 2026 10:35:31 -0400 Subject: [PATCH] Clear MDPI's Akamai bot challenge during Find Full Text MDPI serves a JS proof-of-work interstitial in place of the article page, so Find Full Text found nothing. Run the challenge in a hidden browser when a page's meta refresh points to a registered challenge host, then retry the page. Also match meta refresh URL= case-insensitively, since MDPI's is uppercase, and bound meta refreshes by the redirect limit. https://forums.zotero.org/discussion/132837/ https://forums.zotero.org/discussion/134079/ --- chrome/content/zotero/xpcom/attachments.js | 25 +++++++++ chrome/content/zotero/xpcom/browserRequest.js | 20 +++++++ chrome/content/zotero/xpcom/http.js | 2 +- test/tests/attachmentsTest.js | 48 +++++++++++++++++ test/tests/browserRequestTest.js | 54 +++++++++++++++++++ 5 files changed, 148 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index 0a5a10f0f5..9b971fdb90 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -2132,6 +2132,7 @@ Zotero.Attachments = new function () { let domains = new Set(); let redirectLimit = 10; let redirectURLTries = new Map(); + let clearedChallenges = new Set(); while (true) { if (redirectLimit == 0) { Zotero.debug("Too many redirects -- stopping"); @@ -2215,6 +2216,29 @@ Zotero.Attachments = new function () { // Check for a meta redirect on HTML pages let refreshURL = Zotero.HTTP.getHTMLMetaRefreshURL(doc, responseURL); if (refreshURL) { + // If the refresh points at a known bot-challenge host, the + // interstitial runs JS that a plain request can't satisfy. Run + // it once in a hidden browser to bank the resulting cookies, + // then retry the page over the normal path. + let challengeEntry = Zotero.BrowserRequest.getEntryForURL(refreshURL); + if (challengeEntry && !clearedChallenges.has(responseURL)) { + clearedChallenges.add(responseURL); + try { + await Zotero.BrowserRequest.clearChallenge( + responseURL, + { entry: challengeEntry } + ); + doc = null; + nextURL = responseURL; + redirectLimit--; + continue; + } + catch (e) { + Zotero.debug(`Failed to clear challenge at ${responseURL}: ${e}`); + skip = true; + break; + } + } if (isTriedURL(refreshURL)) { Zotero.debug("Meta refresh URL has already been tried -- skipping"); skip = true; @@ -2222,6 +2246,7 @@ Zotero.Attachments = new function () { } doc = null; nextURL = refreshURL; + redirectLimit--; continue; } // ProQuest does a JS redirect back to the original page after authenticating diff --git a/chrome/content/zotero/xpcom/browserRequest.js b/chrome/content/zotero/xpcom/browserRequest.js index fcd3b70afb..0c37d2ca6c 100644 --- a/chrome/content/zotero/xpcom/browserRequest.js +++ b/chrome/content/zotero/xpcom/browserRequest.js @@ -55,6 +55,17 @@ Zotero.BrowserRequest = { match: '://pmc.ncbi.nlm.nih.gov', captchaLocator: null }, + { + // MDPI fronts pages with an Akamai Bot Manager interstitial: a 200 + // whose body is a meta-refresh to a ?bm-verify= URL plus a script + // that computes a proof-of-work and POSTs it to /_sec/verify before + // reloading the page without the token. There's no CAPTCHA to show a + // user -- it's a fully automatic JS handshake -- so a hidden browser + // runs it to completion and banks the resulting cookies, with no + // success cookie or viewer escalation. + match: '://www.mdpi.com', + captchaLocator: null, + }, ], PLAIN_UA_HOSTS: [ @@ -62,6 +73,7 @@ Zotero.BrowserRequest = { 'www.sciencedirect.com', 'pdf.sciencedirectassets.com', 'search.worldcat.org', + 'www.mdpi.com', ], /** @@ -118,6 +130,7 @@ Zotero.BrowserRequest = { // and resolves as soon as successCookie appears, which may be well // before the page fully settles (or redirects somewhere else). let hiddenBrowser; + let settled = false; try { hiddenBrowser = new HiddenBrowser({ userContextId, customUserAgent }); await hiddenBrowser._createdPromise; @@ -125,6 +138,7 @@ Zotero.BrowserRequest = { successCookie, userContextId }); + settled = true; } catch (e) { Zotero.debug('BrowserRequest: Hidden browser attempt failed'); @@ -142,6 +156,12 @@ Zotero.BrowserRequest = { return; } } + // For a fully automatic interstitial with no explicit success signal or + // captcha, a clean settle means the challenge JS ran to completion and + // left its cookies in the shared jar. + else if (settled && entry && !entry.captchaLocator) { + return; + } if (!allowViewer) { throw new Error(`BrowserRequest: Challenge not cleared at ${url} and viewer escalation is disabled`); diff --git a/chrome/content/zotero/xpcom/http.js b/chrome/content/zotero/xpcom/http.js index d00802e73c..260bc31b73 100644 --- a/chrome/content/zotero/xpcom/http.js +++ b/chrome/content/zotero/xpcom/http.js @@ -1052,7 +1052,7 @@ Zotero.HTTP = new function () { if (!content) { return false; } - var parts = content.split(/;\s*url=/); + var parts = content.split(/;\s*url=/i); // If there's a redirect to another URL in less than 15 seconds, // follow it if (parts.length === 2 && parseInt(parts[0]) <= 15) { diff --git a/test/tests/attachmentsTest.js b/test/tests/attachmentsTest.js index b14df7bf81..c9234aef35 100644 --- a/test/tests/attachmentsTest.js +++ b/test/tests/attachmentsTest.js @@ -684,6 +684,8 @@ describe("Zotero.Attachments", function () { var pageURL9 = 'http://website/article9'; var pageURL10 = 'http://website/refresh'; var pageURL11 = 'http://website/book'; + var pageURL12 = 'https://www.mdpi.com/2073-4433/16/2/169'; + var mdpiChallengeCleared = false; var httpd; var port = 16213; @@ -890,6 +892,20 @@ describe("Zotero.Attachments", function () { let html = ``; return makeHTMLResponseFromType(html, options.responseType, pageURL10); } + + // MDPI serves an Akamai interstitial (meta-refresh to a bm-verify + // URL) until the challenge has been cleared in a hidden browser, + // then the real page with a PDF link. + if (url == pageURL12) { + let html; + if (mdpiChallengeCleared) { + html = getHTMLPage(true); + } + else { + html = ``; + } + return makeHTMLResponseFromType(html, options.responseType, pageURL12); + } // OA PDF lookup if (url.startsWith(ZOTERO_CONFIG.SERVICES_URL)) { @@ -1315,6 +1331,38 @@ describe("Zotero.Attachments", function () { assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize); }); + it("should clear a bot challenge in a browser and retry the page", async function () { + mdpiChallengeCleared = false; + // Simulate the hidden browser solving the interstitial: once + // clearChallenge() runs, the page stops serving the challenge. + let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").callsFake(async () => { + mdpiChallengeCleared = true; + }); + try { + var item = createUnsavedDataObject('item', { itemType: 'journalArticle' }); + item.setField('title', 'Test'); + item.setField('url', pageURL12); + await item.saveTx(); + var attachment = await Zotero.Attachments.addAvailableFile(item); + + // The challenge is cleared for the page itself, not the one-shot + // bm-verify token URL. + assert.isTrue(clearStub.calledOnce); + assert.equal(clearStub.firstCall.args[0], pageURL12); + // Interstitial, then the retried page + assert.equal(requestStub.getCall(0).args[1], pageURL12); + assert.equal(requestStub.getCall(1).args[1], pageURL12); + assert.ok(attachment); + var json = attachment.toJSON(); + assert.equal(json.url, pdfURL); + assert.equal(json.contentType, 'application/pdf'); + assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize); + } + finally { + clearStub.restore(); + } + }); + it("should stop after too many redirects to the same URL", async function () { var item = createUnsavedDataObject('item', { itemType: 'journalArticle' }); item.setField('url', 'http://website/redirect_loop1'); diff --git a/test/tests/browserRequestTest.js b/test/tests/browserRequestTest.js index a1b81d6b58..63c383cd40 100644 --- a/test/tests/browserRequestTest.js +++ b/test/tests/browserRequestTest.js @@ -155,6 +155,60 @@ describe("Zotero.BrowserRequest", function () { }); }); + describe("#clearChallenge()", function () { + it("resolves once the hidden browser settles for an automatic interstitial", async function () { + // An entry with no successCookie (e.g., an Akamai proof-of-work + // interstitial) has no explicit success signal, so a clean settle + // is what counts as cleared. + let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").resolves(); + try { + await Zotero.BrowserRequest.clearChallenge('https://www.mdpi.com/2073-4433/16/2/169', { + entry: { match: '://www.mdpi.com', captchaLocator: null } + }); + assert.isTrue(settleStub.calledOnce); + } + finally { + settleStub.restore(); + } + }); + + it("doesn't treat a settle as cleared for an entry with a captcha", async function () { + let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").resolves(); + let err; + try { + await Zotero.BrowserRequest.clearChallenge('https://www.sciencedirect.com/science/article/pii/S0000000000000000', { + entry: { match: '://www.sciencedirect.com', captchaLocator: '#captcha-box' } + }); + } + catch (e) { + err = e; + } + finally { + settleStub.restore(); + } + assert.isDefined(err); + assert.include(err.message, 'not cleared'); + }); + + it("throws when the hidden browser fails to settle and no viewer is allowed", async function () { + let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").rejects(new Error("timed out")); + let err; + try { + await Zotero.BrowserRequest.clearChallenge('https://www.mdpi.com/2073-4433/16/2/169', { + entry: { match: '://www.mdpi.com', captchaLocator: null } + }); + } + catch (e) { + err = e; + } + finally { + settleStub.restore(); + } + assert.isDefined(err); + assert.include(err.message, 'not cleared'); + }); + }); + describe("translator request retry", function () { function makeUtils() { let fakeTranslate = {