Clear MDPI's Akamai bot challenge during Find Full Text

MDPI serves a JS proof-of-work interstitial in place of the article
page, so Find Full Text found nothing. Run the challenge in a hidden
browser when a page's meta refresh points to a registered challenge
host, then retry the page.

Also match meta refresh URL= case-insensitively, since MDPI's is
uppercase, and bound meta refreshes by the redirect limit.

https://forums.zotero.org/discussion/132837/
https://forums.zotero.org/discussion/134079/

chrome/content/zotero/xpcom/attachments.js

@@ -2132,6 +2132,7 @@ Zotero.Attachments = new function () {
 					let domains = new Set();
 					let redirectLimit = 10;
 					let redirectURLTries = new Map();
+					let clearedChallenges = new Set();
 					while (true) {
 						if (redirectLimit == 0) {
 							Zotero.debug("Too many redirects -- stopping");
@@ -2215,6 +2216,29 @@ Zotero.Attachments = new function () {
 							// Check for a meta redirect on HTML pages
 							let refreshURL = Zotero.HTTP.getHTMLMetaRefreshURL(doc, responseURL);
 							if (refreshURL) {
+								// If the refresh points at a known bot-challenge host, the
+								// interstitial runs JS that a plain request can't satisfy. Run
+								// it once in a hidden browser to bank the resulting cookies,
+								// then retry the page over the normal path.
+								let challengeEntry = Zotero.BrowserRequest.getEntryForURL(refreshURL);
+								if (challengeEntry && !clearedChallenges.has(responseURL)) {
+									clearedChallenges.add(responseURL);
+									try {
+										await Zotero.BrowserRequest.clearChallenge(
+											responseURL,
+											{ entry: challengeEntry }
+										);
+										doc = null;
+										nextURL = responseURL;
+										redirectLimit--;
+										continue;
+									}
+									catch (e) {
+										Zotero.debug(`Failed to clear challenge at ${responseURL}: ${e}`);
+										skip = true;
+										break;
+									}
+								}
 								if (isTriedURL(refreshURL)) {
 									Zotero.debug("Meta refresh URL has already been tried -- skipping");
 									skip = true;
@@ -2222,6 +2246,7 @@ Zotero.Attachments = new function () {
 								}
 								doc = null;
 								nextURL = refreshURL;
+								redirectLimit--;
 								continue;
 							}
 							// ProQuest does a JS redirect back to the original page after authenticating

chrome/content/zotero/xpcom/browserRequest.js

@@ -55,6 +55,17 @@ Zotero.BrowserRequest = {
 			match: '://pmc.ncbi.nlm.nih.gov',
 			captchaLocator: null
 		},
+		{
+			// MDPI fronts pages with an Akamai Bot Manager interstitial: a 200
+			// whose body is a meta-refresh to a ?bm-verify= URL plus a script
+			// that computes a proof-of-work and POSTs it to /_sec/verify before
+			// reloading the page without the token. There's no CAPTCHA to show a
+			// user -- it's a fully automatic JS handshake -- so a hidden browser
+			// runs it to completion and banks the resulting cookies, with no
+			// success cookie or viewer escalation.
+			match: '://www.mdpi.com',
+			captchaLocator: null,
+		},
 	],
 	
 	PLAIN_UA_HOSTS: [
@@ -62,6 +73,7 @@ Zotero.BrowserRequest = {
 		'www.sciencedirect.com',
 		'pdf.sciencedirectassets.com',
 		'search.worldcat.org',
+		'www.mdpi.com',
 	],
 
 	/**
@@ -118,6 +130,7 @@ Zotero.BrowserRequest = {
 		// and resolves as soon as successCookie appears, which may be well
 		// before the page fully settles (or redirects somewhere else).
 		let hiddenBrowser;
+		let settled = false;
 		try {
 			hiddenBrowser = new HiddenBrowser({ userContextId, customUserAgent });
 			await hiddenBrowser._createdPromise;
@@ -125,6 +138,7 @@ Zotero.BrowserRequest = {
 				successCookie,
 				userContextId
 			});
+			settled = true;
 		}
 		catch (e) {
 			Zotero.debug('BrowserRequest: Hidden browser attempt failed');
@@ -142,6 +156,12 @@ Zotero.BrowserRequest = {
 				return;
 			}
 		}
+		// For a fully automatic interstitial with no explicit success signal or
+		// captcha, a clean settle means the challenge JS ran to completion and
+		// left its cookies in the shared jar.
+		else if (settled && entry && !entry.captchaLocator) {
+			return;
+		}
 
 		if (!allowViewer) {
 			throw new Error(`BrowserRequest: Challenge not cleared at ${url} and viewer escalation is disabled`);

chrome/content/zotero/xpcom/http.js

@@ -1052,7 +1052,7 @@ Zotero.HTTP = new function () {
 		if (!content) {
 			return false;
 		}
-		var parts = content.split(/;\s*url=/);
+		var parts = content.split(/;\s*url=/i);
 		// If there's a redirect to another URL in less than 15 seconds,
 		// follow it
 		if (parts.length === 2 && parseInt(parts[0]) <= 15) {

test/tests/attachmentsTest.js

@@ -684,6 +684,8 @@ describe("Zotero.Attachments", function () {
 		var pageURL9 = 'http://website/article9';
 		var pageURL10 = 'http://website/refresh';
 		var pageURL11 = 'http://website/book';
+		var pageURL12 = 'https://www.mdpi.com/2073-4433/16/2/169';
+		var mdpiChallengeCleared = false;
 		
 		var httpd;
 		var port = 16213;
@@ -890,6 +892,20 @@ describe("Zotero.Attachments", function () {
 					let html = `<html><head><meta http-equiv=\"refresh\" content=\"2;url=${pageURL1}\"/></head><body></body></html>`;
 					return makeHTMLResponseFromType(html, options.responseType, pageURL10);
 				}
+
+				// MDPI serves an Akamai interstitial (meta-refresh to a bm-verify
+				// URL) until the challenge has been cleared in a hidden browser,
+				// then the real page with a PDF link.
+				if (url == pageURL12) {
+					let html;
+					if (mdpiChallengeCleared) {
+						html = getHTMLPage(true);
+					}
+					else {
+						html = `<html><head><meta http-equiv="refresh" content="5; URL='${pageURL12}?bm-verify=TOKEN'"/></head><body></body></html>`;
+					}
+					return makeHTMLResponseFromType(html, options.responseType, pageURL12);
+				}
 				
 				// OA PDF lookup
 				if (url.startsWith(ZOTERO_CONFIG.SERVICES_URL)) {
@@ -1315,6 +1331,38 @@ describe("Zotero.Attachments", function () {
 			assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize);
 		});
 		
+		it("should clear a bot challenge in a browser and retry the page", async function () {
+			mdpiChallengeCleared = false;
+			// Simulate the hidden browser solving the interstitial: once
+			// clearChallenge() runs, the page stops serving the challenge.
+			let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").callsFake(async () => {
+				mdpiChallengeCleared = true;
+			});
+			try {
+				var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
+				item.setField('title', 'Test');
+				item.setField('url', pageURL12);
+				await item.saveTx();
+				var attachment = await Zotero.Attachments.addAvailableFile(item);
+
+				// The challenge is cleared for the page itself, not the one-shot
+				// bm-verify token URL.
+				assert.isTrue(clearStub.calledOnce);
+				assert.equal(clearStub.firstCall.args[0], pageURL12);
+				// Interstitial, then the retried page
+				assert.equal(requestStub.getCall(0).args[1], pageURL12);
+				assert.equal(requestStub.getCall(1).args[1], pageURL12);
+				assert.ok(attachment);
+				var json = attachment.toJSON();
+				assert.equal(json.url, pdfURL);
+				assert.equal(json.contentType, 'application/pdf');
+				assert.equal(await OS.File.stat(attachment.getFilePath()).size, pdfSize);
+			}
+			finally {
+				clearStub.restore();
+			}
+		});
+		
 		it("should stop after too many redirects to the same URL", async function () {
 			var item = createUnsavedDataObject('item', { itemType: 'journalArticle' });
 			item.setField('url', 'http://website/redirect_loop1');

test/tests/browserRequestTest.js

@@ -155,6 +155,60 @@ describe("Zotero.BrowserRequest", function () {
 		});
 	});
 
+	describe("#clearChallenge()", function () {
+		it("resolves once the hidden browser settles for an automatic interstitial", async function () {
+			// An entry with no successCookie (e.g., an Akamai proof-of-work
+			// interstitial) has no explicit success signal, so a clean settle
+			// is what counts as cleared.
+			let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").resolves();
+			try {
+				await Zotero.BrowserRequest.clearChallenge('https://www.mdpi.com/2073-4433/16/2/169', {
+					entry: { match: '://www.mdpi.com', captchaLocator: null }
+				});
+				assert.isTrue(settleStub.calledOnce);
+			}
+			finally {
+				settleStub.restore();
+			}
+		});
+
+		it("doesn't treat a settle as cleared for an entry with a captcha", async function () {
+			let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").resolves();
+			let err;
+			try {
+				await Zotero.BrowserRequest.clearChallenge('https://www.sciencedirect.com/science/article/pii/S0000000000000000', {
+					entry: { match: '://www.sciencedirect.com', captchaLocator: '#captcha-box' }
+				});
+			}
+			catch (e) {
+				err = e;
+			}
+			finally {
+				settleStub.restore();
+			}
+			assert.isDefined(err);
+			assert.include(err.message, 'not cleared');
+		});
+
+		it("throws when the hidden browser fails to settle and no viewer is allowed", async function () {
+			let settleStub = sinon.stub(Zotero.BrowserRequest, "_loadAndSettle").rejects(new Error("timed out"));
+			let err;
+			try {
+				await Zotero.BrowserRequest.clearChallenge('https://www.mdpi.com/2073-4433/16/2/169', {
+					entry: { match: '://www.mdpi.com', captchaLocator: null }
+				});
+			}
+			catch (e) {
+				err = e;
+			}
+			finally {
+				settleStub.restore();
+			}
+			assert.isDefined(err);
+			assert.include(err.message, 'not cleared');
+		});
+	});
+
 	describe("translator request retry", function () {
 		function makeUtils() {
 			let fakeTranslate = {