diff --git a/chrome/content/zotero/HiddenBrowser.mjs b/chrome/content/zotero/HiddenBrowser.mjs index 9d7510758b..ac97707366 100644 --- a/chrome/content/zotero/HiddenBrowser.mjs +++ b/chrome/content/zotero/HiddenBrowser.mjs @@ -50,6 +50,8 @@ export class HiddenBrowser { * Must be set to false if intending to call print(). * @param {Number} [options.cookieContextId] - userContextId from Zotero.HTTP.newCookieContext() * for cookie isolation + * @param {String} [options.customUserAgent] - Override User-Agent for all requests + * from this browser's browsing context */ constructor(options = {}) { this._destroyed = false; @@ -96,6 +98,10 @@ export class HiddenBrowser { this._blockingObserver.register(browser); } + if (options.customUserAgent) { + browser.browsingContext.customUserAgent = options.customUserAgent; + } + this._browser = browser; })(); diff --git a/chrome/content/zotero/actors/PageDataChild.mjs b/chrome/content/zotero/actors/PageDataChild.mjs index 04ac68f412..87093301c1 100644 --- a/chrome/content/zotero/actors/PageDataChild.mjs +++ b/chrome/content/zotero/actors/PageDataChild.mjs @@ -29,6 +29,9 @@ export class PageDataChild extends JSWindowActorChild { case "documentHTML": return new XMLSerializer().serializeToString(document); + case "querySelectorMatches": + return !!document.querySelector(message.data.selector); + case "channelInfo": { let docShell = this.contentWindow.docShell; try { diff --git a/chrome/content/zotero/standalone/basicViewer.js b/chrome/content/zotero/standalone/basicViewer.js index 163dfa04a8..314915dfda 100644 --- a/chrome/content/zotero/standalone/basicViewer.js +++ b/chrome/content/zotero/standalone/basicViewer.js @@ -32,31 +32,31 @@ const SANDBOXED_SCRIPTS = 0x80; var browser; window.addEventListener("load", /*async */function () { + let { uri, options } = window.arguments[0].wrappedJSObject; + browser = document.querySelector('browser'); + if (options?.cookieContextId) { + // Set usercontextid on new so it takes effect + let newBrowser = document.createXULElement('browser'); + for (let { name, value } of browser.attributes) { + newBrowser.setAttribute(name, value); + } + newBrowser.setAttribute('usercontextid', String(options.cookieContextId)); + browser.replaceWith(newBrowser); + browser = newBrowser; + } window.gBrowser = browser; // For ZoomManager browser.addEventListener('pagetitlechanged', () => { document.title = browser.contentTitle || browser.currentURI.spec; }); - - /* - browser.setAttribute("remote", "true"); - //browser.setAttribute("remoteType", E10SUtils.EXTENSION_REMOTE_TYPE); - - await new Promise((resolve) => { - browser.addEventListener("XULFrameLoaderCreated", () => resolve()); - }); - */ - - /*browser.messageManager.loadFrameScript( - 'chrome://zotero/content/standalone/basicViewerContent.js', - false - );*/ - //browser.docShellIsActive = false; - // Get URI and options passed in via openWindow() - let { uri, options } = window.arguments[0].wrappedJSObject; + if (options?.customUserAgent) { + browser.browsingContext.customUserAgent = options.customUserAgent; + } + window.viewerOriginalURI = uri; + window.viewerCookieContextId = options?.cookieContextId; loadURI(Services.io.newURI(uri), options); }, false); diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index cea9c32c8b..54e3939706 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -1200,8 +1200,8 @@ Zotero.Attachments = new function () { && (e instanceof this.InvalidPDFException // Thrown by HTTP.download() || (e instanceof Zotero.HTTP.UnexpectedStatusException && e.status == 403))) { - if (Zotero.BrowserDownload.shouldAttemptDownloadViaBrowser(url)) { - return Zotero.BrowserDownload.downloadPDF(url, path, options); + if (Zotero.BrowserRequest.getEntryForURL(url)) { + return Zotero.BrowserRequest.downloadPDF(url, path, options); } } throw e; diff --git a/chrome/content/zotero/xpcom/browserDownload.js b/chrome/content/zotero/xpcom/browserDownload.js deleted file mode 100644 index e47f584adf..0000000000 --- a/chrome/content/zotero/xpcom/browserDownload.js +++ /dev/null @@ -1,243 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2023 Corporation for Digital Scholarship - Vienna, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - -const { HiddenBrowser } = ChromeUtils.importESModule("chrome://zotero/content/HiddenBrowser.mjs"); - -Zotero.BrowserDownload = { - HANDLED_URLS: { - 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html': "html", - '://www.sciencedirect.com': "#captcha-box" - }, - - /** - * Checks whether the url can be handled as a hidden browser download - * @param {String} url - */ - shouldAttemptDownloadViaBrowser: function (url) { - const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); - for (let unproxiedUrl of unproxiedUrls) { - for (let checkUrl in this.HANDLED_URLS) { - if (unproxiedUrl.includes(checkUrl)) { - return checkUrl; - } - } - } - return false; - }, - - getCaptchaLocator(url) { - const handlerKey = this.shouldAttemptDownloadViaBrowser(url); - return this.HANDLED_URLS[handlerKey]; - }, - - _makePDFMIMETypeHandler(browser, onPDFFound = () => 0) { - let isOurPDF, channelBrowser; - let trackedBrowser = browser; - return { - onStartRequest: function (name, _, channel) { - Zotero.debug(`BrowserDownload: Sniffing a PDF loaded at ${name}`); - // try the browser - try { - channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; - } - catch (e) {} - if (channelBrowser) { - isOurPDF = trackedBrowser === channelBrowser; - } - else { - // try the document for the load group - try { - channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) - .topFrameElement; - } - catch (e) {} - if (channelBrowser) { - isOurPDF = trackedBrowser === channelBrowser; - } - } - }, - onContent: async (blob, name) => { - if (isOurPDF) { - Zotero.debug(`BrowserDownload: Found our PDF at ${name}`); - onPDFFound(blob); - return true; - } - else { - Zotero.debug(`BrowserDownload: Not our PDF at ${name}`); - return false; - } - } - }; - }, - - /** - * @param {String} url - * @param {String} path - * @param {Object} [options] - * @param {Boolean} [options.shouldDisplayCaptcha=false] - */ - async downloadPDF(url, path, options = {}) { - Zotero.debug(`BrowserDownload: Downloading file via a hidden browser from ${url}`); - - let hiddenBrowser; - let pdfMIMETypeHandler; - let pdfFoundDeferred = Zotero.Promise.defer(); - - // Technically this is not a download, but the full operation (load, redirect, etc) timeout - const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); - const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout'); - - try { - hiddenBrowser = new HiddenBrowser(); - await hiddenBrowser._createdPromise; - - let pdfLoaded = false; - pdfMIMETypeHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, pdfFoundDeferred.resolve); - Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); - - let onLoadTimeoutDeferred = Zotero.Promise.defer(); - let currentUrl = ""; - hiddenBrowser.webProgress.addProgressListener({ - QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), - async onLocationChange() { - let url = hiddenBrowser.currentURI.spec; - if (currentUrl) { - Zotero.debug(`BrowserDownload: A JS redirect occurred to ${url}`); - } - currentUrl = url; - Zotero.debug(`BrowserDownload: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); - await Zotero.Promise.delay(onLoadTimeout); - // If URL changed that means we got redirected and the onLoadTimeout needs to restart - if (currentUrl === url && !pdfLoaded) { - onLoadTimeoutDeferred.reject(new Error(`BrowserDownload: Loading PDF via a hidden browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); - } - } - }, Ci.nsIWebProgress.NOTIFY_LOCATION); - - hiddenBrowser.load(url); - let blob = await Promise.race([ - onLoadTimeoutDeferred.promise, - Zotero.Promise.delay(downloadTimeout).then(() => { - if (!pdfLoaded) { - throw new Error(`BrowserDownload: Loading PDF via a hidden browser timed out after ${downloadTimeout}ms`); - } - }), - // Resolves PDF blob - pdfFoundDeferred.promise - ]); - - pdfLoaded = true; - await Zotero.File.putContentsAsync(path, blob); - } - catch (e) { - try { - await OS.File.remove(path, { ignoreAbsent: true }); - } - catch (err) { - Zotero.logError(err); - } - if (options?.shouldDisplayCaptcha) { - Zotero.debug(`BrowserDownload: Downloading via a hidden browser failed due to ${e.message}`); - const captchaLocator = this.getCaptchaLocator(url); - if (captchaLocator) { - let doc = await hiddenBrowser.getDocument(); - let elem = doc.querySelector(captchaLocator); - if (elem) { - return this.downloadPDFViaViewer(url, path, options); - } - } - } - throw e; - } - finally { - Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); - if (hiddenBrowser) { - hiddenBrowser.destroy(); - } - } - }, - - async downloadPDFViaViewer(url, path, options) { - Zotero.debug(`BrowserDownload: Downloading file via the document viewer for captcha clearing from ${url}`); - - let win, browser, xulWin, wmListener; - let pdfMIMETypeHandler; - let pdfFound; - let pdfFoundDeferred = Zotero.Promise.defer(); - const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); - - try { - wmListener = { - onOpenWindow(xulWindow) { - xulWin = xulWin || xulWindow; - }, - onCloseWindow(xulWindow) { - if (xulWin === xulWindow && !pdfFound) { - pdfFoundDeferred.reject(new Error("BrowserDownload: User closed the document viewer")); - } - } - }; - Services.wm.addListener(wmListener); - await new Promise((resolve) => { - win = Zotero.openInViewer(url); - win.addEventListener('load', resolve); - }); - browser = win.document.querySelector('browser'); - Zotero.Utilities.Internal.activate(win); - - pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve); - Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); - - Zotero.debug(`BrowserDownload: Awaiting the user to clear the captcha or timeout after ${downloadTimeout}`); - let pdfBlob = await Promise.race([ - Zotero.Promise.delay(downloadTimeout).then(() => { - if (!pdfFound) { - throw new Error(`BrowserDownload: Loading PDF via document viewer timed out after ${downloadTimeout}ms`); - } - }), - // Resolves PDF blob - pdfFoundDeferred.promise - ]); - pdfFound = true; - await Zotero.File.putContentsAsync(path, pdfBlob); - } - catch (e) { - try { - await OS.File.remove(path, { ignoreAbsent: true }); - } - catch (err) { - Zotero.logError(err); - } - throw e; - } - finally { - Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); - Services.wm.removeListener(wmListener); - if (win) { - win.close(); - } - } - }, -}; diff --git a/chrome/content/zotero/xpcom/browserRequest.js b/chrome/content/zotero/xpcom/browserRequest.js new file mode 100644 index 0000000000..d25e7e462c --- /dev/null +++ b/chrome/content/zotero/xpcom/browserRequest.js @@ -0,0 +1,647 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +const { HiddenBrowser } = ChromeUtils.importESModule("chrome://zotero/content/HiddenBrowser.mjs"); + +Zotero.BrowserRequest = { + // Registry of URL patterns that need browser-mediated handling + CHALLENGE_URLS: [ + { + match: 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', + captchaLocator: 'html' + }, + { + match: '://www.sciencedirect.com', + captchaLocator: '#captcha-box' + }, + { + match: '://search.worldcat.org', + // When /api/search returns 403 with turnstile_required, the user- + // facing /search?q= page runs the matching invisible Turnstile + // widget, POSTs the token to /api/turnstile-verify, and gets back + // Set-Cookie: turnstile_passed. Loading that page in a hidden + // browser reproduces the flow end-to-end with no user interaction. + getChallengeURL: url => url.replace(/\/api\/search\b/, '/search'), + // Invisible (managed) Turnstile; no user captcha interaction needed + captchaLocator: null, + // Wait for WorldCat's own Turnstile cookie to land + successCookie: { host: 'search.worldcat.org', name: 'turnstile_passed' }, + // turnstile_passed is signed against (IP, UA), so we must use the + // same plain-Firefox UA for the translator's follow-up requests as + // the hidden browser used when earning it. + plainUAHost: 'search.worldcat.org', + detectBlock: (status, body) => status === 403 && /turnstile_required/.test(body) + } + ], + + /** + * Look up a challenge entry for a URL, applying proxy unwrapping. + * @param {string} url + * @returns {object|null} + */ + getEntryForURL(url) { + const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); + for (let unproxiedUrl of unproxiedUrls) { + for (let entry of this.CHALLENGE_URLS) { + if (unproxiedUrl.includes(entry.match)) { + return entry; + } + } + } + return null; + }, + + /** + * Navigate to a URL in a hidden browser, running its JS long enough for any + * client-side redirects or cookie-setting challenges to settle. + * + * Cookies acquired by the browser remain in the shared jar keyed on + * cookieContextId; a subsequent Zotero.HTTP.request using the same ID will + * see them. + * + * On timeout, if the page contains the entry's captchaLocator and + * allowViewer is set, escalates to clearChallengeInViewer(). + * + * @param {string} url + * @param {object} [options] + * @param {number} [options.cookieContextId] + * @param {object} [options.entry] - registry entry controlling escalation + * @param {boolean} [options.allowViewer=false] + * @returns {Promise} + */ + async clearChallenge(url, options = {}) { + Zotero.debug(`BrowserRequest: Clearing challenge at ${url}`); + + let { cookieContextId, entry, allowViewer = false } = options; + let successCookie = entry?.successCookie; + // Capture the cookie's current value (if any) before the attempt so + // we can tell a freshly-issued cookie from a stale one left over from + // a previous session. + let initialCookieValue = successCookie + ? this._readCookieValue({ ...successCookie, cookieContextId }) + : null; + // Cloudflare Turnstile rejects the "Zotero/[version]" suffix. + // A plain Firefox UA on just this browsing context lets the widget run. + let customUserAgent = Zotero.VersionHeader.getPlainFirefoxUA(); + + // Try the hidden browser first. _loadAndSettle() polls the cookie jar + // and resolves as soon as successCookie appears, which may be well + // before the page fully settles (or redirects somewhere else). + let hiddenBrowser; + try { + hiddenBrowser = new HiddenBrowser({ cookieContextId, customUserAgent }); + await hiddenBrowser._createdPromise; + await this._loadAndSettle(hiddenBrowser, url, { + successCookie, + cookieContextId + }); + } + catch (e) { + Zotero.debug('BrowserRequest: Hidden browser attempt failed'); + Zotero.logError(e); + } + finally { + if (hiddenBrowser) { + hiddenBrowser.destroy(); + } + } + + if (successCookie) { + let currentValue = this._readCookieValue({ ...successCookie, cookieContextId }); + if (currentValue && currentValue !== initialCookieValue) { + return; + } + } + + if (!allowViewer) { + throw new Error(`BrowserRequest: Challenge not cleared at ${url} and viewer escalation is disabled`); + } + + // Fall back to the viewer: user may need to click a visible Turnstile + // widget, after which the cookie lands and we can continue. + Zotero.debug(`BrowserRequest: Escalating to viewer for ${url}`); + if (successCookie) { + await this._loadAndWaitForCookieInViewer(url, { + cookieContextId, + customUserAgent, + successCookie + }); + return; + } + await this.clearChallengeInViewer(url, { + cookieContextId, + captchaLocator: entry.captchaLocator + }); + }, + + /** + * Open a visible viewer at the URL and wait until successCookie appears + * in the jar. Necessary for sites where the success signal is a specific + * cookie being set (e.g., WorldCat's turnstile_passed) rather than a + * navigation or change in the DOM. + */ + async _loadAndWaitForCookieInViewer(url, options) { + Zotero.debug(`BrowserRequest: Awaiting user challenge clearance (cookie ${options.successCookie.name}) at ${url}`); + const timeout = Zotero.Prefs.get('browserRequest.timeout'); + const { successCookie, cookieContextId, customUserAgent } = options; + + let win, wmListener, pollInterval; + let done = false; + let cookieDeferred = Zotero.Promise.defer(); + let closedDeferred = Zotero.Promise.defer(); + + try { + wmListener = this._makeViewerCloseListener(() => { + if (!done) closedDeferred.reject(new Error('BrowserRequest: User closed the viewer')); + }); + Services.wm.addListener(wmListener); + await new Promise((resolve) => { + win = Zotero.openInViewer(url, { cookieContextId, customUserAgent }); + win.addEventListener('load', resolve); + }); + Zotero.Utilities.Internal.activate(win); + + pollInterval = this._pollForCookie({ + successCookie, + cookieContextId, + onFound: () => { + done = true; + cookieDeferred.resolve(); + } + }); + + await Promise.race([ + cookieDeferred.promise, + closedDeferred.promise, + Zotero.Promise.delay(timeout).then(() => { + if (!done) { + throw new Error(`BrowserRequest: Viewer cookie wait timed out after ${timeout}ms`); + } + }) + ]); + } + finally { + if (pollInterval) clearInterval(pollInterval); + Services.wm.removeListener(wmListener); + if (win) win.close(); + } + }, + + /** + * Read the value of a named cookie on a given host under an optional + * userContextId. Returns null if the cookie is absent. + */ + _readCookieValue({ host, name, cookieContextId }) { + try { + let cookies = Services.cookies.getCookiesFromHost( + host, + cookieContextId ? { userContextId: cookieContextId } : {} + ); + for (let cookie of cookies) { + if (cookie.name === name) { + return cookie.value; + } + } + } + catch (e) { + Zotero.debug('BrowserRequest: _readCookieValue() failed'); + Zotero.logError(e); + } + return null; + }, + + /** + * Build a Services.wm listener that tracks the first window opened after + * addListener() and invokes onClose when that window closes. Used to + * detect user-closed viewer windows. + * + * @param {Function} onClose + */ + _makeViewerCloseListener(onClose) { + let xulWin; + return { + onOpenWindow(xulWindow) { + xulWin ||= xulWindow; + }, + onCloseWindow(xulWindow) { + if (xulWin === xulWindow) { + onClose(); + } + } + }; + }, + + /** + * Invoke onFound as soon as successCookie appears with a value different + * from the one observed at poll start. Captures the initial value to + * avoid declaring success on a stale cookie left over from a previous + * session (the server-signed HMAC would no longer validate). + * Caller is responsible for clearing the returned interval handle. + * + * @param {object} opts + * @param {{host: string, name: string}} opts.successCookie + * @param {number} [opts.cookieContextId] + * @param {Function} opts.onFound + * @param {number} [opts.intervalMs=250] + * @returns {number} interval handle + */ + _pollForCookie({ successCookie, cookieContextId, onFound, intervalMs = 250 }) { + let { host, name } = successCookie; + let initialValue = this._readCookieValue({ host, name, cookieContextId }); + return setInterval(() => { + let currentValue = this._readCookieValue({ host, name, cookieContextId }); + if (currentValue && currentValue !== initialValue) { + onFound(); + } + }, intervalMs); + }, + + /** + * Open a visible browser window at the URL and wait for the user to clear a + * challenge. Resolves once the captchaLocator element disappears and the + * page has been stable for `browserRequest.onLoadTimeout` ms. + * + * @param {string} url + * @param {object} options + * @param {number} [options.cookieContextId] + * @param {string} options.captchaLocator + * @returns {Promise} + */ + async clearChallengeInViewer(url, options) { + Zotero.debug(`BrowserRequest: Awaiting user challenge clearance for ${url}`); + const onLoadTimeout = Zotero.Prefs.get('browserRequest.onLoadTimeout'); + const timeout = Zotero.Prefs.get('browserRequest.timeout'); + + let win, browser, wmListener; + let cleared = false; + let cancelled = false; + let clearedDeferred = Zotero.Promise.defer(); + + try { + wmListener = this._makeViewerCloseListener(() => { + if (!cleared) clearedDeferred.reject(new Error('BrowserRequest: User closed the viewer')); + }); + Services.wm.addListener(wmListener); + await new Promise((resolve) => { + win = Zotero.openInViewer(url, { + cookieContextId: options.cookieContextId + }); + win.addEventListener('load', resolve); + }); + browser = win.document.querySelector('browser'); + Zotero.Utilities.Internal.activate(win); + + // Poll for the captcha element disappearing, then require the page + // to stay stable for onLoadTimeout before we consider it cleared + let lastLocation = browser.currentURI.spec; + let stableSince = null; + let pollInterval = 500; + // Don't allow clearance until we've positively observed the challenge + // at least once; otherwise a transient empty/about:blank document + // during the challenge page's own loading would declare success + // before the user sees anything. + let sawChallenge = false; + await Promise.race([ + clearedDeferred.promise, + Zotero.Promise.delay(timeout).then(() => { + if (!cleared) { + cancelled = true; + throw new Error(`BrowserRequest: Viewer challenge clearance timed out after ${timeout}ms`); + } + }), + (async () => { + // Set above: + // eslint-disable-next-line no-unmodified-loop-condition + while (!cleared && !cancelled) { + await Zotero.Promise.delay(pollInterval); + let currentLocation = browser.currentURI.spec; + if (currentLocation !== lastLocation) { + lastLocation = currentLocation; + stableSince = null; + continue; + } + let stillChallenged = await this._browserMatchesSelector(browser, options.captchaLocator); + if (stillChallenged) { + sawChallenge = true; + stableSince = null; + continue; + } + if (!sawChallenge) { + // Challenge page hasn't rendered yet, or we can't read + // the DOM. Keep waiting without advancing the timer. + stableSince = null; + continue; + } + if (stableSince === null) { + stableSince = Date.now(); + } + else if (Date.now() - stableSince >= onLoadTimeout) { + cleared = true; + clearedDeferred.resolve(); + } + } + })() + ]); + } + finally { + Services.wm.removeListener(wmListener); + if (win) { + win.close(); + } + } + }, + + /** + * Ask the browser's current document whether it has a match for a CSS selector. + * False if the document can't be queried or nothing matches. + */ + async _browserMatchesSelector(browser, selector) { + try { + let actor = browser.browsingContext?.currentWindowGlobal?.getActor('PageData'); + if (!actor) return false; + return await actor.sendQuery('querySelectorMatches', { selector }); + } + catch (e) { + Zotero.logError(e); + return false; + } + }, + + /** + * Load the URL in the given HiddenBrowser and wait for the page to settle. + * Each location change restarts a `browserRequest.onLoadTimeout` window; + * if the location stays stable for that window, we consider the page + * settled. Throws if `browserRequest.timeout` elapses first. + * + * If `onPDF` is provided, also sets up a PDF MIME type handler on the + * browser and resolves early once a PDF is captured; the callback receives + * the blob. + * + * If `successCookie` is provided, polls the cookie jar and resolves as + * soon as a cookie with that name exists on the given host. + * + * @param {HiddenBrowser} hiddenBrowser + * @param {string} url + * @param {object} [opts] + * @param {(blob: Blob) => void} [opts.onPDF] + * @param {{ host: string, name: string }} [opts.successCookie] + * @param {number} [opts.cookieContextId] + * @returns {Promise} + */ + async _loadAndSettle(hiddenBrowser, url, opts = {}) { + const onLoadTimeout = Zotero.Prefs.get('browserRequest.onLoadTimeout'); + const timeout = Zotero.Prefs.get('browserRequest.timeout'); + + let settled = false; + let settleDeferred = Zotero.Promise.defer(); + let pdfDeferred = Zotero.Promise.defer(); + let cookieDeferred = Zotero.Promise.defer(); + let pdfFound = false; + let pdfHandler; + + if (opts.onPDF) { + pdfHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, (blob) => { + pdfFound = true; + opts.onPDF(blob); + pdfDeferred.resolve(); + }); + Zotero.MIMETypeHandler.addHandlers('application/pdf', pdfHandler, true); + } + + try { + let currentUrl = ''; + hiddenBrowser.webProgress.addProgressListener({ + QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), + async onLocationChange() { + let loc = hiddenBrowser.currentURI.spec; + if (currentUrl) { + Zotero.debug(`BrowserRequest: A JS redirect occurred to ${loc}`); + } + currentUrl = loc; + Zotero.debug(`BrowserRequest: Page loaded at ${loc}; waiting ${onLoadTimeout}ms for further JS activity`); + await Zotero.Promise.delay(onLoadTimeout); + if (currentUrl === loc && !settled && !pdfFound) { + settled = true; + settleDeferred.resolve(); + } + } + }, Ci.nsIWebProgress.NOTIFY_LOCATION); + + hiddenBrowser.load(url); + + let cookiePollInterval; + if (opts.successCookie) { + cookiePollInterval = this._pollForCookie({ + successCookie: opts.successCookie, + cookieContextId: opts.cookieContextId, + onFound: () => { + Zotero.debug(`BrowserRequest: successCookie ${opts.successCookie.name} appeared`); + cookieDeferred.resolve(); + } + }); + } + + let races = [ + settleDeferred.promise, + Zotero.Promise.delay(timeout).then(() => { + if (!settled && !pdfFound) { + throw new Error(`BrowserRequest: Browser request timed out after ${timeout}ms`); + } + }) + ]; + if (opts.onPDF) { + races.push(pdfDeferred.promise); + } + if (opts.successCookie) { + races.push(cookieDeferred.promise); + } + try { + await Promise.race(races); + } + finally { + if (cookiePollInterval) clearInterval(cookiePollInterval); + } + } + finally { + if (pdfHandler) { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfHandler); + } + } + }, + + _makePDFMIMETypeHandler(browser, onPDFFound = () => 0) { + let isOurPDF, channelBrowser; + let trackedBrowser = browser; + return { + onStartRequest: function (name, _, channel) { + Zotero.debug(`BrowserRequest: Sniffing a PDF loaded at ${name}`); + try { + channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; + } + catch {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + else { + try { + channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) + .topFrameElement; + } + catch {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + } + }, + onContent: async (blob, name) => { + if (isOurPDF) { + Zotero.debug(`BrowserRequest: Found our PDF at ${name}`); + onPDFFound(blob); + return true; + } + Zotero.debug(`BrowserRequest: Not our PDF at ${name}`); + return false; + } + }; + }, + + /** + * @param {String} url + * @param {String} path + * @param {Object} [options] + * @param {Boolean} [options.shouldDisplayCaptcha=false] + */ + async downloadPDF(url, path, options = {}) { + Zotero.debug(`BrowserRequest: Downloading PDF via hidden browser from ${url}`); + + let hiddenBrowser; + let blob; + try { + hiddenBrowser = new HiddenBrowser(); + await hiddenBrowser._createdPromise; + await this._loadAndSettle(hiddenBrowser, url, { + onPDF: (foundBlob) => { + blob = foundBlob; + } + }); + if (!blob) { + throw new Error('BrowserRequest: Settled without receiving a PDF'); + } + await Zotero.File.putContentsAsync(path, blob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + if (options?.shouldDisplayCaptcha) { + Zotero.debug(`BrowserRequest: Hidden browser PDF download failed: ${e.message}`); + const entry = this.getEntryForURL(url); + if (entry?.captchaLocator && hiddenBrowser) { + let doc; + try { + doc = await hiddenBrowser.getDocument(); + } + catch {} + if (doc && doc.querySelector(entry.captchaLocator)) { + return this.downloadPDFViaViewer(url, path, options); + } + } + } + throw e; + } + finally { + if (hiddenBrowser) { + hiddenBrowser.destroy(); + } + } + return undefined; + }, + + async downloadPDFViaViewer(url, path, _options) { + Zotero.debug(`BrowserRequest: Downloading PDF via viewer for captcha clearing from ${url}`); + + let win, browser, wmListener; + let pdfMIMETypeHandler; + let pdfFound; + let pdfFoundDeferred = Zotero.Promise.defer(); + const timeout = Zotero.Prefs.get('browserRequest.timeout'); + + try { + wmListener = this._makeViewerCloseListener(() => { + if (!pdfFound) pdfFoundDeferred.reject(new Error('BrowserRequest: User closed the viewer')); + }); + Services.wm.addListener(wmListener); + await new Promise((resolve) => { + win = Zotero.openInViewer(url); + win.addEventListener('load', resolve); + }); + browser = win.document.querySelector('browser'); + Zotero.Utilities.Internal.activate(win); + + pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve); + Zotero.MIMETypeHandler.addHandlers('application/pdf', pdfMIMETypeHandler, true); + + Zotero.debug(`BrowserRequest: Awaiting user captcha clearance or timeout after ${timeout}ms`); + let pdfBlob = await Promise.race([ + Zotero.Promise.delay(timeout).then(() => { + if (!pdfFound) { + throw new Error(`BrowserRequest: Viewer PDF download timed out after ${timeout}ms`); + } + }), + pdfFoundDeferred.promise + ]); + pdfFound = true; + await Zotero.File.putContentsAsync(path, pdfBlob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + throw e; + } + finally { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); + Services.wm.removeListener(wmListener); + if (win) { + win.close(); + } + } + }, +}; + +// Register hosts that we intercept Cloudflare Turnstile challenges on, +// so they receive a plain UA everywhere. See comment on +// Zotero.VersionHeader.registerPlainUAHost(). +for (let entry of Zotero.BrowserRequest.CHALLENGE_URLS) { + if (entry.plainUAHost) { + Zotero.VersionHeader.registerPlainUAHost(entry.plainUAHost); + } +} diff --git a/chrome/content/zotero/xpcom/http.js b/chrome/content/zotero/xpcom/http.js index 0065c3ac3f..a99eba3d2c 100644 --- a/chrome/content/zotero/xpcom/http.js +++ b/chrome/content/zotero/xpcom/http.js @@ -298,8 +298,8 @@ Zotero.HTTP = new function () { Zotero.debug("HTTP " + method + " " + dispURL); } - // Translation framework uses cookieSandbox - if (options.cookieSandbox && typeof options.cookieSandbox === 'string') { + // Translation framework uses cookieSandbox to hold userContextId number + if (typeof options.cookieSandbox === 'number') { options.cookieContextId = options.cookieSandbox; delete options.cookieSandbox; } diff --git a/chrome/content/zotero/xpcom/prefs.js b/chrome/content/zotero/xpcom/prefs.js index 66d6d69eb3..bdbd864331 100644 --- a/chrome/content/zotero/xpcom/prefs.js +++ b/chrome/content/zotero/xpcom/prefs.js @@ -46,7 +46,7 @@ Zotero.Prefs = new function () { // Process pref version updates var fromVersion = this.get('prefVersion'); - var toVersion = 21; + var toVersion = 22; if (!fromVersion) { this.set('prefVersion', toVersion); } @@ -220,6 +220,21 @@ Zotero.Prefs = new function () { case 21: this.set('firstRunGuidanceShown.readAloud', false); break; + + // downloadPDFViaBrowser.* -> browserRequest.*; + // module was generalized to handle more than PDF downloads + case 22: + if (this.prefHasUserValue('downloadPDFViaBrowser.onLoadTimeout')) { + this.set('browserRequest.onLoadTimeout', + this.get('downloadPDFViaBrowser.onLoadTimeout')); + this.clear('downloadPDFViaBrowser.onLoadTimeout'); + } + if (this.prefHasUserValue('downloadPDFViaBrowser.downloadTimeout')) { + this.set('browserRequest.timeout', + this.get('downloadPDFViaBrowser.downloadTimeout')); + this.clear('downloadPDFViaBrowser.downloadTimeout'); + } + break; } } this.set('prefVersion', toVersion); diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index 3774486067..d463d782ff 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -1238,6 +1238,10 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte * @param {Object} [options] * @param {Function} [options.onLoad] - Function to run once URI is loaded; passed the loaded document * @param {Boolean} [options.allowJavaScript] - Set to false to disable JavaScript + * @param {Number} [options.cookieContextId] - userContextId to isolate the viewer's cookies + * into the same jar as a Zotero.HTTP.request or HiddenBrowser using the same ID + * @param {String} [options.customUserAgent] - Override the User-Agent for all requests + * from this viewer's browsing context */ this.openInViewer = function (uri, options) { if (options && !options.onLoad && typeof options === 'function') { @@ -1247,7 +1251,7 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte var viewerWins = Services.wm.getEnumerator("zotero:basicViewer"); for (let existingWin of viewerWins) { - if (existingWin.viewerOriginalURI === uri) { + if (existingWin.viewerOriginalURI === uri && existingWin.viewerCookieContextId === options?.cookieContextId) { existingWin.focus(); return existingWin; } @@ -2101,6 +2105,10 @@ Zotero.Keys = new function () { * @namespace */ Zotero.VersionHeader = { + _plainUAHosts: new Set(), + _uaAppSuffixRe: null, + _uaFirefoxComponent: null, + init: function () { this.register(); Zotero.addShutdownListener(this.unregister); @@ -2126,7 +2134,9 @@ Zotero.VersionHeader = { let isAppNameDomain = s3RE.test(domain); if (!isAppNameDomain) { let ua = channel.getRequestHeader('User-Agent'); - ua = this.update(ua); + ua = this.update(ua, { + mode: this._plainUAHosts.has(domain) ? 'plain' : 'full', + }); channel.setRequestHeader('User-Agent', ua, false); } } @@ -2137,23 +2147,57 @@ Zotero.VersionHeader = { }, /** - * Add Firefox/[version] to the default user agent + * Register a host that needs the "Zotero/[version]" component stripped + * from its requests' UA. Currently this is only used for hosts that we + * handle Cloudflare Turnstile challenges on; Turnstile won't pass with + * Zotero/ in the UA string, and future requests need the same UA as the + * one that passed Turnstile, so we have to override for all requests to + * the host. * - * @param {String} ua - User Agent + * @param {string} host */ - update: function (ua) { + registerPlainUAHost: function (host) { + this._plainUAHosts.add(host); + }, + + /** + * @param {String} ua + * @param {'full' | 'plain'} [mode='full'] If 'full', add Firefox/[version] to the default user agent. If 'plain', remove + * Zotero/[version] instead. + * @return {String} + */ + update: function (ua, { mode = 'full' } = {}) { var info = Services.appinfo; - var appName = info.name; - - var pos = ua.indexOf(appName + '/'); - - // Default UA (not a faked UA from the connector) - if (pos != -1) { - ua = ua.substring(0, pos) + `Firefox/${info.platformVersion.match(/^\d+/)[0]}.0 ` + ua.substring(pos); + if (!this._uaAppSuffixRe) { + this._uaAppSuffixRe = new RegExp(`\\s*${info.name}/\\S+`); + this._uaFirefoxComponent = `Firefox/${info.platformVersion.match(/^\d+/)[0]}.0`; + } + if (mode === 'plain') { + ua = ua.replace(this._uaAppSuffixRe, ''); + if (!/\bFirefox\//.test(ua)) { + ua += ` ${this._uaFirefoxComponent}`; + } + } + else { + let pos = ua.indexOf(info.name + '/'); + // Default UA (not a faked UA from the connector) + if (pos != -1) { + ua = ua.substring(0, pos) + `${this._uaFirefoxComponent} ` + ua.substring(pos); + } } - return ua; }, + + /** + * Plain Firefox UA, without the "Zotero/[version]" suffix. + * + * @return {String} + */ + getPlainFirefoxUA: function () { + var ua = Cc["@mozilla.org/network/protocol;1?name=http"] + .getService(Ci.nsIHttpProtocolHandler).userAgent; + return this.update(ua, { mode: 'plain' }); + }, unregister: function () { Services.obs.removeObserver(Zotero.VersionHeader, "http-on-modify-request"); diff --git a/chrome/content/zotero/zotero.mjs b/chrome/content/zotero/zotero.mjs index a8e4649e73..a4e359db8e 100644 --- a/chrome/content/zotero/zotero.mjs +++ b/chrome/content/zotero/zotero.mjs @@ -71,7 +71,7 @@ const xpcomFilesLocal = [ 'api', 'attachments', 'attachmentReadObserver', - 'browserDownload', + 'browserRequest', 'cite', 'citeprocRsBridge', 'data/library', diff --git a/defaults/preferences/zotero.js b/defaults/preferences/zotero.js index 6e3edba9ff..159f152bc9 100644 --- a/defaults/preferences/zotero.js +++ b/defaults/preferences/zotero.js @@ -87,8 +87,8 @@ pref("extensions.zotero.itemPaneHeader.bibEntry.locale", ""); pref("extensions.zotero.tagSelector.showAutomatic", true); pref("extensions.zotero.tagSelector.displayAllTags", false); -pref("extensions.zotero.downloadPDFViaBrowser.onLoadTimeout", 3000); -pref("extensions.zotero.downloadPDFViaBrowser.downloadTimeout", 60000); +pref("extensions.zotero.browserRequest.onLoadTimeout", 3000); +pref("extensions.zotero.browserRequest.timeout", 60000); // Keyboard shortcuts pref("extensions.zotero.keys.saveToZotero", "S"); diff --git a/test/tests/attachmentsTest.js b/test/tests/attachmentsTest.js index 5fc9592967..80225102c3 100644 --- a/test/tests/attachmentsTest.js +++ b/test/tests/attachmentsTest.js @@ -326,8 +326,8 @@ describe("Zotero.Attachments", function () { describe("#importFromURL()", function () { - it("should use BrowserDownload for a JS redirect page", async function () { - let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDF"); + it("should use BrowserRequest for a JS redirect page", async function () { + let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDF"); downloadPDFStub.callsFake(async (_url, path) => { await OS.File.copy(OS.Path.join(getTestDataDirectory().path, 'test.pdf'), path); }); @@ -623,7 +623,7 @@ describe("Zotero.Attachments", function () { }); }); - it("should use BrowserDownload for 403 when enforcing file type", async function () { + it("should use BrowserRequest for 403 when enforcing file type", async function () { let prefix = Zotero.Utilities.randomString(); let testServerPath = 'http://127.0.0.1:' + testServerPort + '/' + prefix; let pdfURL = testServerPath + '/test.pdf'; @@ -638,9 +638,9 @@ describe("Zotero.Attachments", function () { ); let path = OS.Path.join(Zotero.getTempDirectory().path, 'test.pdf'); - let shouldAttemptStub = sinon.stub(Zotero.BrowserDownload, "shouldAttemptDownloadViaBrowser"); - let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDF"); - shouldAttemptStub.returns(true); + let getEntryStub = sinon.stub(Zotero.BrowserRequest, "getEntryForURL"); + let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDF"); + getEntryStub.returns({ match: 'test' }); downloadPDFStub.callsFake(async (_url, path) => { await OS.File.copy(OS.Path.join(getTestDataDirectory().path, 'test.pdf'), path); }); @@ -648,14 +648,14 @@ describe("Zotero.Attachments", function () { try { item = await Zotero.Attachments.downloadFile(pdfURL, path, { enforceFileType: true }); - assert.isTrue(shouldAttemptStub.calledOnce); + assert.isTrue(getEntryStub.calledOnce); assert.isTrue(downloadPDFStub.calledOnce); } finally { // Clean up if (item) await Zotero.Items.erase(item.id); downloadPDFStub.restore(); - shouldAttemptStub.restore(); + getEntryStub.restore(); } }); }); diff --git a/test/tests/browserDownloadTest.js b/test/tests/browserDownloadTest.js deleted file mode 100644 index 4ceef5c7a3..0000000000 --- a/test/tests/browserDownloadTest.js +++ /dev/null @@ -1,78 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2023 Corporation for Digital Scholarship - Vienna, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - -describe("Zotero.BrowserDownload", function () { - describe("#downloadPDF()", function () { - var http, port, baseURL; - var tmpFile = Zotero.getTempDirectory(); - tmpFile.append('browserDownloadTest.pdf'); - - before(async function () { - ({ httpd, port } = await startHTTPServer()); - baseURL = `http://localhost:${port}/`; - }); - - after(async function () { - await new Promise(resolve => httpd.stop(resolve)); - }); - - it("#downloadPDF() should download a PDF from a JS redirect page", async function () { - var dir = getTestDataDirectory().path; - httpd.registerFile( - '/test-pdf-redirect.html', - Zotero.File.pathToFile(PathUtils.join(dir, 'test-pdf-redirect.html')) - ); - httpd.registerFile( - '/test.pdf', - Zotero.File.pathToFile(PathUtils.join(dir, 'test.pdf')) - ); - - await Zotero.BrowserDownload.downloadPDF(`${baseURL}test-pdf-redirect.html`, tmpFile.path); - - var sample = await Zotero.File.getContentsAsync(tmpFile, null, 1000); - assert.equal(Zotero.MIME.sniffForMIMEType(sample), 'application/pdf'); - }); - - // Needs a js-redirect delay in test-pdf-redirect.html - it.skip("should display a viewer to clear a captcha if detected", async function () { - // Make it so that downloadPDF() times out with a hidden browser, which simulates running into a captcha - Zotero.Prefs.set('downloadPDFViaBrowser.downloadTimeout', 10); - let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDFViaViewer"); - - let promise = Zotero.BrowserDownload.downloadPDF('https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', tmpFile.path, - { shouldDisplayCaptcha: true }); - await new Promise(resolve => downloadPDFStub.callsFake((...args) => { - resolve(); - Zotero.Prefs.set('downloadPDFViaBrowser.downloadTimeout', 60e3); - return downloadPDFStub.wrappedMethod.call(Zotero.BrowserDownload, ...args); - })); - - await promise; - - assert.isTrue(downloadPDFStub.calledOnce); - downloadPDFStub.restore(); - }); - }); -}); diff --git a/test/tests/browserRequestTest.js b/test/tests/browserRequestTest.js new file mode 100644 index 0000000000..a1b81d6b58 --- /dev/null +++ b/test/tests/browserRequestTest.js @@ -0,0 +1,290 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +describe("Zotero.BrowserRequest", function () { + describe("#downloadPDF()", function () { + var httpd, port, baseURL; + var tmpFile = Zotero.getTempDirectory(); + tmpFile.append('browserRequestTest.pdf'); + + before(async function () { + ({ httpd, port } = await startHTTPServer()); + baseURL = `http://localhost:${port}/`; + }); + + after(async function () { + await new Promise(resolve => httpd.stop(resolve)); + }); + + it("should download a PDF from a JS redirect page", async function () { + var dir = getTestDataDirectory().path; + httpd.registerFile( + '/test-pdf-redirect.html', + Zotero.File.pathToFile(PathUtils.join(dir, 'test-pdf-redirect.html')) + ); + httpd.registerFile( + '/test.pdf', + Zotero.File.pathToFile(PathUtils.join(dir, 'test.pdf')) + ); + + await Zotero.BrowserRequest.downloadPDF(`${baseURL}test-pdf-redirect.html`, tmpFile.path); + + var sample = await Zotero.File.getContentsAsync(tmpFile, null, 1000); + assert.equal(Zotero.MIME.sniffForMIMEType(sample), 'application/pdf'); + }); + + // Needs a js-redirect delay in test-pdf-redirect.html + it.skip("should display a viewer to clear a captcha if detected", async function () { + // Force downloadPDF() to time out with a hidden browser, which simulates running into a captcha + Zotero.Prefs.set('browserRequest.timeout', 10); + let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDFViaViewer"); + + let promise = Zotero.BrowserRequest.downloadPDF('https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', tmpFile.path, + { shouldDisplayCaptcha: true }); + await new Promise(resolve => downloadPDFStub.callsFake((...args) => { + resolve(); + Zotero.Prefs.set('browserRequest.timeout', 60e3); + return downloadPDFStub.wrappedMethod.call(Zotero.BrowserRequest, ...args); + })); + + await promise; + + assert.isTrue(downloadPDFStub.calledOnce); + downloadPDFStub.restore(); + }); + }); + + describe("#getEntryForURL()", function () { + it("returns null for unrecognized URLs", function () { + assert.isNull(Zotero.BrowserRequest.getEntryForURL('https://example.com/foo')); + }); + + it("matches registered substrings", function () { + let entry = Zotero.BrowserRequest.getEntryForURL( + 'https://search.worldcat.org/api/search?q=bn%3A123' + ); + assert.isNotNull(entry); + assert.equal(entry.match, '://search.worldcat.org'); + }); + + it("sniffs response bodies via detectBlock", function () { + let entry = Zotero.BrowserRequest.getEntryForURL( + 'https://search.worldcat.org/api/search?q=bn%3A123' + ); + assert.isFunction(entry.detectBlock); + assert.isTrue(entry.detectBlock(403, '{"turnstile_required":true}')); + assert.isFalse(entry.detectBlock(403, 'some other 403 body')); + assert.isFalse(entry.detectBlock(200, '{"turnstile_required":true}')); + }); + }); + + describe("#_pollForCookie()", function () { + it("ignores a stale cookie present at poll start", async function () { + // Simulate a leftover cookie from a previous session whose HMAC + // is no longer valid: _pollForCookie must wait for the value to + // change, not declare success on first read. + let readStub = sinon.stub(Zotero.BrowserRequest, "_readCookieValue"); + readStub.returns("stale-value"); + + let onFound = sinon.spy(); + let handle = Zotero.BrowserRequest._pollForCookie({ + successCookie: { host: 'example.com', name: 'c' }, + onFound, + intervalMs: 10 + }); + + try { + await Zotero.Promise.delay(40); + assert.isFalse(onFound.called, "should not fire while value is unchanged"); + + readStub.returns("fresh-value"); + await Zotero.Promise.delay(40); + assert.isTrue(onFound.called, "should fire once a new value appears"); + } + finally { + clearInterval(handle); + readStub.restore(); + } + }); + + it("fires when the cookie first appears from nothing", async function () { + let readStub = sinon.stub(Zotero.BrowserRequest, "_readCookieValue"); + readStub.returns(null); + + let onFound = sinon.spy(); + let handle = Zotero.BrowserRequest._pollForCookie({ + successCookie: { host: 'example.com', name: 'c' }, + onFound, + intervalMs: 10 + }); + + try { + await Zotero.Promise.delay(40); + assert.isFalse(onFound.called); + + readStub.returns("new-value"); + await Zotero.Promise.delay(40); + assert.isTrue(onFound.called); + } + finally { + clearInterval(handle); + readStub.restore(); + } + }); + }); + + describe("translator request retry", function () { + function makeUtils() { + let fakeTranslate = { + resolveURL: url => url, + requestHeaders: {}, + cookieSandbox: undefined + }; + return new Zotero.Utilities.Translate(fakeTranslate); + } + + it("retries via clearChallenge when the initial request throws a registered 403", async function () { + let url = 'https://search.worldcat.org/api/search?q=bn%3A978-0-585-03015-9'; + let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves(); + let call = 0; + let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => { + call++; + if (call === 1) { + let xhr = { + status: 403, + response: '{"turnstile_required":true}', + responseText: '{"turnstile_required":true}', + channel: null + }; + throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden"); + } + return { + status: 200, + response: { ok: true }, + responseText: '{"ok":true}', + getAllResponseHeaders: () => '' + }; + }); + + try { + let utils = makeUtils(); + let result = await utils.request(url); + assert.equal(result.status, 200); + assert.deepEqual(result.body, { ok: true }); + assert.equal(requestStub.callCount, 2); + assert.isTrue(clearStub.calledOnce); + assert.equal(clearStub.firstCall.args[0], 'https://search.worldcat.org/search?q=bn%3A978-0-585-03015-9'); + } + finally { + requestStub.restore(); + clearStub.restore(); + } + }); + + it("retries when the 403's XHR is in responseType=json mode (responseText throws)", async function () { + let url = 'https://search.worldcat.org/api/search?q=bn%3A978-0-585-03015-9'; + let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves(); + let call = 0; + let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => { + call++; + if (call === 1) { + let xhr = { + status: 403, + response: { turnstile_required: true }, + get responseText() { throw new Error('responseText unavailable'); }, + channel: null + }; + throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden"); + } + return { + status: 200, + response: { ok: true }, + responseText: '{"ok":true}', + getAllResponseHeaders: () => '' + }; + }); + + try { + let utils = makeUtils(); + let result = await utils.request(url, { responseType: 'json' }); + assert.equal(result.status, 200); + assert.deepEqual(result.body, { ok: true }); + assert.equal(requestStub.callCount, 2); + assert.isTrue(clearStub.calledOnce); + } + finally { + requestStub.restore(); + clearStub.restore(); + } + }); + + it("does not retry when the URL is not registered", async function () { + let url = 'https://example.com/api/search'; + let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves(); + let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => { + let xhr = { status: 403, response: '', responseText: '', channel: null }; + throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden"); + }); + + try { + let utils = makeUtils(); + let err; + try { await utils.request(url); } + catch (e) { err = e; } + assert.instanceOf(err, Zotero.HTTP.UnexpectedStatusException); + assert.equal(requestStub.callCount, 1); + assert.isTrue(clearStub.notCalled); + } + finally { + requestStub.restore(); + clearStub.restore(); + } + }); + + it("does not retry in browser on a 200 body that matches detectBlock (status must also match)", async function () { + let url = 'https://search.worldcat.org/api/search?q=bn%3A1'; + let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves(); + let requestStub = sinon.stub(Zotero.HTTP, "request").returns({ + status: 200, + response: '{"turnstile_required":true}', + responseText: '{"turnstile_required":true}', + getAllResponseHeaders: () => '' + }); + + try { + let utils = makeUtils(); + // WorldCat's detectBlock requires status === 403; a 200 with the same + // body should not trigger a retry. + await utils.request(url, { responseType: 'text' }); + assert.equal(requestStub.callCount, 1); + assert.isTrue(clearStub.notCalled); + } + finally { + requestStub.restore(); + clearStub.restore(); + } + }); + }); +});