From 8b77c96e9726b9f9729f8e32d3c8fa0674d4af98 Mon Sep 17 00:00:00 2001 From: Adomas Ven Date: Wed, 27 Dec 2023 11:43:50 +0200 Subject: [PATCH] Displays a browser window to clear captcha when saving attachments. (#3526) - Currently enabled only for ScienceDirect. Can be enabled via a whitelist - Matches the HiddenBrowser loaded HTML page for a captcha element. If the captcha element class changes, this will break (but the alternative is potentially displaying a captcha clearing window when something else that is not a captcha guard is loaded). - Captcha clear timeout for 60s. - Doesn't automatically switch focus back to the browser which intiated the item save via the Connector. - Stores the cookies used to clear the captcha for future saves from the same domain. Discards Connector supplied User Agent, since CF bot detector checks UA header against actual UA behavior like TLS handshake and if the UA acts different to what it's supposed to, the bot challenge is not cleared. Other changes: - Adjusted the cookie sandbox to allow multiple cookie sandboxes to be active (and simplified some legacy code that was meant to cover a bug in old FX codebase). - HiddenBrowser API changed to be Object oriented, translator tester in the translate repo will need to be updated after a merge (have the change ready). - Improved Connector Server attachment progress handling --- chrome/content/zotero/HiddenBrowser.jsm | 169 ++++++----- .../content/zotero/standalone/basicViewer.js | 3 + chrome/content/zotero/xpcom/attachments.js | 127 +-------- .../content/zotero/xpcom/browserDownload.js | 267 ++++++++++++++++++ .../xpcom/connector/server_connector.js | 41 ++- chrome/content/zotero/xpcom/cookieSandbox.js | 118 ++++---- chrome/content/zotero/xpcom/data/feedItem.js | 5 +- chrome/content/zotero/xpcom/fulltext.js | 7 +- chrome/content/zotero/xpcom/zotero.js | 4 +- components/zotero-service.js | 1 + test/tests/HiddenBrowserTest.js | 52 ++-- test/tests/RemoteTranslateTest.js | 35 ++- test/tests/attachmentsTest.js | 51 ++-- test/tests/browserDownloadTest.js | 60 ++++ test/tests/server_connectorTest.js | 2 +- test/tests/translateTest.js | 7 +- 16 files changed, 607 insertions(+), 342 deletions(-) create mode 100644 chrome/content/zotero/xpcom/browserDownload.js create mode 100644 test/tests/browserDownloadTest.js diff --git a/chrome/content/zotero/HiddenBrowser.jsm b/chrome/content/zotero/HiddenBrowser.jsm index 5fbc2447ea..3e532fd5bb 100644 --- a/chrome/content/zotero/HiddenBrowser.jsm +++ b/chrome/content/zotero/HiddenBrowser.jsm @@ -50,22 +50,85 @@ ChromeUtils.registerWindowActor("SingleFile", { }); const progressListeners = new Set(); -const browserFrameMap = new WeakMap(); /** * Functions for creating and destroying hidden browser objects **/ -const HiddenBrowser = { +class HiddenBrowser { /** - * @param {String} source - HTTP URL, file: URL, or file path * @param {Object} options * @param {Boolean} [options.allowJavaScript] * @param {Object} [options.docShell] Fields to set on Browser.docShell - * @param {Boolean} [options.requireSuccessfulStatus] * @param {Boolean} [options.blockRemoteResources] Block all remote (non-file:) resources * @param {Zotero.CookieSandbox} [options.cookieSandbox] */ - async create(source, options = {}) { + constructor(options = {}) { + var frame = new HiddenFrame(); + this._createdPromise = (async () => { + var windowlessBrowser = await frame.get(); + windowlessBrowser.browsingContext.allowJavascript = options.allowJavaScript !== false; + windowlessBrowser.docShell.allowImages = false; + if (options.docShell) { + Object.assign(windowlessBrowser.docShell, options.docShell); + } + var doc = windowlessBrowser.document; + var browser = doc.createXULElement("browser"); + browser.setAttribute("type", "content"); + browser.setAttribute("remote", "true"); + browser.setAttribute('maychangeremoteness', 'true'); + browser.setAttribute("disableglobalhistory", "true"); + doc.documentElement.appendChild(browser); + + if (options.cookieSandbox) { + options.cookieSandbox.attachToBrowser(browser); + } + + if (Zotero.Debug.enabled) { + let weakBrowser = new WeakRef(browser); + setTimeout(() => { + let browser = weakBrowser.deref(); + if (browser && browserFrameMap.has(browser)) { + Zotero.debug('Browser object still alive after 60 seconds - memory leak?'); + Zotero.debug('Viewing URI ' + browser.currentURI?.spec) + } + }, 1000 * 60); + } + + if (options.blockRemoteResources) { + RemoteResourceBlockingObserver.watch(browser); + } + + this._browser = browser; + })(); + + this._frame = frame; + return new Proxy(this, { + get(target, prop) { + if (prop in target) { + return target[prop]; + } + if (!target._browser) throw new Error(`Attempting to use the HiddenBrowser before it is fully initialized. Await browser._createdPromise.`); + return Reflect.get(target._browser, prop); + }, + set(target, prop, val) { + if (prop in target) { + target[prop] = val; + } + Reflect.set(target._browser, prop, val) + return true; + } + }); + } + + /** + * + * @param {String} source - HTTP URL, file: URL, or file path + * @param {Object} options + * @param {Boolean} [options.requireSuccessfulStatus] + * @returns {Promise} + */ + async load(source, options) { + await this._createdPromise; let url; if (/^(file|https?|chrome|resource):/.test(source)) { url = source; @@ -74,45 +137,8 @@ const HiddenBrowser = { else { url = Zotero.File.pathToFileURI(source); } - + Zotero.debug(`Loading ${url} in hidden browser`); - - var frame = new HiddenFrame(); - var windowlessBrowser = await frame.get(); - windowlessBrowser.browsingContext.allowJavascript = options.allowJavaScript !== false; - windowlessBrowser.docShell.allowImages = false; - if (options.docShell) { - Object.assign(windowlessBrowser.docShell, options.docShell); - } - var doc = windowlessBrowser.document; - var browser = doc.createXULElement("browser"); - browser.setAttribute("type", "content"); - browser.setAttribute("remote", "true"); - browser.setAttribute('maychangeremoteness', 'true'); - browser.setAttribute("disableglobalhistory", "true"); - doc.documentElement.appendChild(browser); - - if (options.cookieSandbox) { - options.cookieSandbox.attachToBrowser(browser); - } - - browserFrameMap.set(browser, frame); - - if (Zotero.Debug.enabled) { - let weakBrowser = new WeakRef(browser); - setTimeout(() => { - let browser = weakBrowser.deref(); - if (browser && browserFrameMap.has(browser)) { - Zotero.debug('Browser object still alive after 60 seconds - memory leak?'); - Zotero.debug('Viewing URI ' + browser.currentURI?.spec) - } - }, 1000 * 60); - } - - if (options.blockRemoteResources) { - RemoteResourceBlockingObserver.watch(browser); - } - // Next bit adapted from Mozilla's HeadlessShell.jsm const principal = Services.scriptSecurityManager.getSystemPrincipal(); try { @@ -122,7 +148,7 @@ const HiddenBrowser = { reject(new Error("Page never loaded in hidden browser")); }, 5000); - let oa = E10SUtils.predictOriginAttributes({ browser }); + let oa = E10SUtils.predictOriginAttributes({ browser: this }); let loadURIOptions = { triggeringPrincipal: principal, remoteType: E10SUtils.getRemoteTypeForURI( @@ -134,8 +160,8 @@ const HiddenBrowser = { oa ) }; - browser.loadURI(url, loadURIOptions); - let { webProgress } = browser; + this.loadURI(url, loadURIOptions); + let { webProgress } = this; let progressListener = { onLocationChange(progress, request, location, flags) { @@ -173,11 +199,13 @@ const HiddenBrowser = { return false; } - if (options.requireSuccessfulStatus) { - let { channelInfo } = await this.getPageData(browser, ['channelInfo']); + if (options?.requireSuccessfulStatus) { + let { channelInfo } = await this.getPageData(['channelInfo']); if (channelInfo && (channelInfo.responseStatus < 200 || channelInfo.responseStatus >= 400)) { let response = `${channelInfo.responseStatus} ${channelInfo.responseStatusText}`; - Zotero.debug(`HiddenBrowser.create: ${url} failed with ${response}`, 2); + Zotero.debug(`HiddenBrowser.load: ${url} failed with ${response}`, 2); + // HiddenBrowser will never get returned so we need to clean it up here + this.destroy() throw new Zotero.HTTP.UnexpectedStatusException( { status: channelInfo.responseStatus @@ -187,31 +215,27 @@ const HiddenBrowser = { ); } } - - return browser; - }, + } /** - * @param {Browser} browser * @param {String[]} props - 'characterSet', 'title', 'bodyText', 'documentHTML', 'cookie', 'channelInfo' */ - async getPageData(browser, props) { - var actor = browser.browsingContext.currentWindowGlobal.getActor("PageData"); + async getPageData(props) { + var actor = this.browsingContext.currentWindowGlobal.getActor("PageData"); var data = {}; for (let prop of props) { data[prop] = await actor.sendQuery(prop); } return data; - }, + } /** - * @param {Browser} browser * @returns {Promise} */ - async getDocument(browser) { - let { documentHTML, cookie } = await this.getPageData(browser, ['documentHTML', 'cookie']); + async getDocument() { + let { documentHTML, cookie } = await this.getPageData(['documentHTML', 'cookie']); let doc = new DOMParser().parseFromString(documentHTML, 'text/html'); - let docWithLocation = Zotero.HTTP.wrapDocument(doc, browser.currentURI); + let docWithLocation = Zotero.HTTP.wrapDocument(doc, this.currentURI); return new Proxy(docWithLocation, { get(obj, prop) { if (prop === 'cookie') { @@ -220,24 +244,25 @@ const HiddenBrowser = { return obj[prop]; } }); - }, + } /** - * @param {Browser} browser * @returns {Promise} */ - snapshot(browser) { - let actor = browser.browsingContext.currentWindowGlobal.getActor("SingleFile"); + snapshot() { + let actor = this.browsingContext.currentWindowGlobal.getActor("SingleFile"); return actor.sendQuery('snapshot'); - }, + } - destroy(browser) { - var frame = browserFrameMap.get(browser); - if (frame) { - RemoteResourceBlockingObserver.unwatch(browser); - frame.destroy(); - Zotero.debug("Deleted hidden browser"); - browserFrameMap.delete(browser); + destroy() { + if (this._frame) { + (async () => { + await this._createdPromise; + RemoteResourceBlockingObserver.unwatch(this); + this._frame.destroy(); + this._frame = null; + Zotero.debug("Deleted hidden browser"); + })(); } } }; diff --git a/chrome/content/zotero/standalone/basicViewer.js b/chrome/content/zotero/standalone/basicViewer.js index f2bb443054..575d4e7a9d 100644 --- a/chrome/content/zotero/standalone/basicViewer.js +++ b/chrome/content/zotero/standalone/basicViewer.js @@ -90,6 +90,9 @@ function loadURI(uri, options = {}) { else { browser.browsingContext.sandboxFlags |= SANDBOXED_SCRIPTS; } + if (options.cookieSandbox) { + options.cookieSandbox.attachToBrowser(browser); + } browser.loadURI( uri, { diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index 01393f551d..7b41b99631 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -544,11 +544,11 @@ Zotero.Attachments = new function () { var nativeHandlerImport = async function () { let browser; try { - browser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, + browser = new HiddenBrowser({ docShell: { allowImages: true }, cookieSandbox, }); + await browser.load(url, { requireSuccessfulStatus: true }); return await Zotero.Attachments.importFromDocument({ libraryID, browser, @@ -563,7 +563,7 @@ Zotero.Attachments = new function () { throw e; } finally { - if (browser) HiddenBrowser.destroy(browser); + if (browser) browser.destroy(); } }; @@ -597,7 +597,8 @@ Zotero.Attachments = new function () { { cookieSandbox, referrer, - isPDF: contentType == 'application/pdf' + isPDF: contentType == 'application/pdf', + shouldDisplayCaptcha: true } ); @@ -898,7 +899,7 @@ Zotero.Attachments = new function () { if (browser) { // If we have a full hidden browser, use SingleFile Zotero.debug('Getting snapshot with HiddenBrowser.snapshot()'); - let snapshotContent = yield HiddenBrowser.snapshot(browser); + let snapshotContent = yield browser.snapshot(); // Write main HTML file to disk yield Zotero.File.putContentsAsync(tmpFile, snapshotContent); @@ -1082,6 +1083,7 @@ Zotero.Attachments = new function () { * @param {Object} [options.cookieSandbox] * @param {String} [options.referrer] * @param {Boolean} [options.isPDF] - Delete file if not PDF + * @param {Boolean} [options.shouldDisplayCaptcha] */ this.downloadFile = async function (url, path, options = {}) { Zotero.debug(`Downloading file from ${url}`); @@ -1118,123 +1120,13 @@ Zotero.Attachments = new function () { // Custom handling for PDFs that are bot-guarded // via a JS-redirect if (enforcingPDF && e instanceof this.InvalidPDFException) { - const downloadViaBrowserList = [ - 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', - '://www.sciencedirect.com', - ]; - const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); - for (let unproxiedUrl of unproxiedUrls) { - if (downloadViaBrowserList.some(checkUrl => unproxiedUrl.includes(checkUrl))) { - return this.downloadPDFViaBrowser(url, path, options); - } + if (Zotero.BrowserDownload.shouldAttemptDownloadViaBrowser(url)) { + return Zotero.BrowserDownload.downloadPDF(url, path, options); } } throw e; } }; - - /** - * @param {String} url - * @param {String} path - * @param {Object} [options] - * @param {Object} [options.cookieSandbox] - */ - this.downloadPDFViaBrowser = async function (url, path, options = {}) { - Zotero.debug(`downloadPDFViaBrowser: Downloading file via browser from ${url}`); - const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout'); - // Technically this is not a download, but the full operation timeout - const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); - let channelBrowser, hiddenBrowser; - let hiddenBrowserPDFFoundDeferred = Zotero.Promise.defer(); - - let isOurPDF = false; - var pdfMIMETypeHandler = { - onStartRequest: function (name, _, channel) { - Zotero.debug(`downloadPDFViaBrowser: Sniffing a PDF loaded at ${name}`); - // try the browser - try { - channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; - } - catch (e) {} - if (channelBrowser) { - isOurPDF = hiddenBrowser === channelBrowser; - } - else { - // try the document for the load group - try { - channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) - .topFrameElement; - } - catch(e) {} - if (channelBrowser) { - isOurPDF = hiddenBrowser === channelBrowser; - } - } - }, - onContent: async (blob, name, _, channel) => { - if (isOurPDF) { - Zotero.debug(`downloadPDFViaBrowser: Found our PDF at ${name}`); - await Zotero.File.putContentsAsync(path, blob); - hiddenBrowserPDFFoundDeferred.resolve(); - return true; - } - else { - Zotero.debug(`downloadPDFViaBrowser: Not our PDF at ${name}`); - return false; - } - } - }; - try { - Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); - hiddenBrowser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, - cookieSandbox: options.cookieSandbox, - }); - let onLoadTimeoutDeferred = Zotero.Promise.defer(); - let currentUrl = ""; - hiddenBrowser.webProgress.addProgressListener({ - QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), - async onLocationChange() { - let url = hiddenBrowser.currentURI.spec; - if (currentUrl) { - Zotero.debug(`downloadPDFViaBrowser: A JS redirect occurred to ${url}`); - } - currentUrl = url; - Zotero.debug(`downloadPDFViaBrowser: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); - await Zotero.Promise.delay(onLoadTimeout); - // If URL changed that means we got redirected and the onLoadTimeout needs to restart - if (currentUrl === url && !isOurPDF) { - onLoadTimeoutDeferred.reject(new Error(`downloadPDFViaBrowser: Loading PDF via browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); - } - } - }, Ci.nsIWebProgress.NOTIFY_LOCATION); - await Zotero.Promise.race([ - onLoadTimeoutDeferred.promise, - Zotero.Promise.delay(downloadTimeout).then(() => { - if (!isOurPDF) { - throw new Error(`downloadPDFViaBrowser: Loading PDF via browser timed out after ${downloadTimeout}ms`); - } - }), - hiddenBrowserPDFFoundDeferred.promise - ]); - } - catch (e) { - try { - await OS.File.remove(path, { ignoreAbsent: true }); - } - catch (err) { - Zotero.logError(err); - } - throw e; - } - finally { - Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); - if (hiddenBrowser) { - HiddenBrowser.destroy(hiddenBrowser); - } - } - }; - /** * Make sure a file is a PDF @@ -1845,6 +1737,7 @@ Zotero.Attachments = new function () { tmpFile, { isPDF: true, + shouldDisplayCaptcha: true, onAccessMethodStart: options.onAccessMethodStart, onBeforeRequest: options.onBeforeRequest, onRequestError: options.onRequestError diff --git a/chrome/content/zotero/xpcom/browserDownload.js b/chrome/content/zotero/xpcom/browserDownload.js new file mode 100644 index 0000000000..3d973755b3 --- /dev/null +++ b/chrome/content/zotero/xpcom/browserDownload.js @@ -0,0 +1,267 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + +Zotero.BrowserDownload = { + HANDLED_URLS: { + 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html': "html", + '://www.sciencedirect.com': ".challenge-form" + }, + + /** + * Stores cookie sandboxes for urls where we attempt to clear the captcha + */ + _storedCookieSandboxes: {}, + + /** + * Checks whether the url can be handled as a hidden browser download + * @param {String} url + */ + shouldAttemptDownloadViaBrowser: function (url) { + const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); + for (let unproxiedUrl of unproxiedUrls) { + for (let checkUrl in this.HANDLED_URLS) { + if (unproxiedUrl.includes(checkUrl)) { + return checkUrl; + } + } + } + return false; + }, + + getCaptchaLocator(url) { + const handlerKey = this.shouldAttemptDownloadViaBrowser(url); + return this.HANDLED_URLS[handlerKey]; + }, + + _makePDFMIMETypeHandler(browser, onPDFFound = () => 0) { + let isOurPDF, channelBrowser; + let trackedBrowser = browser; + return { + onStartRequest: function (name, _, channel) { + Zotero.debug(`BrowserDownload: Sniffing a PDF loaded at ${name}`); + // try the browser + try { + channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; + } + catch (e) {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + else { + // try the document for the load group + try { + channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) + .topFrameElement; + } + catch (e) {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + } + }, + onContent: async (blob, name) => { + if (isOurPDF) { + Zotero.debug(`BrowserDownload: Found our PDF at ${name}`); + onPDFFound(blob); + return true; + } + else { + Zotero.debug(`BrowserDownload: Not our PDF at ${name}`); + return false; + } + } + }; + }, + + /** + * @param {String} url + * @param {String} path + * @param {Object} [options] + * @param {Object} [options.cookieSandbox] + * @param {Boolean} [options.shouldDisplayCaptcha=false] + */ + async downloadPDF(url, path, options = {}) { + Zotero.debug(`BrowserDownload: Downloading file via a hidden browser from ${url}`); + + let hiddenBrowser; + let pdfMIMETypeHandler; + let cookieSandbox = options.cookieSandbox?.clone(); + let pdfFoundDeferred = Zotero.Promise.defer(); + + let uri = new URL(url); + if (this._storedCookieSandboxes[uri.host]) { + Zotero.debug(`BrowserDownload: Using a stored cookie sandbox for ${uri.host}`); + cookieSandbox = this._storedCookieSandboxes[uri.host]; + } + + // Technically this is not a download, but the full operation (load, redirect, etc) timeout + const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); + const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout'); + + try { + hiddenBrowser = new HiddenBrowser({ cookieSandbox }); + await hiddenBrowser._createdPromise; + + let pdfLoaded = false; + pdfMIMETypeHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, pdfFoundDeferred.resolve); + Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); + + let onLoadTimeoutDeferred = Zotero.Promise.defer(); + let currentUrl = ""; + hiddenBrowser.webProgress.addProgressListener({ + QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), + async onLocationChange() { + let url = hiddenBrowser.currentURI.spec; + if (currentUrl) { + Zotero.debug(`BrowserDownload: A JS redirect occurred to ${url}`); + } + currentUrl = url; + Zotero.debug(`BrowserDownload: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); + await Zotero.Promise.delay(onLoadTimeout); + // If URL changed that means we got redirected and the onLoadTimeout needs to restart + if (currentUrl === url && !pdfLoaded) { + onLoadTimeoutDeferred.reject(new Error(`BrowserDownload: Loading PDF via a hidden browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); + } + } + }, Ci.nsIWebProgress.NOTIFY_LOCATION); + + hiddenBrowser.load(url); + let blob = await Zotero.Promise.race([ + onLoadTimeoutDeferred.promise, + Zotero.Promise.delay(downloadTimeout).then(() => { + if (!pdfLoaded) { + throw new Error(`BrowserDownload: Loading PDF via a hidden browser timed out after ${downloadTimeout}ms`); + } + }), + // Resolves PDF blob + pdfFoundDeferred.promise + ]); + + pdfLoaded = true; + await Zotero.File.putContentsAsync(path, blob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + delete this._storedCookieSandboxes[uri.host]; + if (options?.shouldDisplayCaptcha) { + Zotero.debug(`BrowserDownload: Downloading via a hidden browser failed due to ${e.message}`); + const captchaLocator = this.getCaptchaLocator(url); + if (captchaLocator) { + let doc = await hiddenBrowser.getDocument(); + let elem = doc.querySelector(captchaLocator); + if (elem) { + return this.downloadPDFViaViewer(url, path, options); + } + } + } + throw e; + } + finally { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); + if (hiddenBrowser) { + hiddenBrowser.destroy(); + } + } + }, + + async downloadPDFViaViewer(url, path, options) { + Zotero.debug(`BrowserDownload: Downloading file via the document viewer for captcha clearing from ${url}`); + + let win, browser, xulWin, wmListener; + let pdfMIMETypeHandler; + let pdfFound; + let pdfFoundDeferred = Zotero.Promise.defer(); + const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); + + let uri = new URL(url); + // Since we are downloading via the viewer it means we failed to download via the + // hidden browser either using the cookies provided by the client or stored cookies. + // We will now use client provided cookies but remove the user agent, since + // the cloudflare bot protection doesn't like it when we e.g. use Chrome UA from + // a Chrome Connector cookie sandbox, while acting like a Mozilla browser. + // Cloudflare's bot protection allegedly examines TLS handshake and the like to + // make sure that you are using the browser you are claiming to be. + delete options.cookieSandbox?.userAgent; + + try { + wmListener = { + onOpenWindow(xulWindow) { + xulWin = xulWin || xulWindow; + }, + onCloseWindow(xulWindow) { + if (xulWin === xulWindow && !pdfFound) { + pdfFoundDeferred.reject(new Error("BrowserDownload: User closed the document viewer")); + } + } + }; + Services.wm.addListener(wmListener); + await new Promise((resolve) => { + win = Zotero.openInViewer(url, { cookieSandbox: options.cookieSandbox }); + win.addEventListener('load', resolve); + }); + browser = win.document.querySelector('browser'); + + pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve); + Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); + + Zotero.debug(`BrowserDownload: Awaiting the user to clear the captcha or timeout after ${downloadTimeout}`); + let pdfBlob = await Zotero.Promise.race([ + Zotero.Promise.delay(downloadTimeout).then(() => { + if (!pdfFound) { + throw new Error(`BrowserDownload: Loading PDF via document viewer timed out after ${downloadTimeout}ms`); + } + }), + // Resolves PDF blob + pdfFoundDeferred.promise + ]); + pdfFound = true; + this._storedCookieSandboxes[uri.host] = options.cookieSandbox; + await Zotero.File.putContentsAsync(path, pdfBlob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + throw e; + } + finally { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); + Services.wm.removeListener(wmListener); + if (win) { + win.close(); + } + } + }, +}; diff --git a/chrome/content/zotero/xpcom/connector/server_connector.js b/chrome/content/zotero/xpcom/connector/server_connector.js index ed770d6b27..380b34c80b 100644 --- a/chrome/content/zotero/xpcom/connector/server_connector.js +++ b/chrome/content/zotero/xpcom/connector/server_connector.js @@ -208,11 +208,20 @@ Zotero.Server.Connector.SaveSession.prototype.onProgress = function (item, progr delete o.progress; delete o.contentType; } + if (o.itemType === item.itemType) { + o.progress = progress; + return; + } o.itemType = item.itemType; o.attachments = item.attachments; - if (item.itemType == 'attachment') { - o.progress = progress; - } +}; + +Zotero.Server.Connector.SaveSession.prototype.isSavingDone = function () { + return this.savingDone + || Object.values(this._progressItems).every(i => i.progress === 100 || typeof i.progress !== "number") + && Object.values(this._progressItems).every((i) => { + return !i.attachments || i.attachments.every(a => a.progress === 100 || typeof i.progress !== "number"); + }); }; Zotero.Server.Connector.SaveSession.prototype.getProgressItem = function (id) { @@ -640,8 +649,6 @@ Zotero.Server.Connector.SavePage.prototype = { } let items = await translate.translate({libraryID, collections: collection ? [collection.id] : false}); session.addItems(items); - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; }.bind(this)); }, @@ -786,10 +793,6 @@ Zotero.Server.Connector.SaveItems.prototype = { // Add items to session once all attachments have been saved .then(function (items) { session.addItems(items); - if (session.pendingAttachments.length === 0) { - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; - } }); } catch (e) { @@ -873,8 +876,9 @@ Zotero.Server.Connector.SaveItems.prototype = { function (attachment, progress, error) { session.onProgress(attachment, progress, error); }, - (...args) => { - if (onTopLevelItemsDone) onTopLevelItemsDone(...args); + (itemsJSON, items) => { + itemsJSON.forEach(item => session.onProgress(item, 100)); + if (onTopLevelItemsDone) onTopLevelItemsDone(itemsJSON, items); }, function (parentItemID, attachment) { session.pendingAttachments.push([parentItemID, attachment]); @@ -984,18 +988,18 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { let url = session.pendingAttachments[0][1].url; - let browser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, + let browser = new HiddenBrowser({ docShell: { allowImages: true }, cookieSandbox, }); + await browser.load(url, { requireSuccessfulStatus: true }); try { - snapshotContent = await HiddenBrowser.snapshot(browser); + snapshotContent = await browser.snapshot(); } finally { - HiddenBrowser.destroy(browser); + browser.destroy(); } } else { @@ -1012,8 +1016,6 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { session.onProgress(attachment, false); } - session.savingDone = true; - return [200, 'text/plain', 'No snapshot content attached.']; } @@ -1074,9 +1076,6 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { session.onProgress(attachment, progress, error); }, ); - - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; } return 201; @@ -1338,7 +1337,7 @@ Zotero.Server.Connector.SessionProgress.prototype = { } return newItem; }), - done: session.savingDone + done: session.isSavingDone() }) ]; } diff --git a/chrome/content/zotero/xpcom/cookieSandbox.js b/chrome/content/zotero/xpcom/cookieSandbox.js index 37739143fc..be3906978f 100755 --- a/chrome/content/zotero/xpcom/cookieSandbox.js +++ b/chrome/content/zotero/xpcom/cookieSandbox.js @@ -28,38 +28,38 @@ * * @constructor * @param {browser} [browser] Hidden browser object - * @param {String|nsIURI} uri URI of page to manage cookies for (cookies for domains that are not + * @param {String|nsIURI} uri URI of page to manage cookies for (cookies for domains that are not * subdomains of this URI are ignored) * @param {String} cookieData Cookies with which to initiate the sandbox * @param {String} userAgent User agent to use for sandboxed requests */ -Zotero.CookieSandbox = function(browser, uri, cookieData, userAgent) { - this._observerService = Components.classes["@mozilla.org/observer-service;1"]. - getService(Components.interfaces.nsIObserverService); - - if(uri instanceof Components.interfaces.nsIURI) { - this.URI = uri; - } else { - this.URI = Components.classes["@mozilla.org/network/io-service;1"] - .getService(Components.interfaces.nsIIOService) - .newURI(uri, null, null); - } - +Zotero.CookieSandbox = function (browser, uri, cookieData, userAgent) { this._cookies = {}; - if(cookieData) { + if (cookieData) { + let URI; + if (uri instanceof Components.interfaces.nsIURI) { + URI = uri; + } else { + URI = Components.classes["@mozilla.org/network/io-service;1"] + .getService(Components.interfaces.nsIIOService) + .newURI(uri, null, null); + } var splitCookies = cookieData.split(/;\s*/); for (let cookie of splitCookies) { - this.setCookie(cookie, this.URI.host); + this.setCookie(cookie, URI.host); } } - if(userAgent) this.userAgent = userAgent; - + if (userAgent) this.userAgent = userAgent; + + this._observerService = Components.classes["@mozilla.org/observer-service;1"]. + getService(Components.interfaces.nsIObserverService); + Zotero.CookieSandbox.Observer.register(); - if(browser) { + if (browser) { this.attachToBrowser(browser); } -} +}; /** * Normalizes the host string: lower-case, remove leading period, some more cleanup @@ -91,6 +91,12 @@ Zotero.CookieSandbox.generateCookieString = function(cookies) { } Zotero.CookieSandbox.prototype = { + clone() { + let clone = new Zotero.CookieSandbox(); + clone._cookies = Zotero.Utilities.deepCopy(this._cookies); + clone.userAgent = this.userAgent; + return clone; + }, /** * Adds cookies to this CookieSandbox based on a cookie header * @param {String} cookieString; @@ -164,8 +170,7 @@ Zotero.CookieSandbox.prototype = { * @param {nsIInterfaceRequestor} ir */ "attachToInterfaceRequestor": function(ir) { - Zotero.CookieSandbox.Observer.trackedInterfaceRequestors.push(Cu.getWeakReference(ir)); - Zotero.CookieSandbox.Observer.trackedInterfaceRequestorSandboxes.push(this); + Zotero.CookieSandbox.Observer.trackedInterfaceRequestors.set(ir.QueryInterface(Components.interfaces.nsIInterfaceRequestor), this); }, /** @@ -276,21 +281,19 @@ Zotero.CookieSandbox.prototype = { * nsIObserver implementation for adding, clearing, and slurping cookies */ Zotero.CookieSandbox.Observer = new function() { - const observeredTopics = ["http-on-examine-response", "http-on-modify-request", "quit-application"]; + const observeredTopics = ["http-on-examine-response", "http-on-modify-request"]; var observerService = Components.classes["@mozilla.org/observer-service;1"]. getService(Components.interfaces.nsIObserverService), observing = false; + this.trackedBrowsers = new WeakMap(); + this.trackedInterfaceRequestors = new WeakMap(); /** * Registers cookie manager and observer, if necessary */ - this.register = function(CookieSandbox) { - this.trackedBrowsers = new WeakMap(); - this.trackedInterfaceRequestors = []; - this.trackedInterfaceRequestorSandboxes = []; - - if(!observing) { + this.register = function () { + if (!observing) { Zotero.debug("CookieSandbox: Registering observers"); for (let topic of observeredTopics) observerService.addObserver(this, topic, false); observing = true; @@ -300,62 +303,51 @@ Zotero.CookieSandbox.Observer = new function() { /** * Implements nsIObserver to watch for new cookies and to add sandboxed cookies */ - this.observe = function(channel, topic) { + this.observe = function (channel, topic) { channel.QueryInterface(Components.interfaces.nsIHttpChannel); - var trackedBy, tested, browser, callbacks, + var trackedBy, tested, browser, channelURI = channel.URI.hostPort, notificationCallbacks = channel.notificationCallbacks; + // Zotero.debug(`CookieSandbox: Observing ${topic} at ${channelURI}`, 5); + // try the notification callbacks - if(notificationCallbacks) { - for(var i=0; i