diff --git a/chrome/content/zotero/HiddenBrowser.jsm b/chrome/content/zotero/HiddenBrowser.jsm index 5fbc2447ea..3e532fd5bb 100644 --- a/chrome/content/zotero/HiddenBrowser.jsm +++ b/chrome/content/zotero/HiddenBrowser.jsm @@ -50,22 +50,85 @@ ChromeUtils.registerWindowActor("SingleFile", { }); const progressListeners = new Set(); -const browserFrameMap = new WeakMap(); /** * Functions for creating and destroying hidden browser objects **/ -const HiddenBrowser = { +class HiddenBrowser { /** - * @param {String} source - HTTP URL, file: URL, or file path * @param {Object} options * @param {Boolean} [options.allowJavaScript] * @param {Object} [options.docShell] Fields to set on Browser.docShell - * @param {Boolean} [options.requireSuccessfulStatus] * @param {Boolean} [options.blockRemoteResources] Block all remote (non-file:) resources * @param {Zotero.CookieSandbox} [options.cookieSandbox] */ - async create(source, options = {}) { + constructor(options = {}) { + var frame = new HiddenFrame(); + this._createdPromise = (async () => { + var windowlessBrowser = await frame.get(); + windowlessBrowser.browsingContext.allowJavascript = options.allowJavaScript !== false; + windowlessBrowser.docShell.allowImages = false; + if (options.docShell) { + Object.assign(windowlessBrowser.docShell, options.docShell); + } + var doc = windowlessBrowser.document; + var browser = doc.createXULElement("browser"); + browser.setAttribute("type", "content"); + browser.setAttribute("remote", "true"); + browser.setAttribute('maychangeremoteness', 'true'); + browser.setAttribute("disableglobalhistory", "true"); + doc.documentElement.appendChild(browser); + + if (options.cookieSandbox) { + options.cookieSandbox.attachToBrowser(browser); + } + + if (Zotero.Debug.enabled) { + let weakBrowser = new WeakRef(browser); + setTimeout(() => { + let browser = weakBrowser.deref(); + if (browser && browserFrameMap.has(browser)) { + Zotero.debug('Browser object still alive after 60 seconds - memory leak?'); + Zotero.debug('Viewing URI ' + browser.currentURI?.spec) + } + }, 1000 * 60); + } + + if (options.blockRemoteResources) { + RemoteResourceBlockingObserver.watch(browser); + } + + this._browser = browser; + })(); + + this._frame = frame; + return new Proxy(this, { + get(target, prop) { + if (prop in target) { + return target[prop]; + } + if (!target._browser) throw new Error(`Attempting to use the HiddenBrowser before it is fully initialized. Await browser._createdPromise.`); + return Reflect.get(target._browser, prop); + }, + set(target, prop, val) { + if (prop in target) { + target[prop] = val; + } + Reflect.set(target._browser, prop, val) + return true; + } + }); + } + + /** + * + * @param {String} source - HTTP URL, file: URL, or file path + * @param {Object} options + * @param {Boolean} [options.requireSuccessfulStatus] + * @returns {Promise} + */ + async load(source, options) { + await this._createdPromise; let url; if (/^(file|https?|chrome|resource):/.test(source)) { url = source; @@ -74,45 +137,8 @@ const HiddenBrowser = { else { url = Zotero.File.pathToFileURI(source); } - + Zotero.debug(`Loading ${url} in hidden browser`); - - var frame = new HiddenFrame(); - var windowlessBrowser = await frame.get(); - windowlessBrowser.browsingContext.allowJavascript = options.allowJavaScript !== false; - windowlessBrowser.docShell.allowImages = false; - if (options.docShell) { - Object.assign(windowlessBrowser.docShell, options.docShell); - } - var doc = windowlessBrowser.document; - var browser = doc.createXULElement("browser"); - browser.setAttribute("type", "content"); - browser.setAttribute("remote", "true"); - browser.setAttribute('maychangeremoteness', 'true'); - browser.setAttribute("disableglobalhistory", "true"); - doc.documentElement.appendChild(browser); - - if (options.cookieSandbox) { - options.cookieSandbox.attachToBrowser(browser); - } - - browserFrameMap.set(browser, frame); - - if (Zotero.Debug.enabled) { - let weakBrowser = new WeakRef(browser); - setTimeout(() => { - let browser = weakBrowser.deref(); - if (browser && browserFrameMap.has(browser)) { - Zotero.debug('Browser object still alive after 60 seconds - memory leak?'); - Zotero.debug('Viewing URI ' + browser.currentURI?.spec) - } - }, 1000 * 60); - } - - if (options.blockRemoteResources) { - RemoteResourceBlockingObserver.watch(browser); - } - // Next bit adapted from Mozilla's HeadlessShell.jsm const principal = Services.scriptSecurityManager.getSystemPrincipal(); try { @@ -122,7 +148,7 @@ const HiddenBrowser = { reject(new Error("Page never loaded in hidden browser")); }, 5000); - let oa = E10SUtils.predictOriginAttributes({ browser }); + let oa = E10SUtils.predictOriginAttributes({ browser: this }); let loadURIOptions = { triggeringPrincipal: principal, remoteType: E10SUtils.getRemoteTypeForURI( @@ -134,8 +160,8 @@ const HiddenBrowser = { oa ) }; - browser.loadURI(url, loadURIOptions); - let { webProgress } = browser; + this.loadURI(url, loadURIOptions); + let { webProgress } = this; let progressListener = { onLocationChange(progress, request, location, flags) { @@ -173,11 +199,13 @@ const HiddenBrowser = { return false; } - if (options.requireSuccessfulStatus) { - let { channelInfo } = await this.getPageData(browser, ['channelInfo']); + if (options?.requireSuccessfulStatus) { + let { channelInfo } = await this.getPageData(['channelInfo']); if (channelInfo && (channelInfo.responseStatus < 200 || channelInfo.responseStatus >= 400)) { let response = `${channelInfo.responseStatus} ${channelInfo.responseStatusText}`; - Zotero.debug(`HiddenBrowser.create: ${url} failed with ${response}`, 2); + Zotero.debug(`HiddenBrowser.load: ${url} failed with ${response}`, 2); + // HiddenBrowser will never get returned so we need to clean it up here + this.destroy() throw new Zotero.HTTP.UnexpectedStatusException( { status: channelInfo.responseStatus @@ -187,31 +215,27 @@ const HiddenBrowser = { ); } } - - return browser; - }, + } /** - * @param {Browser} browser * @param {String[]} props - 'characterSet', 'title', 'bodyText', 'documentHTML', 'cookie', 'channelInfo' */ - async getPageData(browser, props) { - var actor = browser.browsingContext.currentWindowGlobal.getActor("PageData"); + async getPageData(props) { + var actor = this.browsingContext.currentWindowGlobal.getActor("PageData"); var data = {}; for (let prop of props) { data[prop] = await actor.sendQuery(prop); } return data; - }, + } /** - * @param {Browser} browser * @returns {Promise} */ - async getDocument(browser) { - let { documentHTML, cookie } = await this.getPageData(browser, ['documentHTML', 'cookie']); + async getDocument() { + let { documentHTML, cookie } = await this.getPageData(['documentHTML', 'cookie']); let doc = new DOMParser().parseFromString(documentHTML, 'text/html'); - let docWithLocation = Zotero.HTTP.wrapDocument(doc, browser.currentURI); + let docWithLocation = Zotero.HTTP.wrapDocument(doc, this.currentURI); return new Proxy(docWithLocation, { get(obj, prop) { if (prop === 'cookie') { @@ -220,24 +244,25 @@ const HiddenBrowser = { return obj[prop]; } }); - }, + } /** - * @param {Browser} browser * @returns {Promise} */ - snapshot(browser) { - let actor = browser.browsingContext.currentWindowGlobal.getActor("SingleFile"); + snapshot() { + let actor = this.browsingContext.currentWindowGlobal.getActor("SingleFile"); return actor.sendQuery('snapshot'); - }, + } - destroy(browser) { - var frame = browserFrameMap.get(browser); - if (frame) { - RemoteResourceBlockingObserver.unwatch(browser); - frame.destroy(); - Zotero.debug("Deleted hidden browser"); - browserFrameMap.delete(browser); + destroy() { + if (this._frame) { + (async () => { + await this._createdPromise; + RemoteResourceBlockingObserver.unwatch(this); + this._frame.destroy(); + this._frame = null; + Zotero.debug("Deleted hidden browser"); + })(); } } }; diff --git a/chrome/content/zotero/standalone/basicViewer.js b/chrome/content/zotero/standalone/basicViewer.js index f2bb443054..575d4e7a9d 100644 --- a/chrome/content/zotero/standalone/basicViewer.js +++ b/chrome/content/zotero/standalone/basicViewer.js @@ -90,6 +90,9 @@ function loadURI(uri, options = {}) { else { browser.browsingContext.sandboxFlags |= SANDBOXED_SCRIPTS; } + if (options.cookieSandbox) { + options.cookieSandbox.attachToBrowser(browser); + } browser.loadURI( uri, { diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index 01393f551d..7b41b99631 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -544,11 +544,11 @@ Zotero.Attachments = new function () { var nativeHandlerImport = async function () { let browser; try { - browser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, + browser = new HiddenBrowser({ docShell: { allowImages: true }, cookieSandbox, }); + await browser.load(url, { requireSuccessfulStatus: true }); return await Zotero.Attachments.importFromDocument({ libraryID, browser, @@ -563,7 +563,7 @@ Zotero.Attachments = new function () { throw e; } finally { - if (browser) HiddenBrowser.destroy(browser); + if (browser) browser.destroy(); } }; @@ -597,7 +597,8 @@ Zotero.Attachments = new function () { { cookieSandbox, referrer, - isPDF: contentType == 'application/pdf' + isPDF: contentType == 'application/pdf', + shouldDisplayCaptcha: true } ); @@ -898,7 +899,7 @@ Zotero.Attachments = new function () { if (browser) { // If we have a full hidden browser, use SingleFile Zotero.debug('Getting snapshot with HiddenBrowser.snapshot()'); - let snapshotContent = yield HiddenBrowser.snapshot(browser); + let snapshotContent = yield browser.snapshot(); // Write main HTML file to disk yield Zotero.File.putContentsAsync(tmpFile, snapshotContent); @@ -1082,6 +1083,7 @@ Zotero.Attachments = new function () { * @param {Object} [options.cookieSandbox] * @param {String} [options.referrer] * @param {Boolean} [options.isPDF] - Delete file if not PDF + * @param {Boolean} [options.shouldDisplayCaptcha] */ this.downloadFile = async function (url, path, options = {}) { Zotero.debug(`Downloading file from ${url}`); @@ -1118,123 +1120,13 @@ Zotero.Attachments = new function () { // Custom handling for PDFs that are bot-guarded // via a JS-redirect if (enforcingPDF && e instanceof this.InvalidPDFException) { - const downloadViaBrowserList = [ - 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', - '://www.sciencedirect.com', - ]; - const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); - for (let unproxiedUrl of unproxiedUrls) { - if (downloadViaBrowserList.some(checkUrl => unproxiedUrl.includes(checkUrl))) { - return this.downloadPDFViaBrowser(url, path, options); - } + if (Zotero.BrowserDownload.shouldAttemptDownloadViaBrowser(url)) { + return Zotero.BrowserDownload.downloadPDF(url, path, options); } } throw e; } }; - - /** - * @param {String} url - * @param {String} path - * @param {Object} [options] - * @param {Object} [options.cookieSandbox] - */ - this.downloadPDFViaBrowser = async function (url, path, options = {}) { - Zotero.debug(`downloadPDFViaBrowser: Downloading file via browser from ${url}`); - const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout'); - // Technically this is not a download, but the full operation timeout - const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); - let channelBrowser, hiddenBrowser; - let hiddenBrowserPDFFoundDeferred = Zotero.Promise.defer(); - - let isOurPDF = false; - var pdfMIMETypeHandler = { - onStartRequest: function (name, _, channel) { - Zotero.debug(`downloadPDFViaBrowser: Sniffing a PDF loaded at ${name}`); - // try the browser - try { - channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; - } - catch (e) {} - if (channelBrowser) { - isOurPDF = hiddenBrowser === channelBrowser; - } - else { - // try the document for the load group - try { - channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) - .topFrameElement; - } - catch(e) {} - if (channelBrowser) { - isOurPDF = hiddenBrowser === channelBrowser; - } - } - }, - onContent: async (blob, name, _, channel) => { - if (isOurPDF) { - Zotero.debug(`downloadPDFViaBrowser: Found our PDF at ${name}`); - await Zotero.File.putContentsAsync(path, blob); - hiddenBrowserPDFFoundDeferred.resolve(); - return true; - } - else { - Zotero.debug(`downloadPDFViaBrowser: Not our PDF at ${name}`); - return false; - } - } - }; - try { - Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); - hiddenBrowser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, - cookieSandbox: options.cookieSandbox, - }); - let onLoadTimeoutDeferred = Zotero.Promise.defer(); - let currentUrl = ""; - hiddenBrowser.webProgress.addProgressListener({ - QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), - async onLocationChange() { - let url = hiddenBrowser.currentURI.spec; - if (currentUrl) { - Zotero.debug(`downloadPDFViaBrowser: A JS redirect occurred to ${url}`); - } - currentUrl = url; - Zotero.debug(`downloadPDFViaBrowser: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); - await Zotero.Promise.delay(onLoadTimeout); - // If URL changed that means we got redirected and the onLoadTimeout needs to restart - if (currentUrl === url && !isOurPDF) { - onLoadTimeoutDeferred.reject(new Error(`downloadPDFViaBrowser: Loading PDF via browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); - } - } - }, Ci.nsIWebProgress.NOTIFY_LOCATION); - await Zotero.Promise.race([ - onLoadTimeoutDeferred.promise, - Zotero.Promise.delay(downloadTimeout).then(() => { - if (!isOurPDF) { - throw new Error(`downloadPDFViaBrowser: Loading PDF via browser timed out after ${downloadTimeout}ms`); - } - }), - hiddenBrowserPDFFoundDeferred.promise - ]); - } - catch (e) { - try { - await OS.File.remove(path, { ignoreAbsent: true }); - } - catch (err) { - Zotero.logError(err); - } - throw e; - } - finally { - Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); - if (hiddenBrowser) { - HiddenBrowser.destroy(hiddenBrowser); - } - } - }; - /** * Make sure a file is a PDF @@ -1845,6 +1737,7 @@ Zotero.Attachments = new function () { tmpFile, { isPDF: true, + shouldDisplayCaptcha: true, onAccessMethodStart: options.onAccessMethodStart, onBeforeRequest: options.onBeforeRequest, onRequestError: options.onRequestError diff --git a/chrome/content/zotero/xpcom/browserDownload.js b/chrome/content/zotero/xpcom/browserDownload.js new file mode 100644 index 0000000000..3d973755b3 --- /dev/null +++ b/chrome/content/zotero/xpcom/browserDownload.js @@ -0,0 +1,267 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + +Zotero.BrowserDownload = { + HANDLED_URLS: { + 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html': "html", + '://www.sciencedirect.com': ".challenge-form" + }, + + /** + * Stores cookie sandboxes for urls where we attempt to clear the captcha + */ + _storedCookieSandboxes: {}, + + /** + * Checks whether the url can be handled as a hidden browser download + * @param {String} url + */ + shouldAttemptDownloadViaBrowser: function (url) { + const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url)); + for (let unproxiedUrl of unproxiedUrls) { + for (let checkUrl in this.HANDLED_URLS) { + if (unproxiedUrl.includes(checkUrl)) { + return checkUrl; + } + } + } + return false; + }, + + getCaptchaLocator(url) { + const handlerKey = this.shouldAttemptDownloadViaBrowser(url); + return this.HANDLED_URLS[handlerKey]; + }, + + _makePDFMIMETypeHandler(browser, onPDFFound = () => 0) { + let isOurPDF, channelBrowser; + let trackedBrowser = browser; + return { + onStartRequest: function (name, _, channel) { + Zotero.debug(`BrowserDownload: Sniffing a PDF loaded at ${name}`); + // try the browser + try { + channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; + } + catch (e) {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + else { + // try the document for the load group + try { + channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) + .topFrameElement; + } + catch (e) {} + if (channelBrowser) { + isOurPDF = trackedBrowser === channelBrowser; + } + } + }, + onContent: async (blob, name) => { + if (isOurPDF) { + Zotero.debug(`BrowserDownload: Found our PDF at ${name}`); + onPDFFound(blob); + return true; + } + else { + Zotero.debug(`BrowserDownload: Not our PDF at ${name}`); + return false; + } + } + }; + }, + + /** + * @param {String} url + * @param {String} path + * @param {Object} [options] + * @param {Object} [options.cookieSandbox] + * @param {Boolean} [options.shouldDisplayCaptcha=false] + */ + async downloadPDF(url, path, options = {}) { + Zotero.debug(`BrowserDownload: Downloading file via a hidden browser from ${url}`); + + let hiddenBrowser; + let pdfMIMETypeHandler; + let cookieSandbox = options.cookieSandbox?.clone(); + let pdfFoundDeferred = Zotero.Promise.defer(); + + let uri = new URL(url); + if (this._storedCookieSandboxes[uri.host]) { + Zotero.debug(`BrowserDownload: Using a stored cookie sandbox for ${uri.host}`); + cookieSandbox = this._storedCookieSandboxes[uri.host]; + } + + // Technically this is not a download, but the full operation (load, redirect, etc) timeout + const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); + const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout'); + + try { + hiddenBrowser = new HiddenBrowser({ cookieSandbox }); + await hiddenBrowser._createdPromise; + + let pdfLoaded = false; + pdfMIMETypeHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, pdfFoundDeferred.resolve); + Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); + + let onLoadTimeoutDeferred = Zotero.Promise.defer(); + let currentUrl = ""; + hiddenBrowser.webProgress.addProgressListener({ + QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), + async onLocationChange() { + let url = hiddenBrowser.currentURI.spec; + if (currentUrl) { + Zotero.debug(`BrowserDownload: A JS redirect occurred to ${url}`); + } + currentUrl = url; + Zotero.debug(`BrowserDownload: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); + await Zotero.Promise.delay(onLoadTimeout); + // If URL changed that means we got redirected and the onLoadTimeout needs to restart + if (currentUrl === url && !pdfLoaded) { + onLoadTimeoutDeferred.reject(new Error(`BrowserDownload: Loading PDF via a hidden browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); + } + } + }, Ci.nsIWebProgress.NOTIFY_LOCATION); + + hiddenBrowser.load(url); + let blob = await Zotero.Promise.race([ + onLoadTimeoutDeferred.promise, + Zotero.Promise.delay(downloadTimeout).then(() => { + if (!pdfLoaded) { + throw new Error(`BrowserDownload: Loading PDF via a hidden browser timed out after ${downloadTimeout}ms`); + } + }), + // Resolves PDF blob + pdfFoundDeferred.promise + ]); + + pdfLoaded = true; + await Zotero.File.putContentsAsync(path, blob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + delete this._storedCookieSandboxes[uri.host]; + if (options?.shouldDisplayCaptcha) { + Zotero.debug(`BrowserDownload: Downloading via a hidden browser failed due to ${e.message}`); + const captchaLocator = this.getCaptchaLocator(url); + if (captchaLocator) { + let doc = await hiddenBrowser.getDocument(); + let elem = doc.querySelector(captchaLocator); + if (elem) { + return this.downloadPDFViaViewer(url, path, options); + } + } + } + throw e; + } + finally { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); + if (hiddenBrowser) { + hiddenBrowser.destroy(); + } + } + }, + + async downloadPDFViaViewer(url, path, options) { + Zotero.debug(`BrowserDownload: Downloading file via the document viewer for captcha clearing from ${url}`); + + let win, browser, xulWin, wmListener; + let pdfMIMETypeHandler; + let pdfFound; + let pdfFoundDeferred = Zotero.Promise.defer(); + const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout'); + + let uri = new URL(url); + // Since we are downloading via the viewer it means we failed to download via the + // hidden browser either using the cookies provided by the client or stored cookies. + // We will now use client provided cookies but remove the user agent, since + // the cloudflare bot protection doesn't like it when we e.g. use Chrome UA from + // a Chrome Connector cookie sandbox, while acting like a Mozilla browser. + // Cloudflare's bot protection allegedly examines TLS handshake and the like to + // make sure that you are using the browser you are claiming to be. + delete options.cookieSandbox?.userAgent; + + try { + wmListener = { + onOpenWindow(xulWindow) { + xulWin = xulWin || xulWindow; + }, + onCloseWindow(xulWindow) { + if (xulWin === xulWindow && !pdfFound) { + pdfFoundDeferred.reject(new Error("BrowserDownload: User closed the document viewer")); + } + } + }; + Services.wm.addListener(wmListener); + await new Promise((resolve) => { + win = Zotero.openInViewer(url, { cookieSandbox: options.cookieSandbox }); + win.addEventListener('load', resolve); + }); + browser = win.document.querySelector('browser'); + + pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve); + Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); + + Zotero.debug(`BrowserDownload: Awaiting the user to clear the captcha or timeout after ${downloadTimeout}`); + let pdfBlob = await Zotero.Promise.race([ + Zotero.Promise.delay(downloadTimeout).then(() => { + if (!pdfFound) { + throw new Error(`BrowserDownload: Loading PDF via document viewer timed out after ${downloadTimeout}ms`); + } + }), + // Resolves PDF blob + pdfFoundDeferred.promise + ]); + pdfFound = true; + this._storedCookieSandboxes[uri.host] = options.cookieSandbox; + await Zotero.File.putContentsAsync(path, pdfBlob); + } + catch (e) { + try { + await OS.File.remove(path, { ignoreAbsent: true }); + } + catch (err) { + Zotero.logError(err); + } + throw e; + } + finally { + Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); + Services.wm.removeListener(wmListener); + if (win) { + win.close(); + } + } + }, +}; diff --git a/chrome/content/zotero/xpcom/connector/server_connector.js b/chrome/content/zotero/xpcom/connector/server_connector.js index ed770d6b27..380b34c80b 100644 --- a/chrome/content/zotero/xpcom/connector/server_connector.js +++ b/chrome/content/zotero/xpcom/connector/server_connector.js @@ -208,11 +208,20 @@ Zotero.Server.Connector.SaveSession.prototype.onProgress = function (item, progr delete o.progress; delete o.contentType; } + if (o.itemType === item.itemType) { + o.progress = progress; + return; + } o.itemType = item.itemType; o.attachments = item.attachments; - if (item.itemType == 'attachment') { - o.progress = progress; - } +}; + +Zotero.Server.Connector.SaveSession.prototype.isSavingDone = function () { + return this.savingDone + || Object.values(this._progressItems).every(i => i.progress === 100 || typeof i.progress !== "number") + && Object.values(this._progressItems).every((i) => { + return !i.attachments || i.attachments.every(a => a.progress === 100 || typeof i.progress !== "number"); + }); }; Zotero.Server.Connector.SaveSession.prototype.getProgressItem = function (id) { @@ -640,8 +649,6 @@ Zotero.Server.Connector.SavePage.prototype = { } let items = await translate.translate({libraryID, collections: collection ? [collection.id] : false}); session.addItems(items); - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; }.bind(this)); }, @@ -786,10 +793,6 @@ Zotero.Server.Connector.SaveItems.prototype = { // Add items to session once all attachments have been saved .then(function (items) { session.addItems(items); - if (session.pendingAttachments.length === 0) { - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; - } }); } catch (e) { @@ -873,8 +876,9 @@ Zotero.Server.Connector.SaveItems.prototype = { function (attachment, progress, error) { session.onProgress(attachment, progress, error); }, - (...args) => { - if (onTopLevelItemsDone) onTopLevelItemsDone(...args); + (itemsJSON, items) => { + itemsJSON.forEach(item => session.onProgress(item, 100)); + if (onTopLevelItemsDone) onTopLevelItemsDone(itemsJSON, items); }, function (parentItemID, attachment) { session.pendingAttachments.push([parentItemID, attachment]); @@ -984,18 +988,18 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { let url = session.pendingAttachments[0][1].url; - let browser = await HiddenBrowser.create(url, { - requireSuccessfulStatus: true, + let browser = new HiddenBrowser({ docShell: { allowImages: true }, cookieSandbox, }); + await browser.load(url, { requireSuccessfulStatus: true }); try { - snapshotContent = await HiddenBrowser.snapshot(browser); + snapshotContent = await browser.snapshot(); } finally { - HiddenBrowser.destroy(browser); + browser.destroy(); } } else { @@ -1012,8 +1016,6 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { session.onProgress(attachment, false); } - session.savingDone = true; - return [200, 'text/plain', 'No snapshot content attached.']; } @@ -1074,9 +1076,6 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { session.onProgress(attachment, progress, error); }, ); - - // Return 'done: true' so the connector stops checking for updates - session.savingDone = true; } return 201; @@ -1338,7 +1337,7 @@ Zotero.Server.Connector.SessionProgress.prototype = { } return newItem; }), - done: session.savingDone + done: session.isSavingDone() }) ]; } diff --git a/chrome/content/zotero/xpcom/cookieSandbox.js b/chrome/content/zotero/xpcom/cookieSandbox.js index 37739143fc..be3906978f 100755 --- a/chrome/content/zotero/xpcom/cookieSandbox.js +++ b/chrome/content/zotero/xpcom/cookieSandbox.js @@ -28,38 +28,38 @@ * * @constructor * @param {browser} [browser] Hidden browser object - * @param {String|nsIURI} uri URI of page to manage cookies for (cookies for domains that are not + * @param {String|nsIURI} uri URI of page to manage cookies for (cookies for domains that are not * subdomains of this URI are ignored) * @param {String} cookieData Cookies with which to initiate the sandbox * @param {String} userAgent User agent to use for sandboxed requests */ -Zotero.CookieSandbox = function(browser, uri, cookieData, userAgent) { - this._observerService = Components.classes["@mozilla.org/observer-service;1"]. - getService(Components.interfaces.nsIObserverService); - - if(uri instanceof Components.interfaces.nsIURI) { - this.URI = uri; - } else { - this.URI = Components.classes["@mozilla.org/network/io-service;1"] - .getService(Components.interfaces.nsIIOService) - .newURI(uri, null, null); - } - +Zotero.CookieSandbox = function (browser, uri, cookieData, userAgent) { this._cookies = {}; - if(cookieData) { + if (cookieData) { + let URI; + if (uri instanceof Components.interfaces.nsIURI) { + URI = uri; + } else { + URI = Components.classes["@mozilla.org/network/io-service;1"] + .getService(Components.interfaces.nsIIOService) + .newURI(uri, null, null); + } var splitCookies = cookieData.split(/;\s*/); for (let cookie of splitCookies) { - this.setCookie(cookie, this.URI.host); + this.setCookie(cookie, URI.host); } } - if(userAgent) this.userAgent = userAgent; - + if (userAgent) this.userAgent = userAgent; + + this._observerService = Components.classes["@mozilla.org/observer-service;1"]. + getService(Components.interfaces.nsIObserverService); + Zotero.CookieSandbox.Observer.register(); - if(browser) { + if (browser) { this.attachToBrowser(browser); } -} +}; /** * Normalizes the host string: lower-case, remove leading period, some more cleanup @@ -91,6 +91,12 @@ Zotero.CookieSandbox.generateCookieString = function(cookies) { } Zotero.CookieSandbox.prototype = { + clone() { + let clone = new Zotero.CookieSandbox(); + clone._cookies = Zotero.Utilities.deepCopy(this._cookies); + clone.userAgent = this.userAgent; + return clone; + }, /** * Adds cookies to this CookieSandbox based on a cookie header * @param {String} cookieString; @@ -164,8 +170,7 @@ Zotero.CookieSandbox.prototype = { * @param {nsIInterfaceRequestor} ir */ "attachToInterfaceRequestor": function(ir) { - Zotero.CookieSandbox.Observer.trackedInterfaceRequestors.push(Cu.getWeakReference(ir)); - Zotero.CookieSandbox.Observer.trackedInterfaceRequestorSandboxes.push(this); + Zotero.CookieSandbox.Observer.trackedInterfaceRequestors.set(ir.QueryInterface(Components.interfaces.nsIInterfaceRequestor), this); }, /** @@ -276,21 +281,19 @@ Zotero.CookieSandbox.prototype = { * nsIObserver implementation for adding, clearing, and slurping cookies */ Zotero.CookieSandbox.Observer = new function() { - const observeredTopics = ["http-on-examine-response", "http-on-modify-request", "quit-application"]; + const observeredTopics = ["http-on-examine-response", "http-on-modify-request"]; var observerService = Components.classes["@mozilla.org/observer-service;1"]. getService(Components.interfaces.nsIObserverService), observing = false; + this.trackedBrowsers = new WeakMap(); + this.trackedInterfaceRequestors = new WeakMap(); /** * Registers cookie manager and observer, if necessary */ - this.register = function(CookieSandbox) { - this.trackedBrowsers = new WeakMap(); - this.trackedInterfaceRequestors = []; - this.trackedInterfaceRequestorSandboxes = []; - - if(!observing) { + this.register = function () { + if (!observing) { Zotero.debug("CookieSandbox: Registering observers"); for (let topic of observeredTopics) observerService.addObserver(this, topic, false); observing = true; @@ -300,62 +303,51 @@ Zotero.CookieSandbox.Observer = new function() { /** * Implements nsIObserver to watch for new cookies and to add sandboxed cookies */ - this.observe = function(channel, topic) { + this.observe = function (channel, topic) { channel.QueryInterface(Components.interfaces.nsIHttpChannel); - var trackedBy, tested, browser, callbacks, + var trackedBy, tested, browser, channelURI = channel.URI.hostPort, notificationCallbacks = channel.notificationCallbacks; + // Zotero.debug(`CookieSandbox: Observing ${topic} at ${channelURI}`, 5); + // try the notification callbacks - if(notificationCallbacks) { - for(var i=0; i