From 4b99b2715d8b63f4ef7f87a1d0d646174e5f8a7b Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 14 Apr 2023 11:22:23 -0400 Subject: [PATCH 1/5] fx-compat: HiddenBrowser: Add more options - Make docShell properties configurable - Allow requiring a successful HTTP status code --- chrome/content/zotero/HiddenBrowser.jsm | 40 ++++++++++++++++++++++++- test/tests/HiddenBrowserTest.js | 21 +++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/HiddenBrowser.jsm b/chrome/content/zotero/HiddenBrowser.jsm index 33070a5947..f2368307ba 100644 --- a/chrome/content/zotero/HiddenBrowser.jsm +++ b/chrome/content/zotero/HiddenBrowser.jsm @@ -51,7 +51,12 @@ const browserFrameMap = new WeakMap(); **/ const HiddenBrowser = { /** - * @param {String) source - HTTP URL, file: URL, or file path + * @param {String} source - HTTP URL, file: URL, or file path + * @param {Object} options + * @param {Boolean} [options.allowJavaScript] + * @param {Object} [options.docShell] Fields to set on Browser.docShell + * @param {Boolean} [options.requireSuccessfulStatus] + * @param {Zotero.CookieSandbox} [options.cookieSandbox] */ async create(source, options = {}) { let url; @@ -69,6 +74,9 @@ const HiddenBrowser = { var windowlessBrowser = await frame.get(); windowlessBrowser.browsingContext.allowJavascript = options.allowJavaScript !== false; windowlessBrowser.docShell.allowImages = false; + if (options.docShell) { + Object.assign(windowlessBrowser.docShell, options.docShell); + } var doc = windowlessBrowser.document; var browser = doc.createXULElement("browser"); browser.setAttribute("type", "content"); @@ -77,8 +85,23 @@ const HiddenBrowser = { browser.setAttribute("disableglobalhistory", "true"); doc.documentElement.appendChild(browser); + if (options.cookieSandbox) { + options.cookieSandbox.attachToBrowser(browser); + } + browserFrameMap.set(browser, frame); + if (Zotero.Debug.enabled) { + let weakBrowser = new WeakRef(browser); + setTimeout(() => { + let browser = weakBrowser.deref(); + if (browser && browserFrameMap.has(browser)) { + Zotero.debug('Browser object still alive after 60 seconds - memory leak?'); + Zotero.debug('Viewing URI ' + browser.currentURI?.spec) + } + }, 1000 * 60); + } + // Next bit adapted from Mozilla's HeadlessShell.jsm const principal = Services.scriptSecurityManager.getSystemPrincipal(); try { @@ -139,6 +162,21 @@ const HiddenBrowser = { return false; } + if (options.requireSuccessfulStatus) { + let { channelInfo } = await this.getPageData(browser, ['channelInfo']); + if (channelInfo && (channelInfo.responseStatus < 200 || channelInfo.responseStatus >= 400)) { + let response = `${channelInfo.responseStatus} ${channelInfo.responseStatusText}`; + Zotero.debug(`HiddenBrowser.create: ${url} failed with ${response}`, 2); + throw new Zotero.HTTP.UnexpectedStatusException( + { + status: channelInfo.responseStatus + }, + url, + `Invalid response ${response} for ${url}` + ); + } + } + return browser; }, diff --git a/test/tests/HiddenBrowserTest.js b/test/tests/HiddenBrowserTest.js index 924c718049..d7e019c278 100644 --- a/test/tests/HiddenBrowserTest.js +++ b/test/tests/HiddenBrowserTest.js @@ -3,6 +3,27 @@ describe("HiddenBrowser", function() { "chrome://zotero/content/HiddenBrowser.jsm" ); + describe("#create()", function () { + var httpd; + var port = 16213; + var baseURL = `http://127.0.0.1:${port}/`; + + before(function () { + Cu.import("resource://zotero-unit/httpd.js"); + httpd = new HttpServer(); + httpd.start(port); + }); + + after(async function () { + await new Promise(resolve => httpd.stop(resolve)); + }); + + it("should fail on non-2xx response with requireSuccessfulStatus", async function () { + let e = await getPromiseError(HiddenBrowser.create(baseURL + 'nonexistent', { requireSuccessfulStatus: true })); + assert.instanceOf(e, Zotero.HTTP.UnexpectedStatusException); + }); + }); + describe("#getPageData()", function () { it("should handle local UTF-8 HTML file", async function () { var path = OS.Path.join(getTestDataDirectory().path, 'test-hidden.html'); From 667058204d0db141206e7c35eb9da9eed4fd4863 Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 14 Apr 2023 11:23:05 -0400 Subject: [PATCH 2/5] fx-compat: HiddenBrowser: Fix destroy() and URI/path check --- chrome/content/zotero/HiddenBrowser.jsm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chrome/content/zotero/HiddenBrowser.jsm b/chrome/content/zotero/HiddenBrowser.jsm index f2368307ba..36739e87cb 100644 --- a/chrome/content/zotero/HiddenBrowser.jsm +++ b/chrome/content/zotero/HiddenBrowser.jsm @@ -60,7 +60,7 @@ const HiddenBrowser = { */ async create(source, options = {}) { let url; - if (/^(file|https?):/.test(source)) { + if (/^(file|https?|chrome|resource):/.test(source)) { url = source; } // Convert string path to file: URL @@ -198,7 +198,7 @@ const HiddenBrowser = { if (frame) { frame.destroy(); Zotero.debug("Deleted hidden browser"); - browserFrameMap.delete(frame); + browserFrameMap.delete(browser); } } }; From b2947aede0f09c3421c2274e4bf6091ae4228d5e Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 14 Apr 2023 11:27:09 -0400 Subject: [PATCH 3/5] fx-compat: Fix attaching CookieSandbox to a browser --- chrome/content/zotero/xpcom/cookieSandbox.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chrome/content/zotero/xpcom/cookieSandbox.js b/chrome/content/zotero/xpcom/cookieSandbox.js index 7b974e9304..6848681a9b 100755 --- a/chrome/content/zotero/xpcom/cookieSandbox.js +++ b/chrome/content/zotero/xpcom/cookieSandbox.js @@ -336,8 +336,7 @@ Zotero.CookieSandbox.Observer = new function() { } else { // try the browser try { - browser = notificationCallbacks.getInterface(Ci.nsIWebNavigation) - .QueryInterface(Ci.nsIDocShell).chromeEventHandler; + browser = notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; } catch(e) {} if(browser) { tested = true; From 0612a9e6f512533e7e806b00e45f1dbbaeade305 Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 14 Apr 2023 11:37:07 -0400 Subject: [PATCH 4/5] fx-compat: Run translation and SingleFile in [hidden] browser And replace loadDocuments(). --- chrome/content/scaffold/scaffold.js | 176 ++++----- chrome/content/zotero/HiddenBrowser.jsm | 37 +- chrome/content/zotero/RemoteTranslate.jsm | 222 +++++++++++ .../content/zotero/actors/PageDataChild.jsm | 24 +- .../content/zotero/actors/SingleFileChild.jsm | 196 ++++++++++ .../zotero/actors/TranslationChild.jsm | 346 ++++++++++++++++++ .../zotero/actors/TranslationParent.jsm | 78 ++++ .../content/zotero/actors/translation/http.js | 305 +++++++++++++++ .../actors/translation/translate_item.js | 29 ++ chrome/content/zotero/xpcom/attachments.js | 80 ++-- .../xpcom/connector/server_connector.js | 34 +- chrome/content/zotero/xpcom/data/feedItem.js | 132 ++++--- chrome/content/zotero/xpcom/http.js | 163 --------- .../zotero/xpcom/utilities_internal.js | 171 --------- test/tests/HiddenBrowserTest.js | 23 +- test/tests/RemoteTranslateTest.js | 162 ++++++++ test/tests/attachmentsTest.js | 56 +-- test/tests/httpTest.js | 45 +-- test/tests/translateTest.js | 22 +- 19 files changed, 1652 insertions(+), 649 deletions(-) create mode 100644 chrome/content/zotero/RemoteTranslate.jsm create mode 100644 chrome/content/zotero/actors/SingleFileChild.jsm create mode 100644 chrome/content/zotero/actors/TranslationChild.jsm create mode 100644 chrome/content/zotero/actors/TranslationParent.jsm create mode 100644 chrome/content/zotero/actors/translation/http.js create mode 100644 chrome/content/zotero/actors/translation/translate_item.js create mode 100644 test/tests/RemoteTranslateTest.js diff --git a/chrome/content/scaffold/scaffold.js b/chrome/content/scaffold/scaffold.js index 48a08d3782..6e9ba0ab76 100644 --- a/chrome/content/scaffold/scaffold.js +++ b/chrome/content/scaffold/scaffold.js @@ -26,6 +26,8 @@ var { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); var { E10SUtils } = ChromeUtils.import("resource://gre/modules/E10SUtils.jsm"); var { Subprocess } = ChromeUtils.import("resource://gre/modules/Subprocess.jsm"); +var { RemoteTranslate } = ChromeUtils.import("chrome://zotero/content/RemoteTranslate.jsm"); +var { ContentDOMReference } = ChromeUtils.import("resource://gre/modules/ContentDOMReference.jsm"); import FilePicker from 'zotero/modules/filePicker'; @@ -959,24 +961,20 @@ var Scaffold = new function () { /* * run translator in given mode with given input */ - function _run(functionToRun, input, selectItems, itemDone, detectHandler, done) { + async function _run(functionToRun, input, selectItems, itemDone, detectHandler, done) { + let translate; + let isRemoteWeb = false; if (functionToRun == "detectWeb" || functionToRun == "doWeb") { - var translate = new Zotero.Translate.Web(); + translate = new RemoteTranslate(); + isRemoteWeb = true; if (!_testTargetRegex(input)) { - _logOutput("Target did not match " + _getDocumentURL(input)); + _logOutput("Target did not match " + _getCurrentURI(input)); if (done) { done(); } return; } - translate.setDocument(input); - - // Use cookies from browser pane - translate.setCookieSandbox(new Zotero.CookieSandbox( - null, - _getDocumentURL(input), - input.cookie - )); + await translate.setBrowser(input); } else if (functionToRun == "detectImport" || functionToRun == "doImport") { translate = new Zotero.Translate.Import(); @@ -1000,15 +998,33 @@ var Scaffold = new function () { // get translator var translator = _getTranslatorFromPane(); if (functionToRun.startsWith('detect')) { - // don't let target prevent translator from operating - translator.target = null; - // generate sandbox - translate.setHandler("translators", detectHandler); - // internal hack to call detect on this translator - translate._potentialTranslators = [translator]; - translate._foundTranslators = []; - translate._currentState = "detect"; - translate._detect(); + if (isRemoteWeb) { + translate.setTranslator(translator); + detectHandler(translate, await translate.detect()); + translate.dispose(); + } + else { + // don't let target prevent translator from operating + translator.target = null; + // generate sandbox + translate.setHandler("translators", detectHandler); + // internal hack to call detect on this translator + translate._potentialTranslators = [translator]; + translate._foundTranslators = []; + translate._currentState = "detect"; + translate._detect(); + } + } + else if (isRemoteWeb) { + translate.setHandler("select", selectItems); + translate.setTranslator(translator); + let items = await translate.translate({ libraryID: false }); + if (items) { + for (let item of items) { + itemDone(translate, item); + } + } + translate.dispose(); } else { // don't let the detectCode prevent the translator from operating @@ -1049,14 +1065,14 @@ var Scaffold = new function () { * Test target regular expression against document URL and log the result */ this.logTargetRegex = async function () { - _logOutput(_testTargetRegex(await _getDocument())); + _logOutput(_testTargetRegex(_browser)); }; /** * Test target regular expression against document URL and return the result */ - function _testTargetRegex(doc) { - var url = _getDocumentURL(doc); + function _testTargetRegex(browser) { + var url = _getCurrentURI(browser); try { var targetRe = new RegExp(document.getElementById('textbox-target').value, "i"); @@ -1191,7 +1207,7 @@ var Scaffold = new function () { async function _getInput(typeOrMethod) { typeOrMethod = typeOrMethod.toLowerCase(); if (typeOrMethod.endsWith('web')) { - return _getDocument(); + return _browser; } else if (typeOrMethod.endsWith('import')) { return _getImport(); @@ -1875,12 +1891,12 @@ var Scaffold = new function () { for (let item of items) { item.getElementsByTagName("label")[1].setAttribute("value", "Running"); var test = JSON.parse(item.dataset.testString); - test["ui-item"] = item; + test["ui-item"] = ContentDOMReference.get(item); tests.push(test); } this.runTests(tests, (obj, test, status, message) => { - test["ui-item"].getElementsByTagName("label")[1].setAttribute("value", message); + ContentDOMReference.resolve(test["ui-item"]).getElementsByTagName("label")[1].setAttribute("value", message); }); }; @@ -1975,7 +1991,7 @@ var Scaffold = new function () { this._updateTests(); }; - TestUpdater.prototype._updateTests = function () { + TestUpdater.prototype._updateTests = async function () { if (!this.testsToUpdate.length) { this.doneCallback(this.newTests); return; @@ -1986,49 +2002,52 @@ var Scaffold = new function () { if (test.type == 'web') { _logOutput("Loading web page from " + test.url); - var hiddenBrowser = Zotero.HTTP.loadDocuments( - test.url, - (doc) => { - _logOutput("Page loaded"); - if (test.defer) { - _logOutput("Waiting " + (Zotero_TranslatorTester.DEFER_DELAY / 1000) - + " second(s) for page content to settle" - ); - } - Zotero.setTimeout( - () => { - doc = hiddenBrowser.contentDocument; - if (doc.location.href != test.url) { - _logOutput("Page URL differs from test. Will be updated. " + doc.location.href); - } - this.tester.newTest(doc, - (obj, newTest) => { - Zotero.Browser.deleteHiddenBrowser(hiddenBrowser); - if (test.defer) { - newTest.defer = true; - } - newTest = _sanitizeItemsInTest(newTest); - this.newTests.push(newTest); - this.testDoneCallback(newTest); - this._updateTests(); - }, - _confirmCreateExpectedFailTest); - }, - test.defer ? Zotero_TranslatorTester.DEFER_DELAY : 0, - true - ); - }, - null, - (e) => { - Zotero.logError(e); - this.newTests.push(false); - this.testDoneCallback(false); - this._updateTests(); - }, - true - ); - hiddenBrowser.docShell.allowMetaRedirects = true; + const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + let browser; + try { + browser = await HiddenBrowser.create(test.url, { + requireSuccessfulStatus: true, + docShell: { allowMetaRedirects: true } + }); + + if (test.defer) { + _logOutput("Waiting " + (Zotero_TranslatorTester.DEFER_DELAY / 1000) + + " second(s) for page content to settle"); + await Zotero.Promise.delay(Zotero_TranslatorTester.DEFER_DELAY); + } + else { + // Wait just a bit for things to settle + await Zotero.Promise.delay(1000); + } + + if (browser.currentURI.spec != test.url) { + _logOutput("Page URL differs from test. Will be updated. " + browser.currentURI.spec); + } + + let translate = new RemoteTranslate(); + await translate.setBrowser(browser); + await translate.setTranslatorProvider(_translatorProvider); + translate.setTranslator(_getTranslatorFromPane()); + translate.setHandler("debug", _debug); + translate.setHandler("error", _error); + translate.setHandler("newTestDetectionFailed", _confirmCreateExpectedFailTest); + let newTest = await translate.newTest(); + translate.dispose(); + newTest = _sanitizeItemsInTest(newTest); + this.newTests.push(newTest); + this.testDoneCallback(newTest); + this._updateTests(); + } + catch (e) { + Zotero.logError(e); + this.newTests.push(false); + this.testDoneCallback(false); + this._updateTests(); + } + finally { + if (browser) HiddenBrowser.destroy(browser); + } } else { test.items = []; @@ -2092,23 +2111,8 @@ var Scaffold = new function () { return guid; } - /* - * gets selected frame/document - */ - function _getDocument() { - return new Promise((resolve) => { - window.messageManager.addMessageListener('Scaffold:Document', function onDocument({ data }) { - window.messageManager.removeMessageListener('Scaffold:Document', onDocument); - let doc = new DOMParser().parseFromString(data.html, 'text/html'); - doc = Zotero.HTTP.wrapDocument(doc, data.url); - resolve(doc); - }); - window.messageManager.broadcastAsyncMessage('Scaffold:GetDocument'); - }); - } - - function _getDocumentURL(doc) { - return Zotero.Proxies.proxyToProper(doc.location.href); + function _getCurrentURI(browser) { + return Zotero.Proxies.proxyToProper(browser.currentURI.spec); } function _findTestObjectTops(monaco, model) { diff --git a/chrome/content/zotero/HiddenBrowser.jsm b/chrome/content/zotero/HiddenBrowser.jsm index 36739e87cb..6d841911cc 100644 --- a/chrome/content/zotero/HiddenBrowser.jsm +++ b/chrome/content/zotero/HiddenBrowser.jsm @@ -43,6 +43,12 @@ ChromeUtils.registerWindowActor("PageData", { } }); +ChromeUtils.registerWindowActor("SingleFile", { + child: { + moduleURI: "chrome://zotero/content/actors/SingleFileChild.jsm" + } +}); + const progressListeners = new Set(); const browserFrameMap = new WeakMap(); @@ -182,7 +188,7 @@ const HiddenBrowser = { /** * @param {Browser} browser - * @param {String[]} props - 'characterSet', 'title', 'bodyText' + * @param {String[]} props - 'characterSet', 'title', 'bodyText', 'documentHTML', 'cookie', 'channelInfo' */ async getPageData(browser, props) { var actor = browser.browsingContext.currentWindowGlobal.getActor("PageData"); @@ -192,7 +198,34 @@ const HiddenBrowser = { } return data; }, - + + /** + * @param {Browser} browser + * @returns {Promise} + */ + async getDocument(browser) { + let { documentHTML, cookie } = await this.getPageData(browser, ['documentHTML', 'cookie']); + let doc = new DOMParser().parseFromString(documentHTML, 'text/html'); + let docWithLocation = Zotero.HTTP.wrapDocument(doc, browser.currentURI); + return new Proxy(docWithLocation, { + get(obj, prop) { + if (prop === 'cookie') { + return cookie; + } + return obj[prop]; + } + }); + }, + + /** + * @param {Browser} browser + * @returns {Promise} + */ + snapshot(browser) { + let actor = browser.browsingContext.currentWindowGlobal.getActor("SingleFile"); + return actor.sendQuery('snapshot'); + }, + destroy(browser) { var frame = browserFrameMap.get(browser); if (frame) { diff --git a/chrome/content/zotero/RemoteTranslate.jsm b/chrome/content/zotero/RemoteTranslate.jsm new file mode 100644 index 0000000000..854f88364c --- /dev/null +++ b/chrome/content/zotero/RemoteTranslate.jsm @@ -0,0 +1,222 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +var EXPORTED_SYMBOLS = ["RemoteTranslate"]; + +const { XPCOMUtils } = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); + +ChromeUtils.registerWindowActor("Translation", { + parent: { + moduleURI: "chrome://zotero/content/actors/TranslationParent.jsm" + }, + child: { + moduleURI: "chrome://zotero/content/actors/TranslationChild.jsm" + } +}); + +XPCOMUtils.defineLazyModuleGetters(this, { + Zotero: "chrome://zotero/content/include.jsm", + TranslationManager: "chrome://zotero/content/actors/TranslationParent.jsm", +}); + +class RemoteTranslate { + _browser = null; + + _id = Zotero.Utilities.randomString(); + + _translator = null; + + _doneHandlers = []; + + _wasSuccess = false; + + constructor() { + TranslationManager.add(this._id); + TranslationManager.setHandler(this._id, 'done', success => this._wasSuccess = success); + } + + /** + * @param {Browser} browser + * @return {Promise} + */ + async setBrowser(browser) { + this._browser = browser; + let actor = this._browser.browsingContext.currentWindowGlobal.getActor("Translation"); + await actor.sendAsyncMessage("initTranslation", { + schemaJSON: Zotero.File.getResource('resource://zotero/schema/global/schema.json'), + dateFormatsJSON: Zotero.File.getResource('resource://zotero/schema/dateFormats.json'), + }); + } + + /** + * Set a handler on the proxied Zotero.Translate instance. + * The handler function is passed this RemoteTranslate as its first argument. + * + * Supports all Zotero.Translate handlers in addition to newTestDetectionFailed, which can be called from + * {@link newTest} if detection fails to confirm the creation of an expected-fail test. + * + * @param {String} name + * @param {Function} handler + */ + setHandler(name, handler) { + // 'done' is triggered from translate() + if (name == 'done') { + this._doneHandlers.push(handler); + } + else { + TranslationManager.setHandler(this._id, name, (...args) => handler(this, ...args)); + } + } + + /** + * Clear the handlers for the given type on the proxied Zotero.Translate instance. + * + * @param {String} name + */ + clearHandlers(name) { + // 'done' is triggered from translate() + if (name == 'done') { + this._doneHandlers = []; + } + else { + TranslationManager.clearHandlers(this._id, name); + } + } + + /** + * @param {Zotero.Translators} translatorProvider + */ + setTranslatorProvider(translatorProvider) { + TranslationManager.setTranslatorProvider(this._id, translatorProvider); + } + + /** + * Set the translator used by #detect(), #translate(), #runTest(), and #newTest(). + * + * @param {Zotero.Translator} translator + */ + setTranslator(translator) { + this._translator = translator; + } + + /** + * Run detection on the browser's current page. Sets the translator that will be used by #translate(). + * + * @return {Promise} Resolves to detected translator array (null on error) + */ + async detect() { + let actor = this._browser.browsingContext.currentWindowGlobal.getActor("Translation"); + this._translator = await actor.sendQuery("detect", { translator: this._translator, id: this._id }); + return this._translator; + } + + /** + * Run translation on the browser's current page. + * + * @param {Number | false} [options.libraryID] false to disable saving + * @param {Number[]} [options.collections] + * @return {Promise} Resolves to returned items (null on error) + */ + async translate(options = {}) { + let actor = this._browser.browsingContext.currentWindowGlobal.getActor("Translation"); + let items = null; + try { + items = await actor.sendQuery("translate", { translator: this._translator, id: this._id }); + if (options.libraryID !== false) { + let itemSaver = new Zotero.Translate.ItemSaver({ + libraryID: options.libraryID, + collections: options.collections, + attachmentMode: Zotero.Translate.ItemSaver.ATTACHMENT_MODE_DOWNLOAD, + forceTagType: 1, + referrer: this._browser.currentURI.spec, + // proxy: unimplemented in the client + }); + + // Call itemDone on each completed item + let itemsDoneCallback = (jsonItems, dbItems) => { + for (let i = 0; i < jsonItems.length; i++) { + let jsonItem = jsonItems[i]; + let dbItem = dbItems[i]; + TranslationManager.runHandler(this._id, 'itemDone', dbItem, jsonItem); + } + }; + await itemSaver.saveItems(items, () => {}, itemsDoneCallback); + } + + // And call done (saved in #setHandler() above) at the end + // The Zotero.Translate instance running in the content process has already tried to call done by now, + // but we prevented it from reaching the caller. Now that we've run ItemSaver#saveItems() on this side, + // we can pass it through. + this._callDoneHandlers(this._wasSuccess); + } + catch (e) { + this._callDoneHandlers(false); + throw e; + } + return items; + } + + /** + * Run a test on the browser's current page. + * + * @param {Object} test Test object + * @return {Promise<{ test: Object, status: String, message: String } | null>} Null on error + */ + runTest(test) { + let actor = this._browser.browsingContext.currentWindowGlobal.getActor("Translation"); + return actor.sendQuery("runTest", { translator: this._translator, test, id: this._id }); + } + + /** + * Create a test on the browser's current page. + * + * @return {Promise} Resolves to the created test object (null on error) + */ + newTest() { + let actor = this._browser.browsingContext.currentWindowGlobal.getActor("Translation"); + return actor.sendQuery("newTest", { translator: this._translator, id: this._id }); + } + + /** + * Must be called to avoid memory leaks. + */ + dispose() { + if (this._id) { + TranslationManager.remove(this._id); + this._id = null; + } + } + + _callDoneHandlers(wasSuccess) { + for (let doneHandler of this._doneHandlers) { + try { + doneHandler(this, wasSuccess); + } + catch (e) { + Zotero.logError(e); + } + } + } +} diff --git a/chrome/content/zotero/actors/PageDataChild.jsm b/chrome/content/zotero/actors/PageDataChild.jsm index 6a92bec6dc..f0fafb401c 100644 --- a/chrome/content/zotero/actors/PageDataChild.jsm +++ b/chrome/content/zotero/actors/PageDataChild.jsm @@ -16,6 +16,27 @@ class PageDataChild extends JSWindowActorChild { case "bodyText": return document.documentElement.innerText; + + case "cookie": + return document.cookie; + + case "documentHTML": + return new XMLSerializer().serializeToString(document); + + case "channelInfo": { + let docShell = this.contentWindow.docShell; + let channel = (docShell.currentDocumentChannel || docShell.failedChannel) + ?.QueryInterface(Ci.nsIHttpChannel); + if (channel) { + return { + responseStatus: channel.responseStatus, + responseStatusText: channel.responseStatusText + }; + } + else { + return null; + } + } } } @@ -24,9 +45,8 @@ class PageDataChild extends JSWindowActorChild { const contentWindow = this.contentWindow; const document = this.document; - // Make sure the document element has been created function readyEnough() { - return document.readyState !== "uninitialized" && document.documentElement; + return document.readyState === "complete"; } if (readyEnough()) { diff --git a/chrome/content/zotero/actors/SingleFileChild.jsm b/chrome/content/zotero/actors/SingleFileChild.jsm new file mode 100644 index 0000000000..dfe252e67f --- /dev/null +++ b/chrome/content/zotero/actors/SingleFileChild.jsm @@ -0,0 +1,196 @@ +var EXPORTED_SYMBOLS = ["SingleFileChild"]; + +var { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +class SingleFileChild extends JSWindowActorChild { + async receiveMessage(message) { + let window = this.contentWindow; + + await this.documentIsReady(); + + if (message.name !== 'snapshot') { + return null; + } + + // Create sandbox for SingleFile + let sandbox = this.createSnapshotSandbox(window); + + const SCRIPTS = [ + // This first script replace in the INDEX_SCRIPTS from the single file cli loader + "lib/single-file.js", + + // Web SCRIPTS + "lib/single-file-hooks-frames.js", + ]; + + console.log('Injecting single file scripts'); + // Run all the scripts of SingleFile scripts in Sandbox + SCRIPTS.forEach( + script => Services.scriptloader.loadSubScript('resource://zotero/SingleFile/' + script, sandbox) + ); + // Import config + Services.scriptloader.loadSubScript('chrome://zotero/content/xpcom/singlefile.js', sandbox); + + // In the client we turn off this auto-zooming feature because it does not work + // since the hidden browser does not have a clientHeight. + Cu.evalInSandbox( + 'Zotero.SingleFile.CONFIG.loadDeferredImagesKeepZoomLevel = true;', + sandbox + ); + + console.log('Injecting single file scripts into frames'); + + // List of scripts from: + // resource/SingleFile/extension/lib/single-file/core/bg/scripts.js + const frameScripts = [ + "lib/single-file-hooks-frames.js", + "lib/single-file-frames.js", + ]; + + // Create sandboxes for all the frames we find + const frameSandboxes = []; + for (let i = 0; i < sandbox.window.frames.length; ++i) { + let frameSandbox = this.createSnapshotSandbox(sandbox.window.frames[i]); + + // Run all the scripts of SingleFile scripts in Sandbox + frameScripts.forEach( + script => Services.scriptloader.loadSubScript('resource://zotero/SingleFile/' + script, frameSandbox) + ); + + frameSandboxes.push(frameSandbox); + } + + // Use SingleFile to retrieve the html + const pageData = await Cu.evalInSandbox( + `this.singlefile.getPageData( + Zotero.SingleFile.CONFIG, + { fetch: ZoteroFetch } + );`, + sandbox + ); + + // Clone so we can nuke the sandbox + let content = pageData.content; + + // Nuke frames and then main sandbox + frameSandboxes.forEach(frameSandbox => Cu.nukeSandbox(frameSandbox)); + Cu.nukeSandbox(sandbox); + + return content; + } + + createSnapshotSandbox(view) { + let sandbox = new Cu.Sandbox(view, { + wantGlobalProperties: ["XMLHttpRequest", "fetch"], + sandboxPrototype: view + }); + sandbox.browser = false; + + sandbox.Zotero = Cu.cloneInto({ HTTP: {} }, sandbox); + sandbox.Zotero.debug = Cu.exportFunction(obj => console.log(obj), sandbox); + // Mostly copied from: + // resources/SingleFile/extension/lib/single-file/fetch/bg/fetch.js::fetchResource + sandbox.coFetch = Cu.exportFunction( + function (url, options, onDone) { + const xhrRequest = new XMLHttpRequest(); + xhrRequest.withCredentials = true; + xhrRequest.responseType = "arraybuffer"; + xhrRequest.onerror = () => { + let error = { error: `Request failed for ${url}` }; + onDone(Cu.cloneInto(error, sandbox)); + }; + xhrRequest.onreadystatechange = () => { + if (xhrRequest.readyState == XMLHttpRequest.DONE) { + if (xhrRequest.status || xhrRequest.response.byteLength) { + let res = { + array: new Uint8Array(xhrRequest.response), + headers: { "content-type": xhrRequest.getResponseHeader("Content-Type") }, + status: xhrRequest.status + }; + // Ensure sandbox will have access to response by cloning + onDone(Cu.cloneInto(res, sandbox)); + } + else { + let error = { error: 'Bad Status or Length' }; + onDone(Cu.cloneInto(error, sandbox)); + } + } + }; + xhrRequest.open("GET", url, true); + if (options && options.headers) { + for (const entry of Object.entries(options.headers)) { + xhrRequest.setRequestHeader(entry[0], entry[1]); + } + } + xhrRequest.send(); + }, + sandbox + ); + + // First we try regular fetch, then proceed with fetch outside sandbox to evade CORS + // restrictions, partly from: + // resources/SingleFile/extension/lib/single-file/fetch/content/content-fetch.js::fetch + Cu.evalInSandbox( + ` + ZoteroFetch = async function (url, options) { + try { + let response = await fetch(url, { cache: "force-cache", headers: options.headers }); + return response; + } + catch (error) { + let response = await new Promise((resolve, reject) => { + coFetch(url, { headers: options.headers }, (response) => { + if (response.error) { + Zotero.debug("Error retrieving url: " + url); + Zotero.debug(response); + reject(new Error(response.error)); + } + else { + resolve(response); + } + }); + }); + + return { + status: response.status, + headers: { get: headerName => response.headers[headerName] }, + arrayBuffer: async () => response.array.buffer + }; + } + };`, + sandbox + ); + + return sandbox; + } + + // From Mozilla's ScreenshotsComponentChild.jsm + documentIsReady() { + const contentWindow = this.contentWindow; + const document = this.document; + + function readyEnough() { + return document.readyState === "complete"; + } + + if (readyEnough()) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + function onChange(event) { + if (event.type === "pagehide") { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + reject(new Error("document unloaded before it was ready")); + } + else if (readyEnough()) { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + resolve(); + } + } + document.addEventListener("readystatechange", onChange); + contentWindow.addEventListener("pagehide", onChange, { once: true }); + }); + } +} diff --git a/chrome/content/zotero/actors/TranslationChild.jsm b/chrome/content/zotero/actors/TranslationChild.jsm new file mode 100644 index 0000000000..6f516ba8f4 --- /dev/null +++ b/chrome/content/zotero/actors/TranslationChild.jsm @@ -0,0 +1,346 @@ +var EXPORTED_SYMBOLS = ["TranslationChild"]; + +var { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +const TRANSLATE_SCRIPT_PATHS = [ + 'src/zotero.js', + 'src/promise.js', + 'modules/utilities/openurl.js', + 'modules/utilities/date.js', + 'modules/utilities/xregexp-all.js', + 'modules/utilities/xregexp-unicode-zotero.js', + 'modules/utilities/utilities.js', + 'modules/utilities/utilities_item.js', + 'modules/utilities/schema.js', + 'modules/utilities/resource/zoteroTypeSchemaData.js', + 'modules/utilities/cachedTypes.js', + 'src/utilities_translate.js', + 'src/debug.js', + 'src/http.js', + 'src/translator.js', + 'src/translators.js', + 'src/repo.js', + 'src/translation/translate.js', + 'src/translation/sandboxManager.js', + 'src/translation/translate_item.js', + 'src/tlds.js', + 'src/proxy.js', + 'src/rdf/init.js', + 'src/rdf/uri.js', + 'src/rdf/term.js', + 'src/rdf/identity.js', + 'src/rdf/n3parser.js', + 'src/rdf/rdfparser.js', + 'src/rdf/serialize.js', + 'testTranslators/translatorTester.js', +]; + +const OTHER_SCRIPT_URIS = [ + 'chrome://zotero/content/actors/translation/http.js', + 'chrome://zotero/content/actors/translation/translate_item.js', +]; + +class TranslationChild extends JSWindowActorChild { + _sandbox = null; + + async receiveMessage(message) { + await this.documentIsReady(); + + let { name, data } = message; + switch (name) { + case 'initTranslation': { + let { schemaJSON, dateFormatsJSON } = data; + this._sandbox = this._loadTranslationFramework(schemaJSON, dateFormatsJSON); + break; + } + case 'detect': { + let { translator, id } = data; + let Zotero = this._sandbox.Zotero.wrappedJSObject; + try { + let translate = new Zotero.Translate.Web(); + translate.setTranslatorProvider(this._makeTranslatorProvider(id)); + translate.setDocument(this.document); + this._initHandlers(id, translate); + if (translator) { + translate.setTranslator(Cu.cloneInto(translator, this._sandbox)); + } + return await translate.getTranslators(false, !!translator); + } + catch (e) { + this._error(id, e); + return null; + } + } + case 'translate': { + let { translator, id } = data; + let Zotero = this._sandbox.Zotero.wrappedJSObject; + try { + let translate = new Zotero.Translate.Web(); + translate.setTranslatorProvider(this._makeTranslatorProvider(id)); + translate.setDocument(this.document); + this._initHandlers(id, translate); + if (translator) { + translate.setTranslator(Cu.cloneInto(translator, this._sandbox)); + } + return await translate.translate(); + } + catch (e) { + this._error(id, e); + return null; + } + } + case 'runTest': { + let { translator, test, id } = data; + let Zotero_TranslatorTester = this._sandbox.Zotero_TranslatorTester.wrappedJSObject; + try { + let tester = new Zotero_TranslatorTester( + Cu.cloneInto(translator, this._sandbox), + test.type, + (_tester, obj) => this._debug(id, obj), + this._makeTranslatorProvider(id), + ); + return await new Promise((resolve) => { + tester.runTest( + Cu.cloneInto(test, this._sandbox), + this.contentWindow.document, + Cu.exportFunction( + (_, test, status, message) => resolve({ test, status, message }), + this._sandbox + ) + ); + }); + } + catch (e) { + this._error(id, e); + return null; + } + } + case 'newTest': { + let { translator, id } = data; + let Zotero_TranslatorTester = this._sandbox.Zotero_TranslatorTester.wrappedJSObject; + try { + let tester = new Zotero_TranslatorTester( + Cu.cloneInto(translator, this._sandbox), + 'web', + (_tester, obj) => this._debug(id, obj), + this._makeTranslatorProvider(id), + ); + return await new Promise((resolve) => { + tester.newTest( + this.contentWindow.document, + Cu.exportFunction( + (_, test) => resolve(test), + this._sandbox + ), + Cu.exportFunction( + () => this._sendQuerySafe('Translate:runHandler', { + id, + name: 'newTestDetectionFailed' + }), + this._sandbox + ) + ); + }); + } + catch (e) { + this._error(id, e); + return null; + } + } + } + } + + _makeTranslatorProvider(id) { + let Zotero = this._sandbox.Zotero.wrappedJSObject; + let makeProxy = method => ( + (...args) => this._sandbox.Promise.resolve( + this.sendQuery('Translators:call', { id, method, args }) + ).then(result => Cu.cloneInto(result, this._sandbox)) + ); + return Cu.cloneInto({ + ...Zotero.Translators, + get: makeProxy('get'), + getCodeForTranslator: makeProxy('getCodeForTranslator'), + getAllForType: makeProxy('getAllForType'), + getWebTranslatorsForLocation: makeProxy('getWebTranslatorsForLocation'), + }, this._sandbox, { cloneFunctions: true }); + } + + /** + * Wraps a call to sendQuery() so that any returned value or error is safe to access from the content window, + * and the returned promise can be `then`ed from the content window. + * + * @param {String} message + * @param {Object} value + * @return {Promise} + */ + _sendQuerySafe(message, value) { + return new this._sandbox.Promise((resolve, reject) => { + this.sendQuery(message, value) + .then(rv => Cu.cloneInto(rv, this._sandbox)) + .catch(e => this._sandbox.Promise.reject(new this._sandbox.Error(e.message))) + .then(resolve, reject); + }); + } + + /** + * Run the debug handler on the Zotero.Translate instance with the given ID + * @return {Promise} + */ + _debug(id, arg) { + let Zotero = this._sandbox.Zotero.wrappedJSObject; + if (typeof arg !== 'string') { + arg = Zotero.Utilities.varDump(arg); + } + // 8096K ought to be enough for anybody + // (And Firefox will throw an error when serializing very large values in fx102. + // Limit seems to have been removed in later versions.) + if (arg.length > 1024 * 8096) { + arg = arg.substring(0, 1024 * 1024); + } + return this._sendQuerySafe('Translate:runHandler', { + id, + name: 'debug', + arg + }); + } + + /** + * Run the error handler on the Zotero.Translate instance with the given ID + * @return {Promise} + */ + _error(id, arg) { + return this._sendQuerySafe('Translate:runHandler', { + id, + name: 'error', + arg: String(arg) + }); + } + + /** + * Initialize proxied handlers on the provided Zotero.Translate instance. + */ + _initHandlers(id, translate) { + let names = [ + "select", + "itemDone", + "collectionDone", + "done", + "debug", + "error", + "translators", + "pageModified", + ]; + for (let name of names) { + let handler; + if (name == 'debug') { + handler = (_, arg) => this._debug(id, arg); + } + else if (name == 'error') { + handler = (_, arg) => this._error(id, arg); + } + else if (name == 'select') { + handler = (_, items, callback) => { + this.sendQuery('Translate:runHandler', { id, name, arg: items }).then(items => { + callback(Cu.cloneInto(items, this._sandbox)); + }); + }; + } + else { + handler = (_, arg) => this._sendQuerySafe('Translate:runHandler', { id, name, arg }); + } + translate.setHandler(name, Cu.exportFunction(handler, this._sandbox)); + } + } + + /** + * Load the translation framework into the current page. + * @param {Object | String} schemaJSON + * @param {Object | String} dateFormatsJSON + * @return {Sandbox} + */ + _loadTranslationFramework(schemaJSON, dateFormatsJSON) { + let sandbox = new Cu.Sandbox(this.contentWindow, { + wantGlobalProperties: [ + "atob", + "btoa", + "Blob", + "crypto", + "CSS", + "CSSRule", + "Document", + "DOMParser", + "DOMTokenList", + "Element", + "Event", + "fetch", + "FormData", + "Headers", + "Node", + "NodeFilter", + "TextDecoder", + "TextEncoder", + "URL", + "URLSearchParams", + "Window", + "XMLHttpRequest" + ], + sandboxPrototype: this.contentWindow + }); + + let scriptURIs = [ + ...TRANSLATE_SCRIPT_PATHS.map(path => 'chrome://zotero/content/xpcom/translate/' + path), + ...OTHER_SCRIPT_URIS, + ]; + for (let scriptURI of scriptURIs) { + Services.scriptloader.loadSubScript(scriptURI, sandbox); + } + + let Zotero = sandbox.Zotero.wrappedJSObject; + + Zotero.Debug.init(1); + Zotero.Debug.setStore(true); + + Zotero.Translators._initialized = true; + Zotero.Schema.init(schemaJSON); + Zotero.Date.init(dateFormatsJSON); + + return sandbox; + } + + // From Mozilla's ScreenshotsComponentChild.jsm + documentIsReady() { + const contentWindow = this.contentWindow; + const document = this.document; + + function readyEnough() { + return document.readyState === "complete"; + } + + if (readyEnough()) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + function onChange(event) { + if (event.type === "pagehide") { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + reject(new Error("document unloaded before it was ready")); + } + else if (readyEnough()) { + document.removeEventListener("readystatechange", onChange); + contentWindow.removeEventListener("pagehide", onChange); + resolve(); + } + } + document.addEventListener("readystatechange", onChange); + contentWindow.addEventListener("pagehide", onChange, { once: true }); + }); + } + + didDestroy() { + if (this._sandbox) { + Cu.nukeSandbox(this._sandbox); + } + } +} diff --git a/chrome/content/zotero/actors/TranslationParent.jsm b/chrome/content/zotero/actors/TranslationParent.jsm new file mode 100644 index 0000000000..8bf693dc83 --- /dev/null +++ b/chrome/content/zotero/actors/TranslationParent.jsm @@ -0,0 +1,78 @@ +var EXPORTED_SYMBOLS = ["TranslationParent", "TranslationManager"]; + +const Zotero = Components.classes['@zotero.org/Zotero;1'] + .getService(Components.interfaces.nsISupports) + .wrappedJSObject; + +const TranslationManager = new class { + _registeredRemoteTranslates = new Map(); + + add(id) { + this._registeredRemoteTranslates.set(id, { + translatorProvider: null, + handlers: {}, + }); + } + + remove(id) { + this._registeredRemoteTranslates.delete(id); + } + + getTranslatorProvider(id) { + return this._registeredRemoteTranslates.get(id).translatorProvider; + } + + setTranslatorProvider(id, provider) { + this._registeredRemoteTranslates.get(id).translatorProvider = provider; + } + + setHandler(id, name, handler) { + if (this._registeredRemoteTranslates.get(id).handlers[name]) { + this._registeredRemoteTranslates.get(id).handlers[name] = [ + ...this._registeredRemoteTranslates.get(id).handlers[name], + handler + ]; + } + else { + this._registeredRemoteTranslates.get(id).handlers[name] = [handler]; + } + } + + clearHandlers(id, name) { + this._registeredRemoteTranslates.get(id).handlers[name] = null; + } + + async runHandler(id, name, ...args) { + let handlers = this._registeredRemoteTranslates.get(id).handlers[name]; + let returnValue = null; + if (handlers) { + for (let handler of handlers) { + try { + returnValue = await handler(...args); + } + catch (e) { + Zotero.logError(e); + } + } + } + return returnValue; + } +}; + +class TranslationParent extends JSWindowActorParent { + async receiveMessage(message) { + let { name, data } = message; + switch (name) { + case 'Translators:call': { + let { id, method, args } = data; + let provider = TranslationManager.getTranslatorProvider(id) || Zotero.Translators; + return provider[method](...args); + } + + case 'Translate:runHandler': { + let { id, name, arg } = data; + return TranslationManager.runHandler(id, name, arg); + } + } + } +} diff --git a/chrome/content/zotero/actors/translation/http.js b/chrome/content/zotero/actors/translation/http.js new file mode 100644 index 0000000000..61890f231c --- /dev/null +++ b/chrome/content/zotero/actors/translation/http.js @@ -0,0 +1,305 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2021 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * Functions for performing HTTP requests, both via XMLHTTPRequest and using a hidden browser + * @namespace + */ +Zotero.HTTP = new function() { + this.StatusError = function(xmlhttp, url) { + this.message = `HTTP request to ${url} rejected with status ${xmlhttp.status}`; + this.status = xmlhttp.status; + try { + this.responseText = typeof xmlhttp.responseText == 'string' ? xmlhttp.responseText : undefined; + } catch (e) {} + }; + this.StatusError.prototype = Object.create(Error.prototype); + + this.TimeoutError = function(ms) { + this.message = `HTTP request has timed out after ${ms}ms`; + }; + this.TimeoutError.prototype = Object.create(Error.prototype); + + /** + * Get a promise for a HTTP request + * + * @param {String} method The method of the request ("GET", "POST", "HEAD", or "OPTIONS") + * @param {String} url URL to request + * @param {Object} [options] Options for HTTP request:
    + *
  • body - The body of a POST request
    • + *
  • headers - Object of HTTP headers to send with the request
    • + *
  • debug - Log response text and status code
    • + *
  • logBodyLength - Length of request body to log
    • + *
  • timeout - Request timeout specified in milliseconds [default 15000]
    • + *
  • responseType - The response type of the request from the XHR spec
    • + *
  • responseCharset - The charset the response should be interpreted as
    • + *
  • successCodes - HTTP status codes that are considered successful, or FALSE to allow all
    • + *
+ * @return {Promise} A promise resolved with the XMLHttpRequest object if the + * request succeeds, or rejected if the browser is offline or a non-2XX status response + * code is received (or a code not in options.successCodes if provided). + */ + this.request = function(method, url, options = {}) { + // Default options + options = Object.assign({ + body: null, + headers: {}, + debug: false, + logBodyLength: 1024, + timeout: 15000, + responseType: '', + responseCharset: null, + successCodes: null + }, options); + + + let logBody = ''; + if (['GET', 'HEAD'].includes(method)) { + if (options.body != null) { + throw new Error(`HTTP ${method} cannot have a request body (${options.body})`) + } + } else if(options.body) { + options.body = typeof options.body == 'string' ? options.body : JSON.stringify(options.body); + + if (!options.headers) options.headers = {}; + if (!options.headers["Content-Type"]) { + options.headers["Content-Type"] = "application/x-www-form-urlencoded"; + } + else if (options.headers["Content-Type"] == 'multipart/form-data') { + // Allow XHR to set Content-Type with boundary for multipart/form-data + delete options.headers["Content-Type"]; + } + + logBody = `: ${options.body.substr(0, options.logBodyLength)}` + + options.body.length > options.logBodyLength ? '...' : ''; + // TODO: make sure below does its job in every API call instance + // Don't display password or session id in console + logBody = logBody.replace(/password":"[^"]+/, 'password":"********'); + logBody = logBody.replace(/password=[^&]+/, 'password=********'); + } + Zotero.debug(`HTTP ${method} ${url}${logBody}`); + + var xmlhttp = new XMLHttpRequest(); + xmlhttp.timeout = options.timeout; + var promise = Zotero.HTTP._attachHandlers(url, xmlhttp, options); + + xmlhttp.open(method, url, true); + + for (let header in options.headers) { + xmlhttp.setRequestHeader(header, options.headers[header]); + } + + xmlhttp.responseType = options.responseType || ''; + + // Maybe should provide "mimeType" option instead. This is xpcom legacy, where responseCharset + // could be controlled manually + if (options.responseCharset) { + xmlhttp.overrideMimeType("text/plain; charset=" + options.responseCharset); + } + + xmlhttp.send(options.body); + + return promise.then(function(xmlhttp) { + if (options.debug) { + if (xmlhttp.responseType == '' || xmlhttp.responseType == 'text') { + Zotero.debug(`HTTP ${xmlhttp.status} response: ${xmlhttp.responseText}`); + } + else { + Zotero.debug(`HTTP ${xmlhttp.status} response`); + } + } + + let invalidDefaultStatus = options.successCodes === null && !xmlhttp.responseURL.startsWith("file://") && + (xmlhttp.status < 200 || xmlhttp.status >= 300); + let invalidStatus = Array.isArray(options.successCodes) && !options.successCodes.includes(xmlhttp.status); + if (invalidDefaultStatus || invalidStatus) { + throw new Zotero.HTTP.StatusError(xmlhttp, url); + } + return xmlhttp; + }); + }; + /** + * Send an HTTP GET request via XMLHTTPRequest + * + * @deprecated Use {@link Zotero.HTTP.request} + * @param {String} url URL to request + * @param {Function} onDone Callback to be executed upon request completion + * @param {String} responseCharset + * @param {N/A} cookieSandbox Not used in Connector + * @param {Object} headers HTTP headers to include with the request + * @return {Boolean} True if the request was sent, or false if the browser is offline + */ + this.doGet = function(url, onDone, responseCharset, cookieSandbox, headers) { + Zotero.debug('Zotero.HTTP.doGet is deprecated. Use Zotero.HTTP.request'); + this.request('GET', url, {responseCharset, headers}) + .then(onDone, function(e) { + onDone({status: e.status, responseText: e.responseText}); + throw (e); + }); + return true; + }; + + /** + * Send an HTTP POST request via XMLHTTPRequest + * + * @deprecated Use {@link Zotero.HTTP.request} + * @param {String} url URL to request + * @param {String|Object[]} body Request body + * @param {Function} onDone Callback to be executed upon request completion + * @param {String} headers Request HTTP headers + * @param {String} responseCharset + * @return {Boolean} True if the request was sent, or false if the browser is offline + */ + this.doPost = function(url, body, onDone, headers, responseCharset) { + Zotero.debug('Zotero.HTTP.doPost is deprecated. Use Zotero.HTTP.request'); + this.request('POST', url, {body, responseCharset, headers}) + .then(onDone, function(e) { + onDone({status: e.status, responseText: e.responseText}); + throw (e); + }); + return true; + }; + + + /** + * Load one or more documents via XMLHttpRequest + * + * Based on equivalent code from zotero-connectors. + * + * @param {String|String[]} urls - URL(s) of documents to load + * @param {Function} processor - Callback to be executed for each document loaded + * @return {Promise} - A promise for an array of results from the processor runs + */ + this.processDocuments = async function (urls, processor) { + // Handle old signature: urls, processor, onDone, onError + if (typeof arguments[2] == 'function' || typeof arguments[3] == 'function') { + Zotero.debug("Zotero.HTTP.processDocuments() no longer takes onDone or onError -- update your code"); + var onDone = arguments[2]; + var onError = arguments[3]; + } + + if (typeof urls == "string") urls = [urls]; + var funcs = urls.map(url => () => { + return Zotero.HTTP.request( + "GET", + url, + { + responseType: 'document' + } + ) + .then((xhr) => { + let doc = Zotero.HTTP.wrapDocument(xhr.response, url); + return processor(doc, url); + }); + }); + + // Run processes serially + // TODO: Add some concurrency? + var f; + var results = []; + while ((f = funcs.shift())) { + try { + results.push(await f()); + } + catch (e) { + if (onError) { + onError(e); + } + throw e; + } + } + + // Deprecated + if (onDone) { + onDone(); + } + + return results; + }; + + + /** + * Adds a ES6 Proxied location attribute + * @param doc + * @param docUrl + */ + this.wrapDocument = function(doc, docURL) { + docURL = new URL(docURL); + docURL.toString = () => this.href; + var wrappedDoc = new Proxy(doc, { + get: function (t, prop) { + if (prop === 'location') { + return docURL; + } + else if (prop == 'evaluate') { + // If you pass the document itself into doc.evaluate as the second argument + // it fails, because it receives a proxy, which isn't of type `Node` for some reason. + // Native code magic. + return function() { + if (arguments[1] == wrappedDoc) { + arguments[1] = t; + } + return t.evaluate.apply(t, arguments) + } + } + else { + if (typeof t[prop] == 'function') { + return t[prop].bind(t); + } + return t[prop]; + } + } + }); + return wrappedDoc; + }; + + + /** + * Adds request handlers to the XMLHttpRequest and returns a promise that resolves when + * the request is complete. xmlhttp.send() still needs to be called, this just attaches the + * handler + * + * See {@link Zotero.HTTP.request} for parameters + * @private + */ + this._attachHandlers = function(url, xmlhttp, options) { + var deferred = Zotero.Promise.defer(); + xmlhttp.onload = () => deferred.resolve(xmlhttp); + xmlhttp.onerror = xmlhttp.onabort = function() { + var e = new Zotero.HTTP.StatusError(xmlhttp, url); + if (options.successCodes === false) { + deferred.resolve(xmlhttp); + } else { + deferred.reject(e); + } + }; + xmlhttp.ontimeout = function() { + var e = new Zotero.HTTP.TimeoutError(xmlhttp.timeout); + Zotero.logError(e); + deferred.reject(e); + }; + return deferred.promise; + }; +} diff --git a/chrome/content/zotero/actors/translation/translate_item.js b/chrome/content/zotero/actors/translation/translate_item.js new file mode 100644 index 0000000000..909a730bac --- /dev/null +++ b/chrome/content/zotero/actors/translation/translate_item.js @@ -0,0 +1,29 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2021 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +Zotero.Translate.ItemSaver.prototype.saveItems = async function (jsonItems) { + this.items = (this.items || []).concat(jsonItems); + return jsonItems; +}; diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index b20d245664..c1dece9f12 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -24,6 +24,8 @@ */ Zotero.Attachments = new function(){ + const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + // Keep in sync with Zotero.Schema.integrityCheck() and this.linkModeToName() this.LINK_MODE_IMPORTED_FILE = 0; this.LINK_MODE_IMPORTED_URL = 1; @@ -539,39 +541,30 @@ Zotero.Attachments = new function(){ } // Save using a hidden browser - var nativeHandlerImport = function () { - return new Zotero.Promise(function (resolve, reject) { - var browser = Zotero.HTTP.loadDocuments( - url, - Zotero.Promise.coroutine(function* () { - try { - let attachmentItem = yield Zotero.Attachments.importFromDocument({ - libraryID, - document: browser.contentDocument, - parentItemID, - title, - collections, - saveOptions - }); - resolve(attachmentItem); - } - catch (e) { - Zotero.logError(e); - reject(e); - } - finally { - Zotero.Browser.deleteHiddenBrowser(browser); - } - }), - undefined, - (e) => { - reject(e); - }, - true, + var nativeHandlerImport = async function () { + let browser; + try { + browser = await HiddenBrowser.create(url, { + requireSuccessfulStatus: true, + docShell: { allowImages: true }, cookieSandbox, - { allowImages: true } - ); - }); + }); + return await Zotero.Attachments.importFromDocument({ + libraryID, + browser, + parentItemID, + title, + collections, + saveOptions + }); + } + catch (e) { + Zotero.logError(e); + throw e; + } + finally { + if (browser) HiddenBrowser.destroy(browser); + } }; // Save using remote web browser persist @@ -855,7 +848,7 @@ Zotero.Attachments = new function(){ /** * Save a snapshot from a Document * - * @param {Object} options - 'libraryID', 'document', 'parentItemID', 'forceTitle', 'collections' + * @param {Object} options - 'libraryID', 'document', 'browser', 'parentItemID', 'forceTitle', 'collections' * @param {Object} [options.saveOptions] - Options to pass to Zotero.Item::save() * @return {Promise} - A promise for the created attachment item */ @@ -864,6 +857,7 @@ Zotero.Attachments = new function(){ var libraryID = options.libraryID; var document = options.document; + var browser = options.browser; var parentItemID = options.parentItemID; var title = options.title; var collections = options.collections; @@ -873,10 +867,14 @@ Zotero.Attachments = new function(){ throw new Error("parentItemID and parentCollectionIDs cannot both be provided"); } - var url = document.location.href; - title = title ? title : document.title; - var contentType = document.contentType; - if (Zotero.Attachments.isPDFJS(document)) { + if (!document && !browser) { + throw new Error("Either document or browser must be provided"); + } + + var url = document ? document.location.href : browser.currentURI.spec; + title = title ? title : (document ? document.title : browser.contentTitle); + var contentType = document ? document.contentType : browser.documentContentType; + if (document ? Zotero.Attachments.isPDFJSDocument(document) : Zotero.Attachments.isPDFJSBrowser(browser)) { contentType = "application/pdf"; } @@ -900,11 +898,11 @@ Zotero.Attachments = new function(){ if ((contentType === 'text/html' || contentType === 'application/xhtml+xml') // Documents from XHR don't work here - && Zotero.Translate.DOMWrapper.unwrap(document) instanceof Document) { - if (document.defaultView.window) { + && (browser || Zotero.Translate.DOMWrapper.unwrap(document) instanceof Document)) { + if (browser) { // If we have a full hidden browser, use SingleFile - Zotero.debug('Getting snapshot with snapshotDocument()'); - let snapshotContent = yield Zotero.Utilities.Internal.snapshotDocument(document); + Zotero.debug('Getting snapshot with HiddenBrowser.snapshot()'); + let snapshotContent = yield HiddenBrowser.snapshot(browser); // Write main HTML file to disk yield Zotero.File.putContentsAsync(tmpFile, snapshotContent); diff --git a/chrome/content/zotero/xpcom/connector/server_connector.js b/chrome/content/zotero/xpcom/connector/server_connector.js index 4a04a09e0b..fdfcb305b9 100644 --- a/chrome/content/zotero/xpcom/connector/server_connector.js +++ b/chrome/content/zotero/xpcom/connector/server_connector.js @@ -930,6 +930,8 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { * Save SingleFile snapshot to pending attachments */ init: async function (requestData) { + const { HiddenBrowser } = ChromeUtils.import('chrome://zotero/content/HiddenBrowser.jsm'); + // Retrieve payload let data = requestData.data; @@ -982,27 +984,19 @@ Zotero.Server.Connector.SaveSingleFile.prototype = { let url = session.pendingAttachments[0][1].url; - snapshotContent = await new Zotero.Promise(function (resolve, reject) { - var browser = Zotero.HTTP.loadDocuments( - url, - Zotero.Promise.coroutine(function* () { - try { - resolve(yield Zotero.Utilities.Internal.snapshotDocument(browser.contentDocument)); - } - catch (e) { - Zotero.logError(e); - reject(e); - } - finally { - Zotero.Browser.deleteHiddenBrowser(browser); - } - }), - undefined, - undefined, - true, - cookieSandbox - ); + let browser = await HiddenBrowser.create(url, { + requireSuccessfulStatus: true, + docShell: { + allowImages: true + }, + cookieSandbox, }); + try { + snapshotContent = await HiddenBrowser.snapshot(browser); + } + finally { + HiddenBrowser.destroy(browser); + } } else { snapshotContent = data.snapshotContent; diff --git a/chrome/content/zotero/xpcom/data/feedItem.js b/chrome/content/zotero/xpcom/data/feedItem.js index affff08a8e..0cd445a2d0 100644 --- a/chrome/content/zotero/xpcom/data/feedItem.js +++ b/chrome/content/zotero/xpcom/data/feedItem.js @@ -207,103 +207,99 @@ Zotero.FeedItem.prototype.toggleRead = Zotero.Promise.coroutine(function* (state * @param collectionID {Integer} add item to collection * @return {Promise} translated feed item */ -Zotero.FeedItem.prototype.translate = Zotero.Promise.coroutine(function* (libraryID, collectionID) { +Zotero.FeedItem.prototype.translate = async function (libraryID, collectionID) { + const { RemoteTranslate } = ChromeUtils.import("chrome://zotero/content/RemoteTranslate.jsm"); + const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); + Zotero.debug("Translating feed item " + this.id + " with URL " + this.getField('url'), 2); if (Zotero.locked) { Zotero.debug('Zotero locked, skipping feed item translation'); return; } - let deferred = Zotero.Promise.defer(); let error = function(e) { }; - let translate = new Zotero.Translate.Web(); + let translate = new RemoteTranslate(); var win = Services.wm.getMostRecentWindow("navigator:browser"); let progressWindow = win.ZoteroPane.progressWindow; if (libraryID) { // Show progress notifications when scraping to a library. - translate.clearHandlers("done"); - translate.clearHandlers("itemDone"); translate.setHandler("done", progressWindow.Translation.doneHandler); translate.setHandler("itemDone", progressWindow.Translation.itemDoneHandler()); if (collectionID) { - var collection = yield Zotero.Collections.getAsync(collectionID); + var collection = await Zotero.Collections.getAsync(collectionID); } progressWindow.show(); progressWindow.Translation.scrapingTo(libraryID, collection); } - // Load document + // Load document in hidden browser and point the RemoteTranslate to it + let browser = await HiddenBrowser.create(this.getField('url')); try { - yield Zotero.HTTP.processDocuments(this.getField('url'), doc => deferred.resolve(doc)); - } catch (e) { - Zotero.debug(e, 1); - deferred.reject(e); - } - let doc = yield deferred.promise; - - // Set translate document - translate.setDocument(doc); - - // Load translators - deferred = Zotero.Promise.defer(); - translate.setHandler('translators', (me, translators) => deferred.resolve(translators)); - translate.getTranslators(); - let translators = yield deferred.promise; - if (!translators || !translators.length) { - Zotero.debug("No translators detected for feed item " + this.id + " with URL " + this.getField('url') + - ' -- cloning item instead', 2); - let item = yield this.clone(libraryID, collectionID, doc); - progressWindow.Translation.itemDoneHandler()(null, null, item); - progressWindow.Translation.doneHandler(null, true); - return; - } - translate.setTranslator(translators[0]); - - deferred = Zotero.Promise.defer(); - - if (libraryID) { - let result = yield translate.translate({libraryID, collections: collectionID ? [collectionID] : false}) - .then(items => items ? items[0] : false); - if (!result) { - let item = yield this.clone(libraryID, collectionID, doc); + await translate.setBrowser(browser); + + // Load translators + let translators = await translate.detect(); + if (!translators || !translators.length) { + Zotero.debug("No translators detected for feed item " + this.id + " with URL " + this.getField('url') + + ' -- cloning item instead', 2); + let item = await this.clone(libraryID, collectionID, browser); progressWindow.Translation.itemDoneHandler()(null, null, item); progressWindow.Translation.doneHandler(null, true); return; } - return result; + + if (libraryID) { + let result = await translate.translate({ + libraryID, + collections: collectionID ? [collectionID] : false + }).then(items => items ? items[0] : false); + if (!result) { + let item = await this.clone(libraryID, collectionID, browser); + progressWindow.Translation.itemDoneHandler()(null, null, item); + progressWindow.Translation.doneHandler(null, true); + return; + } + return result; + } + + translate.setHandler('error', error); + + let itemData = await translate.translate({ libraryID: false, saveAttachments: false }); + if (itemData.length) { + itemData = itemData[0]; + } + else { + Zotero.debug('Zotero.FeedItem#translate: Translation failed'); + return null; + } + + // clean itemData + const deleteFields = ['attachments', 'notes', 'id', 'itemID', 'path', 'seeAlso', 'version', 'dateAdded', 'dateModified']; + for (let field of deleteFields) { + delete itemData[field]; + } + + this.fromJSON(itemData); + this.isTranslated = true; + await this.saveTx(); + + return this; } - - // Clear these to prevent saving - translate.clearHandlers('itemDone'); - translate.clearHandlers('itemsDone'); - translate.setHandler('error', error); - translate.setHandler('itemDone', (_, items) => deferred.resolve(items)); - - translate.translate({libraryID: false, saveAttachments: false}); - - let itemData = yield deferred.promise; - - // clean itemData - const deleteFields = ['attachments', 'notes', 'id', 'itemID', 'path', 'seeAlso', 'version', 'dateAdded', 'dateModified']; - for (let field of deleteFields) { - delete itemData[field]; - } - - this.fromJSON(itemData); - this.isTranslated = true; - yield this.saveTx(); - - return this; -}); + finally { + HiddenBrowser.destroy(browser); + translate.dispose(); + } +}; /** * Clones the feed item (usually, when proper translation is unavailable) - * @param libraryID {Integer} save item in library - * @param collectionID {Integer} add item to collection + * @param {Integer} libraryID save item in library + * @param {Integer} [collectionID] add item to collection + * @param {Browser} [browser] * @return {Promise} translated feed item */ -Zotero.FeedItem.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, collectionID, doc) { +Zotero.FeedItem.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, collectionID, browser) { let dbItem = Zotero.Item.prototype.clone.call(this, libraryID); if (collectionID) { dbItem.addToCollection(collectionID); @@ -313,10 +309,10 @@ Zotero.FeedItem.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, let item = {title: dbItem.getField('title'), itemType: dbItem.itemType, attachments: []}; // Add snapshot - if (Zotero.Libraries.get(libraryID).filesEditable) { + if (Zotero.Libraries.get(libraryID).filesEditable && browser) { item.attachments = [{title: "Snapshot"}]; yield Zotero.Attachments.importFromDocument({ - document: doc, + browser, parentItemID: dbItem.id }); } diff --git a/chrome/content/zotero/xpcom/http.js b/chrome/content/zotero/xpcom/http.js index e826ef9f74..8729738471 100644 --- a/chrome/content/zotero/xpcom/http.js +++ b/chrome/content/zotero/xpcom/http.js @@ -1198,169 +1198,6 @@ Zotero.HTTP = new function() { return results; }; - - /** - * Load one or more documents in a hidden browser - * - * @param {String|String[]} urls URL(s) of documents to load - * @param {Function} processor - Callback to be executed for each document loaded; if function returns - * a promise, it's waited for before continuing - * @param {Function} onDone - Callback to be executed after all documents have been loaded - * @param {Function} onError - Callback to be executed if an error occurs - * @param {Boolean} dontDelete Don't delete the hidden browser upon completion; calling function - * must call deleteHiddenBrowser itself. - * @param {Zotero.CookieSandbox} [cookieSandbox] Cookie sandbox object - * @param {Object} [docShellPrefs] See Zotero.Browser.createHiddenBrowser - * @return {browser} Hidden browser used for loading - */ - this.loadDocuments = function (urls, processor, onDone, onError, dontDelete, cookieSandbox, docShellPrefs={}) { - // (Approximately) how many seconds to wait if the document is left in the loading state and - // pageshow is called before we call pageshow with an incomplete document - const LOADING_STATE_TIMEOUT = 120; - var firedLoadEvent = 0; - - /** - * Loads the next page - * @inner - */ - var doLoad = function() { - if(currentURL < urls.length) { - var url = urls[currentURL], - hiddenBrowser = hiddenBrowsers[currentURL]; - firedLoadEvent = 0; - currentURL++; - try { - Zotero.debug("Zotero.HTTP.loadDocuments: Loading " + url); - let loadURIOptions = { - triggeringPrincipal: null, - csp: null, - loadFlags: Components.interfaces.nsIWebNavigation.LOAD_FLAGS_BYPASS_HISTORY, - referrerInfo: null, - postData: null, - }; - hiddenBrowser.loadURI(url, loadURIOptions); - } catch(e) { - if (onError) { - onError(e); - return; - } else { - if(!dontDelete) Zotero.Browser.deleteHiddenBrowser(hiddenBrowsers); - throw(e); - } - } - } else { - if(!dontDelete) Zotero.Browser.deleteHiddenBrowser(hiddenBrowsers); - if (onDone) onDone(); - } - }; - - /** - * Callback to be executed when a page load completes - * @inner - */ - var onLoad = function(e) { - var hiddenBrowser = e.currentTarget, - doc = hiddenBrowser.contentDocument; - if(hiddenBrowser.zotero_loaded) return; - if(!doc) return; - var url = doc.documentURI; - if(url === "about:blank") return; - if(doc.readyState === "loading" && (firedLoadEvent++) < 120) { - // Try again in a second - Zotero.setTimeout(onLoad.bind(this, {"currentTarget":hiddenBrowser}), 1000); - return; - } - - hiddenBrowser.removeEventListener("load", onLoad, true); - hiddenBrowser.zotero_loaded = true; - - let channel = hiddenBrowser.docShell.currentDocumentChannel; - if (channel && (channel instanceof Components.interfaces.nsIHttpChannel)) { - if (channel.responseStatus < 200 || channel.responseStatus >= 400) { - let response = `${channel.responseStatus} ${channel.responseStatusText}`; - Zotero.debug(`Zotero.HTTP.loadDocuments: ${url} failed with ${response}`, 2); - let e = new Zotero.HTTP.UnexpectedStatusException( - { - status: channel.responseStatus, - channel - }, - url, - `Invalid response ${response} for ${url}` - ); - if (onError) { - onError(e); - } - else { - throw e; - } - return; - } - } - - Zotero.debug("Zotero.HTTP.loadDocuments: " + url + " loaded"); - - var maybePromise; - var error; - try { - maybePromise = processor(doc); - } - catch (e) { - error = e; - } - - // If processor returns a promise, wait for it - if (maybePromise && maybePromise.then) { - maybePromise.then(() => doLoad()) - .catch(e => { - if (onError) { - onError(e); - } - else { - throw e; - } - }); - return; - } - - try { - if (error) { - if (onError) { - onError(error); - } - else { - throw error; - } - } - } - finally { - doLoad(); - } - }; - - if(typeof(urls) == "string") urls = [urls]; - - var hiddenBrowsers = [], - currentURL = 0; - for(var i=0; i loadSubScript('resource://zotero/SingleFile/' + script, sandbox) - ); - // Import config - loadSubScript('chrome://zotero/content/xpcom/singlefile.js', sandbox); - - // In the client we turn off this auto-zooming feature because it does not work - // since the hidden browser does not have a clientHeight. - Components.utils.evalInSandbox( - 'Zotero.SingleFile.CONFIG.loadDeferredImagesKeepZoomLevel = true;', - sandbox - ); - - Zotero.debug('Injecting single file scripts into frames'); - - // List of scripts from: - // resource/SingleFile/extension/lib/single-file/core/bg/scripts.js - const frameScripts = [ - "lib/single-file-hooks-frames.js", - "lib/single-file-frames.js", - ]; - - // Create sandboxes for all the frames we find - const frameSandboxes = []; - for (let i = 0; i < sandbox.window.frames.length; ++i) { - let frameSandbox = Zotero.Utilities.Internal.createSnapshotSandbox(sandbox.window.frames[i]); - - // Run all the scripts of SingleFile scripts in Sandbox - frameScripts.forEach( - script => loadSubScript('resource://zotero/SingleFile/' + script, frameSandbox) - ); - - frameSandboxes.push(frameSandbox); - } - - // Use SingleFile to retrieve the html - const pageData = await Components.utils.evalInSandbox( - `this.singlefile.getPageData( - Zotero.SingleFile.CONFIG, - { fetch: ZoteroFetch } - );`, - sandbox - ); - - // Clone so we can nuke the sandbox - let content = pageData.content; - - // Nuke frames and then main sandbox - frameSandboxes.forEach(frameSandbox => Components.utils.nukeSandbox(frameSandbox)); - Components.utils.nukeSandbox(sandbox); - - return content; - }, - - - createSnapshotSandbox: function (view) { - let sandbox = new Components.utils.Sandbox(view, { - wantGlobalProperties: ["XMLHttpRequest", "fetch"], - sandboxPrototype: view - }); - sandbox.window = view.window; - sandbox.document = sandbox.window.document; - sandbox.browser = false; - // See comment in babel-worker.js - sandbox.globalThis = view.window; - - sandbox.Zotero = Components.utils.cloneInto({ HTTP: {} }, sandbox); - sandbox.Zotero.debug = Components.utils.exportFunction(Zotero.debug, sandbox); - // Mostly copied from: - // resources/SingleFile/extension/lib/single-file/fetch/bg/fetch.js::fetchResource - sandbox.coFetch = Components.utils.exportFunction( - function (url, options, onDone) { - const xhrRequest = new XMLHttpRequest(); - xhrRequest.withCredentials = true; - xhrRequest.responseType = "arraybuffer"; - xhrRequest.onerror = () => { - let error = { error: `Request failed for ${url}` }; - onDone(Components.utils.cloneInto(error, sandbox)); - }; - xhrRequest.onreadystatechange = () => { - if (xhrRequest.readyState == XMLHttpRequest.DONE) { - if (xhrRequest.status || xhrRequest.response.byteLength) { - let res = { - array: new Uint8Array(xhrRequest.response), - headers: { "content-type": xhrRequest.getResponseHeader("Content-Type") }, - status: xhrRequest.status - }; - // Ensure sandbox will have access to response by cloning - onDone(Components.utils.cloneInto(res, sandbox)); - } - else { - let error = { error: 'Bad Status or Length' }; - onDone(Components.utils.cloneInto(error, sandbox)); - } - } - }; - xhrRequest.open("GET", url, true); - if (options && options.headers) { - for (const entry of Object.entries(options.headers)) { - xhrRequest.setRequestHeader(entry[0], entry[1]); - } - } - xhrRequest.send(); - }, - sandbox - ); - - // First we try regular fetch, then proceed with fetch outside sandbox to evade CORS - // restrictions, partly from: - // resources/SingleFile/extension/lib/single-file/fetch/content/content-fetch.js::fetch - Components.utils.evalInSandbox( - ` - ZoteroFetch = async function (url, options) { - try { - let response = await fetch(url, { cache: "force-cache", headers: options.headers }); - return response; - } - catch (error) { - let response = await new Promise((resolve, reject) => { - coFetch(url, { headers: options.headers }, (response) => { - if (response.error) { - Zotero.debug("Error retrieving url: " + url); - Zotero.debug(response); - reject(new Error(response.error)); - } - else { - resolve(response); - } - }); - }); - - return { - status: response.status, - headers: { get: headerName => response.headers[headerName] }, - arrayBuffer: async () => response.array.buffer - }; - } - };`, - sandbox - ); - - return sandbox; - }, - - /** * Launch a process * @param {nsIFile|String} cmd Path to command to launch diff --git a/test/tests/HiddenBrowserTest.js b/test/tests/HiddenBrowserTest.js index d7e019c278..d47a79ba12 100644 --- a/test/tests/HiddenBrowserTest.js +++ b/test/tests/HiddenBrowserTest.js @@ -57,4 +57,25 @@ describe("HiddenBrowser", function() { assert.equal(bodyText, '这是一个测试文件。'); }); }); -}); \ No newline at end of file + + describe("#getDocument()", function () { + it("should provide a Document object", async function () { + let path = OS.Path.join(getTestDataDirectory().path, 'test-hidden.html'); + let browser = await HiddenBrowser.create(path); + let document = await HiddenBrowser.getDocument(browser); + assert.include(document.documentElement.innerHTML, 'test'); + assert.ok(document.location); + assert.strictEqual(document.cookie, ''); + }); + }); + + describe("#snapshot()", function () { + it("should return a SingleFile snapshot", async function () { + let path = OS.Path.join(getTestDataDirectory().path, 'test-hidden.html'); + let browser = await HiddenBrowser.create(path); + let snapshot = await HiddenBrowser.snapshot(browser); + assert.include(snapshot, 'Page saved with SingleFile'); + assert.include(snapshot, 'This is hidden text.'); + }); + }); +}); diff --git a/test/tests/RemoteTranslateTest.js b/test/tests/RemoteTranslateTest.js new file mode 100644 index 0000000000..9a93c28dc6 --- /dev/null +++ b/test/tests/RemoteTranslateTest.js @@ -0,0 +1,162 @@ +"use strict"; + +const { HiddenBrowser } = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm"); +const { RemoteTranslate } = ChromeUtils.import("chrome://zotero/content/RemoteTranslate.jsm"); + +describe("RemoteTranslate", function () { + let dummyTranslator; + let translatorProvider; + before(function () { + dummyTranslator = buildDummyTranslator('web', ` + function detectWeb() { + Zotero.debug("test string"); + return "book"; + } + + function doWeb() { + let item = new Zotero.Item("book"); + item.title = "Title"; + item.complete(); + } + `); + + translatorProvider = Zotero.Translators.makeTranslatorProvider({ + get(translatorID) { + if (translatorID == dummyTranslator.translatorID) { + return dummyTranslator; + } + return false; + }, + + async getAllForType(type) { + var translators = []; + if (type == 'web') { + translators.push(dummyTranslator); + } + return translators; + } + }); + }); + + describe("#setHandler()", function () { + it("should receive handler calls from the translator", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + await translate.setTranslator(dummyTranslator); + + let debug = sinon.spy(); + translate.setHandler('debug', debug); + await translate.detect(); + sinon.assert.calledWith(debug, translate, 'test string'); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + }); + + describe("#setTranslatorProvider()", function () { + it("should cause the passed provider to be queried instead of Zotero.Translators", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + translate.setTranslatorProvider(translatorProvider); + + let detectedTranslators = await translate.detect(); + assert.deepEqual(detectedTranslators.map(t => t.translatorID), [dummyTranslator.translatorID]); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + }); + + describe("#translate()", function () { + it("should return items without saving when libraryID is false", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + translate.setTranslatorProvider(translatorProvider); + + let detectedTranslators = await translate.detect(); + assert.equal(detectedTranslators[0].translatorID, dummyTranslator.translatorID); + + let itemDone = sinon.spy(); + translate.setHandler('itemDone', itemDone); + + let items = await translate.translate({ libraryID: false }); + sinon.assert.notCalled(itemDone); // No items should be saved + assert.equal(items[0].title, 'Title'); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + + it("should save items and call itemDone when libraryID is not false", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + translate.setTranslator(dummyTranslator); + + let itemDone = sinon.spy(); + translate.setHandler('itemDone', itemDone); + + let items = await translate.translate({ libraryID: null }); // User library + sinon.assert.calledWith(itemDone, translate, + sinon.match({ + libraryID: Zotero.Libraries.userLibraryID + }), + sinon.match({ + title: 'Title' + })); + // Item should still be returned + assert.equal(items[0].title, 'Title'); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + + it("should save items and call itemDone when libraryID is not false", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + translate.setTranslator(dummyTranslator); + + let itemDone = sinon.spy(); + translate.setHandler('itemDone', itemDone); + + let items = await translate.translate({ libraryID: null }); // User library + sinon.assert.calledWith(itemDone, translate, + sinon.match({ + libraryID: Zotero.Libraries.userLibraryID + }), + sinon.match({ + title: 'Title' + })); + // Item should still be returned + assert.equal(items[0].title, 'Title'); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + + it("should call itemDone before done", async function () { + let translate = new RemoteTranslate(); + let browser = await HiddenBrowser.create(getTestDataUrl('test.html')); + await translate.setBrowser(browser); + translate.setTranslator(dummyTranslator); + + let itemDone = sinon.spy(); + translate.setHandler('itemDone', itemDone); + let done = sinon.spy(); + translate.setHandler('done', done); + + await translate.translate({ libraryID: null }); // User library + sinon.assert.calledOnce(itemDone); + sinon.assert.calledOnce(done); + assert.isTrue(itemDone.calledBefore(done)); + + HiddenBrowser.destroy(browser); + translate.dispose(); + }); + }); +}); diff --git a/test/tests/attachmentsTest.js b/test/tests/attachmentsTest.js index da62916df6..8341bee3e9 100644 --- a/test/tests/attachmentsTest.js +++ b/test/tests/attachmentsTest.js @@ -1,14 +1,15 @@ describe("Zotero.Attachments", function() { - var win; + var HiddenBrowser; + var browser; - before(function* () { - // Hidden browser, which requires a browser window, needed for charset detection - // (until we figure out a better way) - win = yield loadBrowserWindow(); + before(function () { + HiddenBrowser = ChromeUtils.import("chrome://zotero/content/HiddenBrowser.jsm").HiddenBrowser; }); - after(function () { - if (win) { - win.close(); + + afterEach(function () { + if (browser) { + HiddenBrowser.destroy(browser); + browser = null; } }); @@ -305,15 +306,12 @@ describe("Zotero.Attachments", function() { var item = yield createDataObject('item'); var uri = OS.Path.join(getTestDataDirectory().path, "snapshot", "index.html"); - var deferred = Zotero.Promise.defer(); - win.addEventListener('pageshow', () => deferred.resolve()); - win.loadURI(uri); - yield deferred.promise; + browser = yield HiddenBrowser.create(uri); var file = getTestDataDirectory(); file.append('test.png'); var attachment = yield Zotero.Attachments.linkFromDocument({ - document: win.content.document, + document: yield HiddenBrowser.getDocument(browser), parentItemID: item.id }); @@ -358,13 +356,9 @@ describe("Zotero.Attachments", function() { var uri = OS.Path.join(getTestDataDirectory().path, "snapshot"); httpd.registerDirectory("/" + prefix + "/", new FileUtils.File(uri)); - var deferred = Zotero.Promise.defer(); - win.addEventListener('pageshow', () => deferred.resolve()); - win.loadURI(testServerPath + "/index.html"); - await deferred.promise; - + browser = await HiddenBrowser.create(testServerPath + "/index.html"); var attachment = await Zotero.Attachments.importFromDocument({ - document: win.content.document, + browser, parentItemID: item.id }); @@ -408,13 +402,9 @@ describe("Zotero.Attachments", function() { } ); - var deferred = Zotero.Promise.defer(); - win.addEventListener('pageshow', () => deferred.resolve()); - win.loadURI(testServerPath + "/index.html"); - await deferred.promise; - + browser = await HiddenBrowser.create(testServerPath + "/index.html"); var attachment = await Zotero.Attachments.importFromDocument({ - document: win.content.document, + browser, parentItemID: item.id }); @@ -459,13 +449,9 @@ describe("Zotero.Attachments", function() { } ); - var deferred = Zotero.Promise.defer(); - win.addEventListener('pageshow', () => deferred.resolve()); - win.loadURI(testServerPath + "/index.html"); - await deferred.promise; - + browser = await HiddenBrowser.create(testServerPath + "/index.html"); var attachment = await Zotero.Attachments.importFromDocument({ - document: win.content.document, + browser, parentItemID: item.id }); @@ -509,13 +495,9 @@ describe("Zotero.Attachments", function() { } ); - let deferred = Zotero.Promise.defer(); - win.addEventListener('pageshow', () => deferred.resolve()); - win.loadURI(testServerPath + "/index.html"); - await deferred.promise; - + browser = await HiddenBrowser.create(testServerPath + "/index.html"); let attachment = await Zotero.Attachments.importFromDocument({ - document: win.content.document, + browser, parentItemID: item.id }); diff --git a/test/tests/httpTest.js b/test/tests/httpTest.js index a57ae1e472..4feebfc772 100644 --- a/test/tests/httpTest.js +++ b/test/tests/httpTest.js @@ -349,50 +349,7 @@ describe("Zotero.HTTP", function () { assert.isTrue(called); }); }); - - describe("#loadDocuments()", function () { - var win; - - before(function* () { - // TEMP: createHiddenBrowser currently needs a parent window - win = yield loadBrowserWindow(); - }); - - after(function* () { - win.close(); - }); - - it("should provide a document object", function* () { - var called = false; - yield new Zotero.Promise((resolve) => { - Zotero.HTTP.loadDocuments( - testURL, - function (doc) { - assert.equal(doc.location.href, testURL); - assert.equal(doc.querySelector('p').textContent, 'Test'); - var p = doc.evaluate('//p', doc, null, XPathResult.ANY_TYPE, null).iterateNext(); - assert.equal(p.textContent, 'Test'); - called = true; - }, - resolve - ); - }); - assert.isTrue(called); - }); - - it("should fail on non-2xx response", async function () { - var e = await getPromiseError(new Zotero.Promise((resolve, reject) => { - Zotero.HTTP.loadDocuments( - baseURL + "nonexistent", - () => {}, - resolve, - reject - ); - })); - assert.instanceOf(e, Zotero.HTTP.UnexpectedStatusException); - }); - }); - + describe("CasePreservingHeaders", function () { describe("#constructor()", function () { it("should initialize from an iterable or object", function () { diff --git a/test/tests/translateTest.js b/test/tests/translateTest.js index 2ea466ffb0..3c22d52b47 100644 --- a/test/tests/translateTest.js +++ b/test/tests/translateTest.js @@ -2,6 +2,8 @@ new function() { Components.utils.import("resource://gre/modules/osfile.jsm"); Components.utils.import("resource://zotero-unit/httpd.js"); +const { HiddenBrowser } = ChromeUtils.import('chrome://zotero/content/HiddenBrowser.jsm'); + /** * Create a new translator that saves the specified items * @param {String} translatorType - "import" or "web" @@ -18,10 +20,10 @@ function saveItemsThroughTranslator(translatorType, items, translateOptions = {} } let translate = new Zotero.Translate[tyname](); - let browser; if (translatorType == "web") { - browser = Zotero.Browser.createHiddenBrowser(); - translate.setDocument(browser.contentDocument); + let doc = new DOMParser().parseFromString('', 'text/html'); + doc = Zotero.HTTP.wrapDocument(doc, 'https://www.example.com/'); + translate.setDocument(doc); } else if (translatorType == "import") { translate.setString(""); } @@ -37,7 +39,6 @@ function saveItemsThroughTranslator(translatorType, items, translateOptions = {} " }\n"+ "}")); return translate.translate(translateOptions).then(function(items) { - if (browser) Zotero.Browser.deleteHiddenBrowser(browser); return items; }); } @@ -688,15 +689,8 @@ describe("Zotero.Translate", function() { }); it('web translators should save attachment from browser document', function* () { - let deferred = Zotero.Promise.defer(); - let browser = Zotero.HTTP.loadDocuments( - "http://127.0.0.1:23119/test/translate/test.html", - doc => deferred.resolve(doc), - undefined, - undefined, - true - ); - let doc = yield deferred.promise; + let browser = yield HiddenBrowser.create("http://127.0.0.1:23119/test/translate/test.html"); + let doc = yield HiddenBrowser.getDocument(browser); let translate = new Zotero.Translate.Web(); translate.setDocument(doc); @@ -725,7 +719,7 @@ describe("Zotero.Translate", function() { assert.equal(snapshot.attachmentContentType, "text/html"); checkTestTags(snapshot, true); - Zotero.Browser.deleteHiddenBrowser(browser); + HiddenBrowser.destroy(browser); }); it('web translators should save attachment from non-browser document', function* () { From 893ad2bf625475a181d162bbb4327b9dd432bdf9 Mon Sep 17 00:00:00 2001 From: Abe Jellinek Date: Fri, 14 Apr 2023 11:43:11 -0400 Subject: [PATCH 5/5] fx-compat: Fix PDF and style interception --- chrome/content/zotero/xpcom/attachments.js | 42 ++-- .../content/zotero/xpcom/mimeTypeHandler.js | 214 +++++++++--------- chrome/content/zotero/zoteroPane.js | 2 +- 3 files changed, 128 insertions(+), 130 deletions(-) diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index c1dece9f12..47adfd1fd7 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -1157,8 +1157,7 @@ Zotero.Attachments = new function(){ Zotero.debug(`downloadPDFViaBrowser: Sniffing a PDF loaded at ${name}`); // try the browser try { - channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsIWebNavigation) - .QueryInterface(Ci.nsIDocShell).chromeEventHandler; + channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement; } catch (e) {} if (channelBrowser) { @@ -1167,8 +1166,8 @@ Zotero.Attachments = new function(){ else { // try the document for the load group try { - channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsIWebNavigation) - .QueryInterface(Ci.nsIDocShell).chromeEventHandler; + channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext) + .topFrameElement; } catch(e) {} if (channelBrowser) { @@ -1191,21 +1190,18 @@ Zotero.Attachments = new function(){ }; try { Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true); - function noop() {}; - hiddenBrowser = Zotero.HTTP.loadDocuments([url], noop, noop, noop, true, options.cookieSandbox); + hiddenBrowser = await HiddenBrowser.create(url, { + requireSuccessfulStatus: true, + cookieSandbox: options.cookieSandbox, + }); let onLoadTimeoutDeferred = Zotero.Promise.defer(); let currentUrl = ""; - hiddenBrowser.addProgressListener({ - QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIWebProgressListener, - Components.interfaces.nsISupportsWeakReference]), - onProgressChange: noop, - onStateChange: noop, - onStatusChange: noop, - onSecurityChange: noop, + hiddenBrowser.webProgress.addProgressListener({ + QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]), async onLocationChange() { - let url = hiddenBrowser.contentDocument.location.href; + let url = hiddenBrowser.currentURI.spec; if (currentUrl) { - Zotero.debug(`downloadPDFViaBrowser: A JS redirect occurred to ${hiddenBrowser.contentDocument.location.href}`); + Zotero.debug(`downloadPDFViaBrowser: A JS redirect occurred to ${url}`); } currentUrl = url; Zotero.debug(`downloadPDFViaBrowser: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`); @@ -1215,7 +1211,7 @@ Zotero.Attachments = new function(){ onLoadTimeoutDeferred.reject(new Error(`downloadPDFViaBrowser: Loading PDF via browser timed out on the JS challenge page after ${onLoadTimeout}ms`)); } } - }); + }, Ci.nsIWebProgress.NOTIFY_LOCATION); await Zotero.Promise.race([ onLoadTimeoutDeferred.promise, Zotero.Promise.delay(downloadTimeout).then(() => { @@ -1238,7 +1234,7 @@ Zotero.Attachments = new function(){ finally { Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler); if (hiddenBrowser) { - Zotero.Browser.deleteHiddenBrowser(hiddenBrowser); + HiddenBrowser.destroy(hiddenBrowser); } } }; @@ -2978,7 +2974,7 @@ Zotero.Attachments = new function(){ * Determines if a given document is an instance of PDFJS * @return {Boolean} */ - this.isPDFJS = function(doc) { + this.isPDFJSDocument = function(doc) { // pdf.js HACK // This may no longer be necessary (as of Fx 23) if(doc.contentType === "text/html") { @@ -2992,6 +2988,16 @@ Zotero.Attachments = new function(){ } return false; } + + + /** + * Determines if a given Browser is displaying an instance of PDFJS + * @return {Boolean} + */ + this.isPDFJSBrowser = function (browser) { + // https://searchfox.org/mozilla-esr102/rev/f78d456e055a41106be086c501b271385a973961/browser/base/content/browser.js#5518 + return browser.contentPrincipal?.spec == "resource://pdf.js/web/viewer.html"; + }; this.linkModeToName = function (linkMode) { diff --git a/chrome/content/zotero/xpcom/mimeTypeHandler.js b/chrome/content/zotero/xpcom/mimeTypeHandler.js index f02a7a0712..ce750ec124 100644 --- a/chrome/content/zotero/xpcom/mimeTypeHandler.js +++ b/chrome/content/zotero/xpcom/mimeTypeHandler.js @@ -23,18 +23,35 @@ ***** END LICENSE BLOCK ***** */ +const ArrayBufferInputStream = Components.Constructor( + "@mozilla.org/io/arraybuffer-input-stream;1", + "nsIArrayBufferInputStream" +); +const BinaryInputStream = Components.Constructor( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream" +); +const StorageStream = Components.Constructor( + "@mozilla.org/storagestream;1", + "nsIStorageStream", + "init" +); +const BufferedOutputStream = Components.Constructor( + "@mozilla.org/network/buffered-output-stream;1", + "nsIBufferedOutputStream", + "init" +); + Zotero.MIMETypeHandler = new function () { var _typeHandlers, _ignoreContentDispositionTypes, _observers; - + /** - * Registers URIContentListener to handle MIME types + * Registers nsIObserver to handle MIME types */ this.init = function() { - Zotero.debug("Registering URIContentListener"); - // register our nsIURIContentListener and nsIObserver - Components.classes["@mozilla.org/uriloader;1"]. - getService(Components.interfaces.nsIURILoader). - registerContentListener(_URIContentListener); + Zotero.debug("Registering nsIObserver"); + // register our nsIObserver Components.classes["@mozilla.org/observer-service;1"]. getService(Components.interfaces.nsIObserverService). addObserver(_Observer, "http-on-examine-response", false); @@ -61,16 +78,16 @@ Zotero.MIMETypeHandler = new function () { var data = await Zotero.Utilities.Internal.blobToText(blob); try { await Zotero.Styles.install(data, origin, true); + // Close styles page in basic viewer after installing a style + win?.close(); + return true; } catch (e) { Zotero.logError(e); (new Zotero.Exception.Alert("styles.install.unexpectedError", origin, "styles.install.title", e)).present(); } - // Close styles page in basic viewer after installing a style - if (win) { - win.close(); - } + return false; } }, true); }; @@ -134,81 +151,64 @@ Zotero.MIMETypeHandler = new function () { /** * Called to observe a page load */ - var _Observer = new function() { - this.observe = function(channel) { - if(Zotero.isConnector) return; - + var _Observer = { + observe(channel) { channel.QueryInterface(Components.interfaces.nsIRequest); - if(channel.loadFlags & Components.interfaces.nsIHttpChannel.LOAD_DOCUMENT_URI) { - channel.QueryInterface(Components.interfaces.nsIHttpChannel); - try { - // remove content-disposition headers for EndNote, etc. - var contentType = channel.getResponseHeader("Content-Type").toLowerCase(); - for (let handledType of _ignoreContentDispositionTypes) { - if (contentType.startsWith(handledType)) { - channel.setResponseHeader("Content-Disposition", "inline", false); - break; - } + // https://searchfox.org/mozilla-esr102/rev/f78d456e055a41106be086c501b271385a973961/netwerk/base/nsIChannel.idl#209-211 + if (!channel.isDocument) { + return; + } + channel.QueryInterface(Components.interfaces.nsIHttpChannel); + channel.QueryInterface(Components.interfaces.nsITraceableChannel); + + try { + // remove content-disposition headers for EndNote, etc. + var contentType = channel.getResponseHeader("Content-Type").toLowerCase(); + for (let handledType of _ignoreContentDispositionTypes) { + if (contentType.startsWith(handledType)) { + channel.setResponseHeader("Content-Disposition", "inline", false); + break; } - } catch(e) {} - - for (let observer of _observers) { + } + } + catch (e) { + // getResponseHeader() throws if header is not set; ignore + } + + try { + if (contentType && _typeHandlers[contentType]) { + // Replace listener entirely + // #setNewListener() contract wants us to pass through events to the original (eventually), + // but we will not + let originalListener = channel.setNewListener(new _StreamListener(channel, contentType)); + // Make it look like the connection ended, so the original listener can clean up + originalListener.onStopRequest(channel, Cr.NS_BINDING_ABORTED); + } + } + catch (e) { + Zotero.logError(e); + } + + for (let observer of _observers) { + try { observer(channel); } + catch (e) { + Zotero.logError(e); + } } } - } - - var _URIContentListener = new function() { - /** - * Standard QI definition - */ - this.QueryInterface = function(iid) { - if (iid.equals(Components.interfaces.nsISupports) - || iid.equals(Components.interfaces.nsISupportsWeakReference) - || iid.equals(Components.interfaces.nsIURIContentListener)) { - return this; - } - throw Components.results.NS_ERROR_NO_INTERFACE; - } - - /** - * Called to see if we can handle a content type - */ - this.canHandleContent = this.isPreferred = function(contentType, isContentPreferred, desiredContentType) { - if(Zotero.isConnector) return false; - return !!_typeHandlers[contentType.toLowerCase()]; - } - - /** - * Called to begin handling a content type - */ - this.doContent = function(contentType, isContentPreferred, request, contentHandler) { - Zotero.debug("MIMETypeHandler: handling "+contentType+" from " + request.name); - contentHandler.value = new _StreamListener(request, contentType.toLowerCase()); - return false; - } - - /** - * Called so that we could stop a load before it happened if we wanted to - */ - this.onStartURIOpen = function(URI) { - return true; - } - } + }; /** - * @class Implements nsIStreamListener and nsIRequestObserver interfaces to download MIME types + * @class _StreamListener Implements nsIStreamListener and nsIRequestObserver interfaces to download MIME types * we've registered ourself as the handler for * @param {nsIRequest} request The request to handle * @param {String} contenType The content type being handled */ - var _StreamListener = function(request, contentType) { + var _StreamListener = function (request, contentType) { this._request = request; - this._contentType = contentType - this._storageStream = null; - this._outputStream = null; - this._binaryInputStream = null; + this._contentType = contentType; } /** @@ -223,10 +223,11 @@ Zotero.MIMETypeHandler = new function () { throw Components.results.NS_ERROR_NO_INTERFACE; } - /** - * Called when the request is started - */ _StreamListener.prototype.onStartRequest = async function(channel) { + this._onStartRequestCalled = true; + this._dataBuffer = new StorageStream(4096, 0xffffffff); + this._stream = new BufferedOutputStream(this._dataBuffer.getOutputStream(0), 8192); + try { if (!_typeHandlers[this._contentType]) return; for (let handlers of _typeHandlers[this._contentType]) { @@ -245,44 +246,36 @@ Zotero.MIMETypeHandler = new function () { Zotero.logError(e); } } - - /** - * Called when there's data available; we collect this data and keep it until the request is - * done - */ - _StreamListener.prototype.onDataAvailable = function(request, inputStream, offset, count) { - Zotero.debug(count + " bytes available"); - - if (!this._storageStream) { - this._storageStream = Components.classes["@mozilla.org/storagestream;1"]. - createInstance(Components.interfaces.nsIStorageStream); - this._storageStream.init(16384, 4294967295, null); // PR_UINT32_MAX - this._outputStream = this._storageStream.getOutputStream(0); - - this._binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]. - createInstance(Components.interfaces.nsIBinaryInputStream); - this._binaryInputStream.setInputStream(inputStream); + + _StreamListener.prototype.onDataAvailable = async function (channel, inputStream, offset, count) { + if (!this._onStartRequestCalled) { + await this.onStartRequest(channel); } - var bytes = this._binaryInputStream.readBytes(count); - this._outputStream.write(bytes, count); - } + this._stream.writeFrom(inputStream, count); + }; /** * Called when the request is done */ - _StreamListener.prototype.onStopRequest = async function (channel, status) { + _StreamListener.prototype.onStopRequest = async function (channel, statusCode) { + if (!this._onStartRequestCalled) { + await this.onStartRequest(channel); + } + Zotero.debug("charset is " + channel.contentCharset); - var inputStream = this._storageStream.newInputStream(0); - var stream = Components.classes["@mozilla.org/binaryinputstream;1"] - .createInstance(Components.interfaces.nsIBinaryInputStream); - stream.setInputStream(inputStream); - let buffer = new ArrayBuffer(this._storageStream.length); + this._stream.close(); + this._stream = null; + + if (!Components.isSuccessCode(statusCode)) { + throw Components.Exception('Failed to load', statusCode); + } + + let stream = new BinaryInputStream(this._dataBuffer.newInputStream(0)); + let buffer = new ArrayBuffer(this._dataBuffer.length); stream.readArrayBuffer(buffer.byteLength, buffer); - stream.close(); - inputStream.close(); - let blob = new (Zotero.getMainWindow()).Blob([buffer], { type: this._contentType }); + let blob = new Blob([buffer], { type: this._contentType }); var handled = false; try { @@ -306,26 +299,25 @@ Zotero.MIMETypeHandler = new function () { Zotero.logError(e); } - if (handled === false) { + if (!handled) { // Handle using nsIExternalHelperAppService let externalHelperAppService = Components.classes["@mozilla.org/uriloader/external-helper-app-service;1"] .getService(Components.interfaces.nsIExternalHelperAppService); let frontWindow = Components.classes["@mozilla.org/embedcomp/window-watcher;1"] .getService(Components.interfaces.nsIWindowWatcher).activeWindow; - let inputStream = this._storageStream.newInputStream(0); + let inputStream = new ArrayBufferInputStream(); + inputStream.setData(buffer, 0, buffer.byteLength); let streamListener = externalHelperAppService.doContent( this._contentType, this._request, frontWindow, null ); if (streamListener) { streamListener.onStartRequest(channel); streamListener.onDataAvailable( - this._request, inputStream, 0, this._storageStream.length + this._request, inputStream, 0, buffer.byteLength ); - streamListener.onStopRequest(channel, status); + streamListener.onStopRequest(channel, statusCode); } } - - this._storageStream.close(); }; } diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js index 8a86bd5f14..fe94a90c00 100644 --- a/chrome/content/zotero/zoteroPane.js +++ b/chrome/content/zotero/zoteroPane.js @@ -4147,7 +4147,7 @@ var ZoteroPane = new function() if (itemType == 'temporaryPDFHack') { itemType = null; var isPDF = false; - if (doc.title.indexOf('application/pdf') != -1 || Zotero.Attachments.isPDFJS(doc) + if (doc.title.indexOf('application/pdf') != -1 || Zotero.Attachments.isPDFJSDocument(doc) || doc.contentType == 'application/pdf') { isPDF = true; }