mirror of
https://github.com/zotero/zotero.git
synced 2026-10-09 03:18:01 +00:00
Add support for clearing challenge in browser during translation
This commit is contained in:
Abe Jellinek
parent
b8611db419
commit
5ebe8ea15f
14 changed files with 1051 additions and 367 deletions
|
|
@ -50,6 +50,8 @@ export class HiddenBrowser {
|
|||
* Must be set to false if intending to call print().
|
||||
* @param {Number} [options.cookieContextId] - userContextId from Zotero.HTTP.newCookieContext()
|
||||
* for cookie isolation
|
||||
* @param {String} [options.customUserAgent] - Override User-Agent for all requests
|
||||
* from this browser's browsing context
|
||||
*/
|
||||
constructor(options = {}) {
|
||||
this._destroyed = false;
|
||||
|
|
@ -96,6 +98,10 @@ export class HiddenBrowser {
|
|||
this._blockingObserver.register(browser);
|
||||
}
|
||||
|
||||
if (options.customUserAgent) {
|
||||
browser.browsingContext.customUserAgent = options.customUserAgent;
|
||||
}
|
||||
|
||||
this._browser = browser;
|
||||
})();
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ export class PageDataChild extends JSWindowActorChild {
|
|||
case "documentHTML":
|
||||
return new XMLSerializer().serializeToString(document);
|
||||
|
||||
case "querySelectorMatches":
|
||||
return !!document.querySelector(message.data.selector);
|
||||
|
||||
case "channelInfo": {
|
||||
let docShell = this.contentWindow.docShell;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -32,31 +32,31 @@ const SANDBOXED_SCRIPTS = 0x80;
|
|||
var browser;
|
||||
|
||||
window.addEventListener("load", /*async */function () {
|
||||
let { uri, options } = window.arguments[0].wrappedJSObject;
|
||||
|
||||
browser = document.querySelector('browser');
|
||||
if (options?.cookieContextId) {
|
||||
// Set usercontextid on new <browser> so it takes effect
|
||||
let newBrowser = document.createXULElement('browser');
|
||||
for (let { name, value } of browser.attributes) {
|
||||
newBrowser.setAttribute(name, value);
|
||||
}
|
||||
newBrowser.setAttribute('usercontextid', String(options.cookieContextId));
|
||||
browser.replaceWith(newBrowser);
|
||||
browser = newBrowser;
|
||||
}
|
||||
window.gBrowser = browser; // For ZoomManager
|
||||
|
||||
browser.addEventListener('pagetitlechanged', () => {
|
||||
document.title = browser.contentTitle || browser.currentURI.spec;
|
||||
});
|
||||
|
||||
/*
|
||||
browser.setAttribute("remote", "true");
|
||||
//browser.setAttribute("remoteType", E10SUtils.EXTENSION_REMOTE_TYPE);
|
||||
|
||||
await new Promise((resolve) => {
|
||||
browser.addEventListener("XULFrameLoaderCreated", () => resolve());
|
||||
});
|
||||
*/
|
||||
|
||||
/*browser.messageManager.loadFrameScript(
|
||||
'chrome://zotero/content/standalone/basicViewerContent.js',
|
||||
false
|
||||
);*/
|
||||
//browser.docShellIsActive = false;
|
||||
|
||||
// Get URI and options passed in via openWindow()
|
||||
let { uri, options } = window.arguments[0].wrappedJSObject;
|
||||
if (options?.customUserAgent) {
|
||||
browser.browsingContext.customUserAgent = options.customUserAgent;
|
||||
}
|
||||
|
||||
window.viewerOriginalURI = uri;
|
||||
window.viewerCookieContextId = options?.cookieContextId;
|
||||
loadURI(Services.io.newURI(uri), options);
|
||||
}, false);
|
||||
|
||||
|
|
|
|||
|
|
@ -1200,8 +1200,8 @@ Zotero.Attachments = new function () {
|
|||
&& (e instanceof this.InvalidPDFException
|
||||
// Thrown by HTTP.download()
|
||||
|| (e instanceof Zotero.HTTP.UnexpectedStatusException && e.status == 403))) {
|
||||
if (Zotero.BrowserDownload.shouldAttemptDownloadViaBrowser(url)) {
|
||||
return Zotero.BrowserDownload.downloadPDF(url, path, options);
|
||||
if (Zotero.BrowserRequest.getEntryForURL(url)) {
|
||||
return Zotero.BrowserRequest.downloadPDF(url, path, options);
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
|
|
|
|||
|
|
@ -1,243 +0,0 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2023 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
http://zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
const { HiddenBrowser } = ChromeUtils.importESModule("chrome://zotero/content/HiddenBrowser.mjs");
|
||||
|
||||
Zotero.BrowserDownload = {
|
||||
HANDLED_URLS: {
|
||||
'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html': "html",
|
||||
'://www.sciencedirect.com': "#captcha-box"
|
||||
},
|
||||
|
||||
/**
|
||||
* Checks whether the url can be handled as a hidden browser download
|
||||
* @param {String} url
|
||||
*/
|
||||
shouldAttemptDownloadViaBrowser: function (url) {
|
||||
const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url));
|
||||
for (let unproxiedUrl of unproxiedUrls) {
|
||||
for (let checkUrl in this.HANDLED_URLS) {
|
||||
if (unproxiedUrl.includes(checkUrl)) {
|
||||
return checkUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
},
|
||||
|
||||
getCaptchaLocator(url) {
|
||||
const handlerKey = this.shouldAttemptDownloadViaBrowser(url);
|
||||
return this.HANDLED_URLS[handlerKey];
|
||||
},
|
||||
|
||||
_makePDFMIMETypeHandler(browser, onPDFFound = () => 0) {
|
||||
let isOurPDF, channelBrowser;
|
||||
let trackedBrowser = browser;
|
||||
return {
|
||||
onStartRequest: function (name, _, channel) {
|
||||
Zotero.debug(`BrowserDownload: Sniffing a PDF loaded at ${name}`);
|
||||
// try the browser
|
||||
try {
|
||||
channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement;
|
||||
}
|
||||
catch (e) {}
|
||||
if (channelBrowser) {
|
||||
isOurPDF = trackedBrowser === channelBrowser;
|
||||
}
|
||||
else {
|
||||
// try the document for the load group
|
||||
try {
|
||||
channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext)
|
||||
.topFrameElement;
|
||||
}
|
||||
catch (e) {}
|
||||
if (channelBrowser) {
|
||||
isOurPDF = trackedBrowser === channelBrowser;
|
||||
}
|
||||
}
|
||||
},
|
||||
onContent: async (blob, name) => {
|
||||
if (isOurPDF) {
|
||||
Zotero.debug(`BrowserDownload: Found our PDF at ${name}`);
|
||||
onPDFFound(blob);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
Zotero.debug(`BrowserDownload: Not our PDF at ${name}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* @param {String} url
|
||||
* @param {String} path
|
||||
* @param {Object} [options]
|
||||
* @param {Boolean} [options.shouldDisplayCaptcha=false]
|
||||
*/
|
||||
async downloadPDF(url, path, options = {}) {
|
||||
Zotero.debug(`BrowserDownload: Downloading file via a hidden browser from ${url}`);
|
||||
|
||||
let hiddenBrowser;
|
||||
let pdfMIMETypeHandler;
|
||||
let pdfFoundDeferred = Zotero.Promise.defer();
|
||||
|
||||
// Technically this is not a download, but the full operation (load, redirect, etc) timeout
|
||||
const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout');
|
||||
const onLoadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.onLoadTimeout');
|
||||
|
||||
try {
|
||||
hiddenBrowser = new HiddenBrowser();
|
||||
await hiddenBrowser._createdPromise;
|
||||
|
||||
let pdfLoaded = false;
|
||||
pdfMIMETypeHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, pdfFoundDeferred.resolve);
|
||||
Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true);
|
||||
|
||||
let onLoadTimeoutDeferred = Zotero.Promise.defer();
|
||||
let currentUrl = "";
|
||||
hiddenBrowser.webProgress.addProgressListener({
|
||||
QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]),
|
||||
async onLocationChange() {
|
||||
let url = hiddenBrowser.currentURI.spec;
|
||||
if (currentUrl) {
|
||||
Zotero.debug(`BrowserDownload: A JS redirect occurred to ${url}`);
|
||||
}
|
||||
currentUrl = url;
|
||||
Zotero.debug(`BrowserDownload: Page with potential JS redirect loaded, giving it ${onLoadTimeout}ms to process`);
|
||||
await Zotero.Promise.delay(onLoadTimeout);
|
||||
// If URL changed that means we got redirected and the onLoadTimeout needs to restart
|
||||
if (currentUrl === url && !pdfLoaded) {
|
||||
onLoadTimeoutDeferred.reject(new Error(`BrowserDownload: Loading PDF via a hidden browser timed out on the JS challenge page after ${onLoadTimeout}ms`));
|
||||
}
|
||||
}
|
||||
}, Ci.nsIWebProgress.NOTIFY_LOCATION);
|
||||
|
||||
hiddenBrowser.load(url);
|
||||
let blob = await Promise.race([
|
||||
onLoadTimeoutDeferred.promise,
|
||||
Zotero.Promise.delay(downloadTimeout).then(() => {
|
||||
if (!pdfLoaded) {
|
||||
throw new Error(`BrowserDownload: Loading PDF via a hidden browser timed out after ${downloadTimeout}ms`);
|
||||
}
|
||||
}),
|
||||
// Resolves PDF blob
|
||||
pdfFoundDeferred.promise
|
||||
]);
|
||||
|
||||
pdfLoaded = true;
|
||||
await Zotero.File.putContentsAsync(path, blob);
|
||||
}
|
||||
catch (e) {
|
||||
try {
|
||||
await OS.File.remove(path, { ignoreAbsent: true });
|
||||
}
|
||||
catch (err) {
|
||||
Zotero.logError(err);
|
||||
}
|
||||
if (options?.shouldDisplayCaptcha) {
|
||||
Zotero.debug(`BrowserDownload: Downloading via a hidden browser failed due to ${e.message}`);
|
||||
const captchaLocator = this.getCaptchaLocator(url);
|
||||
if (captchaLocator) {
|
||||
let doc = await hiddenBrowser.getDocument();
|
||||
let elem = doc.querySelector(captchaLocator);
|
||||
if (elem) {
|
||||
return this.downloadPDFViaViewer(url, path, options);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
finally {
|
||||
Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler);
|
||||
if (hiddenBrowser) {
|
||||
hiddenBrowser.destroy();
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async downloadPDFViaViewer(url, path, options) {
|
||||
Zotero.debug(`BrowserDownload: Downloading file via the document viewer for captcha clearing from ${url}`);
|
||||
|
||||
let win, browser, xulWin, wmListener;
|
||||
let pdfMIMETypeHandler;
|
||||
let pdfFound;
|
||||
let pdfFoundDeferred = Zotero.Promise.defer();
|
||||
const downloadTimeout = Zotero.Prefs.get('downloadPDFViaBrowser.downloadTimeout');
|
||||
|
||||
try {
|
||||
wmListener = {
|
||||
onOpenWindow(xulWindow) {
|
||||
xulWin = xulWin || xulWindow;
|
||||
},
|
||||
onCloseWindow(xulWindow) {
|
||||
if (xulWin === xulWindow && !pdfFound) {
|
||||
pdfFoundDeferred.reject(new Error("BrowserDownload: User closed the document viewer"));
|
||||
}
|
||||
}
|
||||
};
|
||||
Services.wm.addListener(wmListener);
|
||||
await new Promise((resolve) => {
|
||||
win = Zotero.openInViewer(url);
|
||||
win.addEventListener('load', resolve);
|
||||
});
|
||||
browser = win.document.querySelector('browser');
|
||||
Zotero.Utilities.Internal.activate(win);
|
||||
|
||||
pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve);
|
||||
Zotero.MIMETypeHandler.addHandlers("application/pdf", pdfMIMETypeHandler, true);
|
||||
|
||||
Zotero.debug(`BrowserDownload: Awaiting the user to clear the captcha or timeout after ${downloadTimeout}`);
|
||||
let pdfBlob = await Promise.race([
|
||||
Zotero.Promise.delay(downloadTimeout).then(() => {
|
||||
if (!pdfFound) {
|
||||
throw new Error(`BrowserDownload: Loading PDF via document viewer timed out after ${downloadTimeout}ms`);
|
||||
}
|
||||
}),
|
||||
// Resolves PDF blob
|
||||
pdfFoundDeferred.promise
|
||||
]);
|
||||
pdfFound = true;
|
||||
await Zotero.File.putContentsAsync(path, pdfBlob);
|
||||
}
|
||||
catch (e) {
|
||||
try {
|
||||
await OS.File.remove(path, { ignoreAbsent: true });
|
||||
}
|
||||
catch (err) {
|
||||
Zotero.logError(err);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
finally {
|
||||
Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler);
|
||||
Services.wm.removeListener(wmListener);
|
||||
if (win) {
|
||||
win.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
647
chrome/content/zotero/xpcom/browserRequest.js
Normal file
647
chrome/content/zotero/xpcom/browserRequest.js
Normal file
|
|
@ -0,0 +1,647 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2026 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
http://zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
const { HiddenBrowser } = ChromeUtils.importESModule("chrome://zotero/content/HiddenBrowser.mjs");
|
||||
|
||||
Zotero.BrowserRequest = {
|
||||
// Registry of URL patterns that need browser-mediated handling
|
||||
CHALLENGE_URLS: [
|
||||
{
|
||||
match: 'https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html',
|
||||
captchaLocator: 'html'
|
||||
},
|
||||
{
|
||||
match: '://www.sciencedirect.com',
|
||||
captchaLocator: '#captcha-box'
|
||||
},
|
||||
{
|
||||
match: '://search.worldcat.org',
|
||||
// When /api/search returns 403 with turnstile_required, the user-
|
||||
// facing /search?q= page runs the matching invisible Turnstile
|
||||
// widget, POSTs the token to /api/turnstile-verify, and gets back
|
||||
// Set-Cookie: turnstile_passed. Loading that page in a hidden
|
||||
// browser reproduces the flow end-to-end with no user interaction.
|
||||
getChallengeURL: url => url.replace(/\/api\/search\b/, '/search'),
|
||||
// Invisible (managed) Turnstile; no user captcha interaction needed
|
||||
captchaLocator: null,
|
||||
// Wait for WorldCat's own Turnstile cookie to land
|
||||
successCookie: { host: 'search.worldcat.org', name: 'turnstile_passed' },
|
||||
// turnstile_passed is signed against (IP, UA), so we must use the
|
||||
// same plain-Firefox UA for the translator's follow-up requests as
|
||||
// the hidden browser used when earning it.
|
||||
plainUAHost: 'search.worldcat.org',
|
||||
detectBlock: (status, body) => status === 403 && /turnstile_required/.test(body)
|
||||
}
|
||||
],
|
||||
|
||||
/**
|
||||
* Look up a challenge entry for a URL, applying proxy unwrapping.
|
||||
* @param {string} url
|
||||
* @returns {object|null}
|
||||
*/
|
||||
getEntryForURL(url) {
|
||||
const unproxiedUrls = Object.keys(Zotero.Proxies.getPotentialProxies(url));
|
||||
for (let unproxiedUrl of unproxiedUrls) {
|
||||
for (let entry of this.CHALLENGE_URLS) {
|
||||
if (unproxiedUrl.includes(entry.match)) {
|
||||
return entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Navigate to a URL in a hidden browser, running its JS long enough for any
|
||||
* client-side redirects or cookie-setting challenges to settle.
|
||||
*
|
||||
* Cookies acquired by the browser remain in the shared jar keyed on
|
||||
* cookieContextId; a subsequent Zotero.HTTP.request using the same ID will
|
||||
* see them.
|
||||
*
|
||||
* On timeout, if the page contains the entry's captchaLocator and
|
||||
* allowViewer is set, escalates to clearChallengeInViewer().
|
||||
*
|
||||
* @param {string} url
|
||||
* @param {object} [options]
|
||||
* @param {number} [options.cookieContextId]
|
||||
* @param {object} [options.entry] - registry entry controlling escalation
|
||||
* @param {boolean} [options.allowViewer=false]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async clearChallenge(url, options = {}) {
|
||||
Zotero.debug(`BrowserRequest: Clearing challenge at ${url}`);
|
||||
|
||||
let { cookieContextId, entry, allowViewer = false } = options;
|
||||
let successCookie = entry?.successCookie;
|
||||
// Capture the cookie's current value (if any) before the attempt so
|
||||
// we can tell a freshly-issued cookie from a stale one left over from
|
||||
// a previous session.
|
||||
let initialCookieValue = successCookie
|
||||
? this._readCookieValue({ ...successCookie, cookieContextId })
|
||||
: null;
|
||||
// Cloudflare Turnstile rejects the "Zotero/[version]" suffix.
|
||||
// A plain Firefox UA on just this browsing context lets the widget run.
|
||||
let customUserAgent = Zotero.VersionHeader.getPlainFirefoxUA();
|
||||
|
||||
// Try the hidden browser first. _loadAndSettle() polls the cookie jar
|
||||
// and resolves as soon as successCookie appears, which may be well
|
||||
// before the page fully settles (or redirects somewhere else).
|
||||
let hiddenBrowser;
|
||||
try {
|
||||
hiddenBrowser = new HiddenBrowser({ cookieContextId, customUserAgent });
|
||||
await hiddenBrowser._createdPromise;
|
||||
await this._loadAndSettle(hiddenBrowser, url, {
|
||||
successCookie,
|
||||
cookieContextId
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.debug('BrowserRequest: Hidden browser attempt failed');
|
||||
Zotero.logError(e);
|
||||
}
|
||||
finally {
|
||||
if (hiddenBrowser) {
|
||||
hiddenBrowser.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (successCookie) {
|
||||
let currentValue = this._readCookieValue({ ...successCookie, cookieContextId });
|
||||
if (currentValue && currentValue !== initialCookieValue) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!allowViewer) {
|
||||
throw new Error(`BrowserRequest: Challenge not cleared at ${url} and viewer escalation is disabled`);
|
||||
}
|
||||
|
||||
// Fall back to the viewer: user may need to click a visible Turnstile
|
||||
// widget, after which the cookie lands and we can continue.
|
||||
Zotero.debug(`BrowserRequest: Escalating to viewer for ${url}`);
|
||||
if (successCookie) {
|
||||
await this._loadAndWaitForCookieInViewer(url, {
|
||||
cookieContextId,
|
||||
customUserAgent,
|
||||
successCookie
|
||||
});
|
||||
return;
|
||||
}
|
||||
await this.clearChallengeInViewer(url, {
|
||||
cookieContextId,
|
||||
captchaLocator: entry.captchaLocator
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Open a visible viewer at the URL and wait until successCookie appears
|
||||
* in the jar. Necessary for sites where the success signal is a specific
|
||||
* cookie being set (e.g., WorldCat's turnstile_passed) rather than a
|
||||
* navigation or change in the DOM.
|
||||
*/
|
||||
async _loadAndWaitForCookieInViewer(url, options) {
|
||||
Zotero.debug(`BrowserRequest: Awaiting user challenge clearance (cookie ${options.successCookie.name}) at ${url}`);
|
||||
const timeout = Zotero.Prefs.get('browserRequest.timeout');
|
||||
const { successCookie, cookieContextId, customUserAgent } = options;
|
||||
|
||||
let win, wmListener, pollInterval;
|
||||
let done = false;
|
||||
let cookieDeferred = Zotero.Promise.defer();
|
||||
let closedDeferred = Zotero.Promise.defer();
|
||||
|
||||
try {
|
||||
wmListener = this._makeViewerCloseListener(() => {
|
||||
if (!done) closedDeferred.reject(new Error('BrowserRequest: User closed the viewer'));
|
||||
});
|
||||
Services.wm.addListener(wmListener);
|
||||
await new Promise((resolve) => {
|
||||
win = Zotero.openInViewer(url, { cookieContextId, customUserAgent });
|
||||
win.addEventListener('load', resolve);
|
||||
});
|
||||
Zotero.Utilities.Internal.activate(win);
|
||||
|
||||
pollInterval = this._pollForCookie({
|
||||
successCookie,
|
||||
cookieContextId,
|
||||
onFound: () => {
|
||||
done = true;
|
||||
cookieDeferred.resolve();
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.race([
|
||||
cookieDeferred.promise,
|
||||
closedDeferred.promise,
|
||||
Zotero.Promise.delay(timeout).then(() => {
|
||||
if (!done) {
|
||||
throw new Error(`BrowserRequest: Viewer cookie wait timed out after ${timeout}ms`);
|
||||
}
|
||||
})
|
||||
]);
|
||||
}
|
||||
finally {
|
||||
if (pollInterval) clearInterval(pollInterval);
|
||||
Services.wm.removeListener(wmListener);
|
||||
if (win) win.close();
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Read the value of a named cookie on a given host under an optional
|
||||
* userContextId. Returns null if the cookie is absent.
|
||||
*/
|
||||
_readCookieValue({ host, name, cookieContextId }) {
|
||||
try {
|
||||
let cookies = Services.cookies.getCookiesFromHost(
|
||||
host,
|
||||
cookieContextId ? { userContextId: cookieContextId } : {}
|
||||
);
|
||||
for (let cookie of cookies) {
|
||||
if (cookie.name === name) {
|
||||
return cookie.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.debug('BrowserRequest: _readCookieValue() failed');
|
||||
Zotero.logError(e);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Build a Services.wm listener that tracks the first window opened after
|
||||
* addListener() and invokes onClose when that window closes. Used to
|
||||
* detect user-closed viewer windows.
|
||||
*
|
||||
* @param {Function} onClose
|
||||
*/
|
||||
_makeViewerCloseListener(onClose) {
|
||||
let xulWin;
|
||||
return {
|
||||
onOpenWindow(xulWindow) {
|
||||
xulWin ||= xulWindow;
|
||||
},
|
||||
onCloseWindow(xulWindow) {
|
||||
if (xulWin === xulWindow) {
|
||||
onClose();
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Invoke onFound as soon as successCookie appears with a value different
|
||||
* from the one observed at poll start. Captures the initial value to
|
||||
* avoid declaring success on a stale cookie left over from a previous
|
||||
* session (the server-signed HMAC would no longer validate).
|
||||
* Caller is responsible for clearing the returned interval handle.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {{host: string, name: string}} opts.successCookie
|
||||
* @param {number} [opts.cookieContextId]
|
||||
* @param {Function} opts.onFound
|
||||
* @param {number} [opts.intervalMs=250]
|
||||
* @returns {number} interval handle
|
||||
*/
|
||||
_pollForCookie({ successCookie, cookieContextId, onFound, intervalMs = 250 }) {
|
||||
let { host, name } = successCookie;
|
||||
let initialValue = this._readCookieValue({ host, name, cookieContextId });
|
||||
return setInterval(() => {
|
||||
let currentValue = this._readCookieValue({ host, name, cookieContextId });
|
||||
if (currentValue && currentValue !== initialValue) {
|
||||
onFound();
|
||||
}
|
||||
}, intervalMs);
|
||||
},
|
||||
|
||||
/**
|
||||
* Open a visible browser window at the URL and wait for the user to clear a
|
||||
* challenge. Resolves once the captchaLocator element disappears and the
|
||||
* page has been stable for `browserRequest.onLoadTimeout` ms.
|
||||
*
|
||||
* @param {string} url
|
||||
* @param {object} options
|
||||
* @param {number} [options.cookieContextId]
|
||||
* @param {string} options.captchaLocator
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async clearChallengeInViewer(url, options) {
|
||||
Zotero.debug(`BrowserRequest: Awaiting user challenge clearance for ${url}`);
|
||||
const onLoadTimeout = Zotero.Prefs.get('browserRequest.onLoadTimeout');
|
||||
const timeout = Zotero.Prefs.get('browserRequest.timeout');
|
||||
|
||||
let win, browser, wmListener;
|
||||
let cleared = false;
|
||||
let cancelled = false;
|
||||
let clearedDeferred = Zotero.Promise.defer();
|
||||
|
||||
try {
|
||||
wmListener = this._makeViewerCloseListener(() => {
|
||||
if (!cleared) clearedDeferred.reject(new Error('BrowserRequest: User closed the viewer'));
|
||||
});
|
||||
Services.wm.addListener(wmListener);
|
||||
await new Promise((resolve) => {
|
||||
win = Zotero.openInViewer(url, {
|
||||
cookieContextId: options.cookieContextId
|
||||
});
|
||||
win.addEventListener('load', resolve);
|
||||
});
|
||||
browser = win.document.querySelector('browser');
|
||||
Zotero.Utilities.Internal.activate(win);
|
||||
|
||||
// Poll for the captcha element disappearing, then require the page
|
||||
// to stay stable for onLoadTimeout before we consider it cleared
|
||||
let lastLocation = browser.currentURI.spec;
|
||||
let stableSince = null;
|
||||
let pollInterval = 500;
|
||||
// Don't allow clearance until we've positively observed the challenge
|
||||
// at least once; otherwise a transient empty/about:blank document
|
||||
// during the challenge page's own loading would declare success
|
||||
// before the user sees anything.
|
||||
let sawChallenge = false;
|
||||
await Promise.race([
|
||||
clearedDeferred.promise,
|
||||
Zotero.Promise.delay(timeout).then(() => {
|
||||
if (!cleared) {
|
||||
cancelled = true;
|
||||
throw new Error(`BrowserRequest: Viewer challenge clearance timed out after ${timeout}ms`);
|
||||
}
|
||||
}),
|
||||
(async () => {
|
||||
// Set above:
|
||||
// eslint-disable-next-line no-unmodified-loop-condition
|
||||
while (!cleared && !cancelled) {
|
||||
await Zotero.Promise.delay(pollInterval);
|
||||
let currentLocation = browser.currentURI.spec;
|
||||
if (currentLocation !== lastLocation) {
|
||||
lastLocation = currentLocation;
|
||||
stableSince = null;
|
||||
continue;
|
||||
}
|
||||
let stillChallenged = await this._browserMatchesSelector(browser, options.captchaLocator);
|
||||
if (stillChallenged) {
|
||||
sawChallenge = true;
|
||||
stableSince = null;
|
||||
continue;
|
||||
}
|
||||
if (!sawChallenge) {
|
||||
// Challenge page hasn't rendered yet, or we can't read
|
||||
// the DOM. Keep waiting without advancing the timer.
|
||||
stableSince = null;
|
||||
continue;
|
||||
}
|
||||
if (stableSince === null) {
|
||||
stableSince = Date.now();
|
||||
}
|
||||
else if (Date.now() - stableSince >= onLoadTimeout) {
|
||||
cleared = true;
|
||||
clearedDeferred.resolve();
|
||||
}
|
||||
}
|
||||
})()
|
||||
]);
|
||||
}
|
||||
finally {
|
||||
Services.wm.removeListener(wmListener);
|
||||
if (win) {
|
||||
win.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Ask the browser's current document whether it has a match for a CSS selector.
|
||||
* False if the document can't be queried or nothing matches.
|
||||
*/
|
||||
async _browserMatchesSelector(browser, selector) {
|
||||
try {
|
||||
let actor = browser.browsingContext?.currentWindowGlobal?.getActor('PageData');
|
||||
if (!actor) return false;
|
||||
return await actor.sendQuery('querySelectorMatches', { selector });
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.logError(e);
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Load the URL in the given HiddenBrowser and wait for the page to settle.
|
||||
* Each location change restarts a `browserRequest.onLoadTimeout` window;
|
||||
* if the location stays stable for that window, we consider the page
|
||||
* settled. Throws if `browserRequest.timeout` elapses first.
|
||||
*
|
||||
* If `onPDF` is provided, also sets up a PDF MIME type handler on the
|
||||
* browser and resolves early once a PDF is captured; the callback receives
|
||||
* the blob.
|
||||
*
|
||||
* If `successCookie` is provided, polls the cookie jar and resolves as
|
||||
* soon as a cookie with that name exists on the given host.
|
||||
*
|
||||
* @param {HiddenBrowser} hiddenBrowser
|
||||
* @param {string} url
|
||||
* @param {object} [opts]
|
||||
* @param {(blob: Blob) => void} [opts.onPDF]
|
||||
* @param {{ host: string, name: string }} [opts.successCookie]
|
||||
* @param {number} [opts.cookieContextId]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async _loadAndSettle(hiddenBrowser, url, opts = {}) {
|
||||
const onLoadTimeout = Zotero.Prefs.get('browserRequest.onLoadTimeout');
|
||||
const timeout = Zotero.Prefs.get('browserRequest.timeout');
|
||||
|
||||
let settled = false;
|
||||
let settleDeferred = Zotero.Promise.defer();
|
||||
let pdfDeferred = Zotero.Promise.defer();
|
||||
let cookieDeferred = Zotero.Promise.defer();
|
||||
let pdfFound = false;
|
||||
let pdfHandler;
|
||||
|
||||
if (opts.onPDF) {
|
||||
pdfHandler = this._makePDFMIMETypeHandler(hiddenBrowser._browser, (blob) => {
|
||||
pdfFound = true;
|
||||
opts.onPDF(blob);
|
||||
pdfDeferred.resolve();
|
||||
});
|
||||
Zotero.MIMETypeHandler.addHandlers('application/pdf', pdfHandler, true);
|
||||
}
|
||||
|
||||
try {
|
||||
let currentUrl = '';
|
||||
hiddenBrowser.webProgress.addProgressListener({
|
||||
QueryInterface: ChromeUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]),
|
||||
async onLocationChange() {
|
||||
let loc = hiddenBrowser.currentURI.spec;
|
||||
if (currentUrl) {
|
||||
Zotero.debug(`BrowserRequest: A JS redirect occurred to ${loc}`);
|
||||
}
|
||||
currentUrl = loc;
|
||||
Zotero.debug(`BrowserRequest: Page loaded at ${loc}; waiting ${onLoadTimeout}ms for further JS activity`);
|
||||
await Zotero.Promise.delay(onLoadTimeout);
|
||||
if (currentUrl === loc && !settled && !pdfFound) {
|
||||
settled = true;
|
||||
settleDeferred.resolve();
|
||||
}
|
||||
}
|
||||
}, Ci.nsIWebProgress.NOTIFY_LOCATION);
|
||||
|
||||
hiddenBrowser.load(url);
|
||||
|
||||
let cookiePollInterval;
|
||||
if (opts.successCookie) {
|
||||
cookiePollInterval = this._pollForCookie({
|
||||
successCookie: opts.successCookie,
|
||||
cookieContextId: opts.cookieContextId,
|
||||
onFound: () => {
|
||||
Zotero.debug(`BrowserRequest: successCookie ${opts.successCookie.name} appeared`);
|
||||
cookieDeferred.resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let races = [
|
||||
settleDeferred.promise,
|
||||
Zotero.Promise.delay(timeout).then(() => {
|
||||
if (!settled && !pdfFound) {
|
||||
throw new Error(`BrowserRequest: Browser request timed out after ${timeout}ms`);
|
||||
}
|
||||
})
|
||||
];
|
||||
if (opts.onPDF) {
|
||||
races.push(pdfDeferred.promise);
|
||||
}
|
||||
if (opts.successCookie) {
|
||||
races.push(cookieDeferred.promise);
|
||||
}
|
||||
try {
|
||||
await Promise.race(races);
|
||||
}
|
||||
finally {
|
||||
if (cookiePollInterval) clearInterval(cookiePollInterval);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if (pdfHandler) {
|
||||
Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfHandler);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
_makePDFMIMETypeHandler(browser, onPDFFound = () => 0) {
|
||||
let isOurPDF, channelBrowser;
|
||||
let trackedBrowser = browser;
|
||||
return {
|
||||
onStartRequest: function (name, _, channel) {
|
||||
Zotero.debug(`BrowserRequest: Sniffing a PDF loaded at ${name}`);
|
||||
try {
|
||||
channelBrowser = channel.notificationCallbacks.getInterface(Ci.nsILoadContext).topFrameElement;
|
||||
}
|
||||
catch {}
|
||||
if (channelBrowser) {
|
||||
isOurPDF = trackedBrowser === channelBrowser;
|
||||
}
|
||||
else {
|
||||
try {
|
||||
channelBrowser = channel.loadGroup.notificationCallbacks.getInterface(Ci.nsILoadContext)
|
||||
.topFrameElement;
|
||||
}
|
||||
catch {}
|
||||
if (channelBrowser) {
|
||||
isOurPDF = trackedBrowser === channelBrowser;
|
||||
}
|
||||
}
|
||||
},
|
||||
onContent: async (blob, name) => {
|
||||
if (isOurPDF) {
|
||||
Zotero.debug(`BrowserRequest: Found our PDF at ${name}`);
|
||||
onPDFFound(blob);
|
||||
return true;
|
||||
}
|
||||
Zotero.debug(`BrowserRequest: Not our PDF at ${name}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* @param {String} url
|
||||
* @param {String} path
|
||||
* @param {Object} [options]
|
||||
* @param {Boolean} [options.shouldDisplayCaptcha=false]
|
||||
*/
|
||||
async downloadPDF(url, path, options = {}) {
|
||||
Zotero.debug(`BrowserRequest: Downloading PDF via hidden browser from ${url}`);
|
||||
|
||||
let hiddenBrowser;
|
||||
let blob;
|
||||
try {
|
||||
hiddenBrowser = new HiddenBrowser();
|
||||
await hiddenBrowser._createdPromise;
|
||||
await this._loadAndSettle(hiddenBrowser, url, {
|
||||
onPDF: (foundBlob) => {
|
||||
blob = foundBlob;
|
||||
}
|
||||
});
|
||||
if (!blob) {
|
||||
throw new Error('BrowserRequest: Settled without receiving a PDF');
|
||||
}
|
||||
await Zotero.File.putContentsAsync(path, blob);
|
||||
}
|
||||
catch (e) {
|
||||
try {
|
||||
await OS.File.remove(path, { ignoreAbsent: true });
|
||||
}
|
||||
catch (err) {
|
||||
Zotero.logError(err);
|
||||
}
|
||||
if (options?.shouldDisplayCaptcha) {
|
||||
Zotero.debug(`BrowserRequest: Hidden browser PDF download failed: ${e.message}`);
|
||||
const entry = this.getEntryForURL(url);
|
||||
if (entry?.captchaLocator && hiddenBrowser) {
|
||||
let doc;
|
||||
try {
|
||||
doc = await hiddenBrowser.getDocument();
|
||||
}
|
||||
catch {}
|
||||
if (doc && doc.querySelector(entry.captchaLocator)) {
|
||||
return this.downloadPDFViaViewer(url, path, options);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
finally {
|
||||
if (hiddenBrowser) {
|
||||
hiddenBrowser.destroy();
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
|
||||
async downloadPDFViaViewer(url, path, _options) {
|
||||
Zotero.debug(`BrowserRequest: Downloading PDF via viewer for captcha clearing from ${url}`);
|
||||
|
||||
let win, browser, wmListener;
|
||||
let pdfMIMETypeHandler;
|
||||
let pdfFound;
|
||||
let pdfFoundDeferred = Zotero.Promise.defer();
|
||||
const timeout = Zotero.Prefs.get('browserRequest.timeout');
|
||||
|
||||
try {
|
||||
wmListener = this._makeViewerCloseListener(() => {
|
||||
if (!pdfFound) pdfFoundDeferred.reject(new Error('BrowserRequest: User closed the viewer'));
|
||||
});
|
||||
Services.wm.addListener(wmListener);
|
||||
await new Promise((resolve) => {
|
||||
win = Zotero.openInViewer(url);
|
||||
win.addEventListener('load', resolve);
|
||||
});
|
||||
browser = win.document.querySelector('browser');
|
||||
Zotero.Utilities.Internal.activate(win);
|
||||
|
||||
pdfMIMETypeHandler = this._makePDFMIMETypeHandler(browser, pdfFoundDeferred.resolve);
|
||||
Zotero.MIMETypeHandler.addHandlers('application/pdf', pdfMIMETypeHandler, true);
|
||||
|
||||
Zotero.debug(`BrowserRequest: Awaiting user captcha clearance or timeout after ${timeout}ms`);
|
||||
let pdfBlob = await Promise.race([
|
||||
Zotero.Promise.delay(timeout).then(() => {
|
||||
if (!pdfFound) {
|
||||
throw new Error(`BrowserRequest: Viewer PDF download timed out after ${timeout}ms`);
|
||||
}
|
||||
}),
|
||||
pdfFoundDeferred.promise
|
||||
]);
|
||||
pdfFound = true;
|
||||
await Zotero.File.putContentsAsync(path, pdfBlob);
|
||||
}
|
||||
catch (e) {
|
||||
try {
|
||||
await OS.File.remove(path, { ignoreAbsent: true });
|
||||
}
|
||||
catch (err) {
|
||||
Zotero.logError(err);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
finally {
|
||||
Zotero.MIMETypeHandler.removeHandlers('application/pdf', pdfMIMETypeHandler);
|
||||
Services.wm.removeListener(wmListener);
|
||||
if (win) {
|
||||
win.close();
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Register hosts that we intercept Cloudflare Turnstile challenges on,
|
||||
// so they receive a plain UA everywhere. See comment on
|
||||
// Zotero.VersionHeader.registerPlainUAHost().
|
||||
for (let entry of Zotero.BrowserRequest.CHALLENGE_URLS) {
|
||||
if (entry.plainUAHost) {
|
||||
Zotero.VersionHeader.registerPlainUAHost(entry.plainUAHost);
|
||||
}
|
||||
}
|
||||
|
|
@ -298,8 +298,8 @@ Zotero.HTTP = new function () {
|
|||
Zotero.debug("HTTP " + method + " " + dispURL);
|
||||
}
|
||||
|
||||
// Translation framework uses cookieSandbox
|
||||
if (options.cookieSandbox && typeof options.cookieSandbox === 'string') {
|
||||
// Translation framework uses cookieSandbox to hold userContextId number
|
||||
if (typeof options.cookieSandbox === 'number') {
|
||||
options.cookieContextId = options.cookieSandbox;
|
||||
delete options.cookieSandbox;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ Zotero.Prefs = new function () {
|
|||
|
||||
// Process pref version updates
|
||||
var fromVersion = this.get('prefVersion');
|
||||
var toVersion = 21;
|
||||
var toVersion = 22;
|
||||
if (!fromVersion) {
|
||||
this.set('prefVersion', toVersion);
|
||||
}
|
||||
|
|
@ -220,6 +220,21 @@ Zotero.Prefs = new function () {
|
|||
case 21:
|
||||
this.set('firstRunGuidanceShown.readAloud', false);
|
||||
break;
|
||||
|
||||
// downloadPDFViaBrowser.* -> browserRequest.*;
|
||||
// module was generalized to handle more than PDF downloads
|
||||
case 22:
|
||||
if (this.prefHasUserValue('downloadPDFViaBrowser.onLoadTimeout')) {
|
||||
this.set('browserRequest.onLoadTimeout',
|
||||
this.get('downloadPDFViaBrowser.onLoadTimeout'));
|
||||
this.clear('downloadPDFViaBrowser.onLoadTimeout');
|
||||
}
|
||||
if (this.prefHasUserValue('downloadPDFViaBrowser.downloadTimeout')) {
|
||||
this.set('browserRequest.timeout',
|
||||
this.get('downloadPDFViaBrowser.downloadTimeout'));
|
||||
this.clear('downloadPDFViaBrowser.downloadTimeout');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
this.set('prefVersion', toVersion);
|
||||
|
|
|
|||
|
|
@ -1238,6 +1238,10 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte
|
|||
* @param {Object} [options]
|
||||
* @param {Function} [options.onLoad] - Function to run once URI is loaded; passed the loaded document
|
||||
* @param {Boolean} [options.allowJavaScript] - Set to false to disable JavaScript
|
||||
* @param {Number} [options.cookieContextId] - userContextId to isolate the viewer's cookies
|
||||
* into the same jar as a Zotero.HTTP.request or HiddenBrowser using the same ID
|
||||
* @param {String} [options.customUserAgent] - Override the User-Agent for all requests
|
||||
* from this viewer's browsing context
|
||||
*/
|
||||
this.openInViewer = function (uri, options) {
|
||||
if (options && !options.onLoad && typeof options === 'function') {
|
||||
|
|
@ -1247,7 +1251,7 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte
|
|||
|
||||
var viewerWins = Services.wm.getEnumerator("zotero:basicViewer");
|
||||
for (let existingWin of viewerWins) {
|
||||
if (existingWin.viewerOriginalURI === uri) {
|
||||
if (existingWin.viewerOriginalURI === uri && existingWin.viewerCookieContextId === options?.cookieContextId) {
|
||||
existingWin.focus();
|
||||
return existingWin;
|
||||
}
|
||||
|
|
@ -2101,6 +2105,10 @@ Zotero.Keys = new function () {
|
|||
* @namespace
|
||||
*/
|
||||
Zotero.VersionHeader = {
|
||||
_plainUAHosts: new Set(),
|
||||
_uaAppSuffixRe: null,
|
||||
_uaFirefoxComponent: null,
|
||||
|
||||
init: function () {
|
||||
this.register();
|
||||
Zotero.addShutdownListener(this.unregister);
|
||||
|
|
@ -2126,7 +2134,9 @@ Zotero.VersionHeader = {
|
|||
let isAppNameDomain = s3RE.test(domain);
|
||||
if (!isAppNameDomain) {
|
||||
let ua = channel.getRequestHeader('User-Agent');
|
||||
ua = this.update(ua);
|
||||
ua = this.update(ua, {
|
||||
mode: this._plainUAHosts.has(domain) ? 'plain' : 'full',
|
||||
});
|
||||
channel.setRequestHeader('User-Agent', ua, false);
|
||||
}
|
||||
}
|
||||
|
|
@ -2137,23 +2147,57 @@ Zotero.VersionHeader = {
|
|||
},
|
||||
|
||||
/**
|
||||
* Add Firefox/[version] to the default user agent
|
||||
* Register a host that needs the "Zotero/[version]" component stripped
|
||||
* from its requests' UA. Currently this is only used for hosts that we
|
||||
* handle Cloudflare Turnstile challenges on; Turnstile won't pass with
|
||||
* Zotero/ in the UA string, and future requests need the same UA as the
|
||||
* one that passed Turnstile, so we have to override for all requests to
|
||||
* the host.
|
||||
*
|
||||
* @param {String} ua - User Agent
|
||||
* @param {string} host
|
||||
*/
|
||||
update: function (ua) {
|
||||
registerPlainUAHost: function (host) {
|
||||
this._plainUAHosts.add(host);
|
||||
},
|
||||
|
||||
/**
|
||||
* @param {String} ua
|
||||
* @param {'full' | 'plain'} [mode='full'] If 'full', add Firefox/[version] to the default user agent. If 'plain', remove
|
||||
* Zotero/[version] instead.
|
||||
* @return {String}
|
||||
*/
|
||||
update: function (ua, { mode = 'full' } = {}) {
|
||||
var info = Services.appinfo;
|
||||
var appName = info.name;
|
||||
|
||||
var pos = ua.indexOf(appName + '/');
|
||||
|
||||
// Default UA (not a faked UA from the connector)
|
||||
if (pos != -1) {
|
||||
ua = ua.substring(0, pos) + `Firefox/${info.platformVersion.match(/^\d+/)[0]}.0 ` + ua.substring(pos);
|
||||
if (!this._uaAppSuffixRe) {
|
||||
this._uaAppSuffixRe = new RegExp(`\\s*${info.name}/\\S+`);
|
||||
this._uaFirefoxComponent = `Firefox/${info.platformVersion.match(/^\d+/)[0]}.0`;
|
||||
}
|
||||
if (mode === 'plain') {
|
||||
ua = ua.replace(this._uaAppSuffixRe, '');
|
||||
if (!/\bFirefox\//.test(ua)) {
|
||||
ua += ` ${this._uaFirefoxComponent}`;
|
||||
}
|
||||
}
|
||||
else {
|
||||
let pos = ua.indexOf(info.name + '/');
|
||||
// Default UA (not a faked UA from the connector)
|
||||
if (pos != -1) {
|
||||
ua = ua.substring(0, pos) + `${this._uaFirefoxComponent} ` + ua.substring(pos);
|
||||
}
|
||||
}
|
||||
|
||||
return ua;
|
||||
},
|
||||
|
||||
/**
|
||||
* Plain Firefox UA, without the "Zotero/[version]" suffix.
|
||||
*
|
||||
* @return {String}
|
||||
*/
|
||||
getPlainFirefoxUA: function () {
|
||||
var ua = Cc["@mozilla.org/network/protocol;1?name=http"]
|
||||
.getService(Ci.nsIHttpProtocolHandler).userAgent;
|
||||
return this.update(ua, { mode: 'plain' });
|
||||
},
|
||||
|
||||
unregister: function () {
|
||||
Services.obs.removeObserver(Zotero.VersionHeader, "http-on-modify-request");
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ const xpcomFilesLocal = [
|
|||
'api',
|
||||
'attachments',
|
||||
'attachmentReadObserver',
|
||||
'browserDownload',
|
||||
'browserRequest',
|
||||
'cite',
|
||||
'citeprocRsBridge',
|
||||
'data/library',
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ pref("extensions.zotero.itemPaneHeader.bibEntry.locale", "");
|
|||
pref("extensions.zotero.tagSelector.showAutomatic", true);
|
||||
pref("extensions.zotero.tagSelector.displayAllTags", false);
|
||||
|
||||
pref("extensions.zotero.downloadPDFViaBrowser.onLoadTimeout", 3000);
|
||||
pref("extensions.zotero.downloadPDFViaBrowser.downloadTimeout", 60000);
|
||||
pref("extensions.zotero.browserRequest.onLoadTimeout", 3000);
|
||||
pref("extensions.zotero.browserRequest.timeout", 60000);
|
||||
|
||||
// Keyboard shortcuts
|
||||
pref("extensions.zotero.keys.saveToZotero", "S");
|
||||
|
|
|
|||
|
|
@ -326,8 +326,8 @@ describe("Zotero.Attachments", function () {
|
|||
|
||||
|
||||
describe("#importFromURL()", function () {
|
||||
it("should use BrowserDownload for a JS redirect page", async function () {
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDF");
|
||||
it("should use BrowserRequest for a JS redirect page", async function () {
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDF");
|
||||
downloadPDFStub.callsFake(async (_url, path) => {
|
||||
await OS.File.copy(OS.Path.join(getTestDataDirectory().path, 'test.pdf'), path);
|
||||
});
|
||||
|
|
@ -623,7 +623,7 @@ describe("Zotero.Attachments", function () {
|
|||
});
|
||||
});
|
||||
|
||||
it("should use BrowserDownload for 403 when enforcing file type", async function () {
|
||||
it("should use BrowserRequest for 403 when enforcing file type", async function () {
|
||||
let prefix = Zotero.Utilities.randomString();
|
||||
let testServerPath = 'http://127.0.0.1:' + testServerPort + '/' + prefix;
|
||||
let pdfURL = testServerPath + '/test.pdf';
|
||||
|
|
@ -638,9 +638,9 @@ describe("Zotero.Attachments", function () {
|
|||
);
|
||||
|
||||
let path = OS.Path.join(Zotero.getTempDirectory().path, 'test.pdf');
|
||||
let shouldAttemptStub = sinon.stub(Zotero.BrowserDownload, "shouldAttemptDownloadViaBrowser");
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDF");
|
||||
shouldAttemptStub.returns(true);
|
||||
let getEntryStub = sinon.stub(Zotero.BrowserRequest, "getEntryForURL");
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDF");
|
||||
getEntryStub.returns({ match: 'test' });
|
||||
downloadPDFStub.callsFake(async (_url, path) => {
|
||||
await OS.File.copy(OS.Path.join(getTestDataDirectory().path, 'test.pdf'), path);
|
||||
});
|
||||
|
|
@ -648,14 +648,14 @@ describe("Zotero.Attachments", function () {
|
|||
try {
|
||||
item = await Zotero.Attachments.downloadFile(pdfURL, path, { enforceFileType: true });
|
||||
|
||||
assert.isTrue(shouldAttemptStub.calledOnce);
|
||||
assert.isTrue(getEntryStub.calledOnce);
|
||||
assert.isTrue(downloadPDFStub.calledOnce);
|
||||
}
|
||||
finally {
|
||||
// Clean up
|
||||
if (item) await Zotero.Items.erase(item.id);
|
||||
downloadPDFStub.restore();
|
||||
shouldAttemptStub.restore();
|
||||
getEntryStub.restore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2023 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
http://zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
describe("Zotero.BrowserDownload", function () {
|
||||
describe("#downloadPDF()", function () {
|
||||
var http, port, baseURL;
|
||||
var tmpFile = Zotero.getTempDirectory();
|
||||
tmpFile.append('browserDownloadTest.pdf');
|
||||
|
||||
before(async function () {
|
||||
({ httpd, port } = await startHTTPServer());
|
||||
baseURL = `http://localhost:${port}/`;
|
||||
});
|
||||
|
||||
after(async function () {
|
||||
await new Promise(resolve => httpd.stop(resolve));
|
||||
});
|
||||
|
||||
it("#downloadPDF() should download a PDF from a JS redirect page", async function () {
|
||||
var dir = getTestDataDirectory().path;
|
||||
httpd.registerFile(
|
||||
'/test-pdf-redirect.html',
|
||||
Zotero.File.pathToFile(PathUtils.join(dir, 'test-pdf-redirect.html'))
|
||||
);
|
||||
httpd.registerFile(
|
||||
'/test.pdf',
|
||||
Zotero.File.pathToFile(PathUtils.join(dir, 'test.pdf'))
|
||||
);
|
||||
|
||||
await Zotero.BrowserDownload.downloadPDF(`${baseURL}test-pdf-redirect.html`, tmpFile.path);
|
||||
|
||||
var sample = await Zotero.File.getContentsAsync(tmpFile, null, 1000);
|
||||
assert.equal(Zotero.MIME.sniffForMIMEType(sample), 'application/pdf');
|
||||
});
|
||||
|
||||
// Needs a js-redirect delay in test-pdf-redirect.html
|
||||
it.skip("should display a viewer to clear a captcha if detected", async function () {
|
||||
// Make it so that downloadPDF() times out with a hidden browser, which simulates running into a captcha
|
||||
Zotero.Prefs.set('downloadPDFViaBrowser.downloadTimeout', 10);
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserDownload, "downloadPDFViaViewer");
|
||||
|
||||
let promise = Zotero.BrowserDownload.downloadPDF('https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', tmpFile.path,
|
||||
{ shouldDisplayCaptcha: true });
|
||||
await new Promise(resolve => downloadPDFStub.callsFake((...args) => {
|
||||
resolve();
|
||||
Zotero.Prefs.set('downloadPDFViaBrowser.downloadTimeout', 60e3);
|
||||
return downloadPDFStub.wrappedMethod.call(Zotero.BrowserDownload, ...args);
|
||||
}));
|
||||
|
||||
await promise;
|
||||
|
||||
assert.isTrue(downloadPDFStub.calledOnce);
|
||||
downloadPDFStub.restore();
|
||||
});
|
||||
});
|
||||
});
|
||||
290
test/tests/browserRequestTest.js
Normal file
290
test/tests/browserRequestTest.js
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2026 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
http://zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
describe("Zotero.BrowserRequest", function () {
|
||||
describe("#downloadPDF()", function () {
|
||||
var httpd, port, baseURL;
|
||||
var tmpFile = Zotero.getTempDirectory();
|
||||
tmpFile.append('browserRequestTest.pdf');
|
||||
|
||||
before(async function () {
|
||||
({ httpd, port } = await startHTTPServer());
|
||||
baseURL = `http://localhost:${port}/`;
|
||||
});
|
||||
|
||||
after(async function () {
|
||||
await new Promise(resolve => httpd.stop(resolve));
|
||||
});
|
||||
|
||||
it("should download a PDF from a JS redirect page", async function () {
|
||||
var dir = getTestDataDirectory().path;
|
||||
httpd.registerFile(
|
||||
'/test-pdf-redirect.html',
|
||||
Zotero.File.pathToFile(PathUtils.join(dir, 'test-pdf-redirect.html'))
|
||||
);
|
||||
httpd.registerFile(
|
||||
'/test.pdf',
|
||||
Zotero.File.pathToFile(PathUtils.join(dir, 'test.pdf'))
|
||||
);
|
||||
|
||||
await Zotero.BrowserRequest.downloadPDF(`${baseURL}test-pdf-redirect.html`, tmpFile.path);
|
||||
|
||||
var sample = await Zotero.File.getContentsAsync(tmpFile, null, 1000);
|
||||
assert.equal(Zotero.MIME.sniffForMIMEType(sample), 'application/pdf');
|
||||
});
|
||||
|
||||
// Needs a js-redirect delay in test-pdf-redirect.html
|
||||
it.skip("should display a viewer to clear a captcha if detected", async function () {
|
||||
// Force downloadPDF() to time out with a hidden browser, which simulates running into a captcha
|
||||
Zotero.Prefs.set('browserRequest.timeout', 10);
|
||||
let downloadPDFStub = sinon.stub(Zotero.BrowserRequest, "downloadPDFViaViewer");
|
||||
|
||||
let promise = Zotero.BrowserRequest.downloadPDF('https://zotero-static.s3.amazonaws.com/test-pdf-redirect.html', tmpFile.path,
|
||||
{ shouldDisplayCaptcha: true });
|
||||
await new Promise(resolve => downloadPDFStub.callsFake((...args) => {
|
||||
resolve();
|
||||
Zotero.Prefs.set('browserRequest.timeout', 60e3);
|
||||
return downloadPDFStub.wrappedMethod.call(Zotero.BrowserRequest, ...args);
|
||||
}));
|
||||
|
||||
await promise;
|
||||
|
||||
assert.isTrue(downloadPDFStub.calledOnce);
|
||||
downloadPDFStub.restore();
|
||||
});
|
||||
});
|
||||
|
||||
describe("#getEntryForURL()", function () {
|
||||
it("returns null for unrecognized URLs", function () {
|
||||
assert.isNull(Zotero.BrowserRequest.getEntryForURL('https://example.com/foo'));
|
||||
});
|
||||
|
||||
it("matches registered substrings", function () {
|
||||
let entry = Zotero.BrowserRequest.getEntryForURL(
|
||||
'https://search.worldcat.org/api/search?q=bn%3A123'
|
||||
);
|
||||
assert.isNotNull(entry);
|
||||
assert.equal(entry.match, '://search.worldcat.org');
|
||||
});
|
||||
|
||||
it("sniffs response bodies via detectBlock", function () {
|
||||
let entry = Zotero.BrowserRequest.getEntryForURL(
|
||||
'https://search.worldcat.org/api/search?q=bn%3A123'
|
||||
);
|
||||
assert.isFunction(entry.detectBlock);
|
||||
assert.isTrue(entry.detectBlock(403, '{"turnstile_required":true}'));
|
||||
assert.isFalse(entry.detectBlock(403, 'some other 403 body'));
|
||||
assert.isFalse(entry.detectBlock(200, '{"turnstile_required":true}'));
|
||||
});
|
||||
});
|
||||
|
||||
describe("#_pollForCookie()", function () {
|
||||
it("ignores a stale cookie present at poll start", async function () {
|
||||
// Simulate a leftover cookie from a previous session whose HMAC
|
||||
// is no longer valid: _pollForCookie must wait for the value to
|
||||
// change, not declare success on first read.
|
||||
let readStub = sinon.stub(Zotero.BrowserRequest, "_readCookieValue");
|
||||
readStub.returns("stale-value");
|
||||
|
||||
let onFound = sinon.spy();
|
||||
let handle = Zotero.BrowserRequest._pollForCookie({
|
||||
successCookie: { host: 'example.com', name: 'c' },
|
||||
onFound,
|
||||
intervalMs: 10
|
||||
});
|
||||
|
||||
try {
|
||||
await Zotero.Promise.delay(40);
|
||||
assert.isFalse(onFound.called, "should not fire while value is unchanged");
|
||||
|
||||
readStub.returns("fresh-value");
|
||||
await Zotero.Promise.delay(40);
|
||||
assert.isTrue(onFound.called, "should fire once a new value appears");
|
||||
}
|
||||
finally {
|
||||
clearInterval(handle);
|
||||
readStub.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("fires when the cookie first appears from nothing", async function () {
|
||||
let readStub = sinon.stub(Zotero.BrowserRequest, "_readCookieValue");
|
||||
readStub.returns(null);
|
||||
|
||||
let onFound = sinon.spy();
|
||||
let handle = Zotero.BrowserRequest._pollForCookie({
|
||||
successCookie: { host: 'example.com', name: 'c' },
|
||||
onFound,
|
||||
intervalMs: 10
|
||||
});
|
||||
|
||||
try {
|
||||
await Zotero.Promise.delay(40);
|
||||
assert.isFalse(onFound.called);
|
||||
|
||||
readStub.returns("new-value");
|
||||
await Zotero.Promise.delay(40);
|
||||
assert.isTrue(onFound.called);
|
||||
}
|
||||
finally {
|
||||
clearInterval(handle);
|
||||
readStub.restore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("translator request retry", function () {
|
||||
function makeUtils() {
|
||||
let fakeTranslate = {
|
||||
resolveURL: url => url,
|
||||
requestHeaders: {},
|
||||
cookieSandbox: undefined
|
||||
};
|
||||
return new Zotero.Utilities.Translate(fakeTranslate);
|
||||
}
|
||||
|
||||
it("retries via clearChallenge when the initial request throws a registered 403", async function () {
|
||||
let url = 'https://search.worldcat.org/api/search?q=bn%3A978-0-585-03015-9';
|
||||
let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves();
|
||||
let call = 0;
|
||||
let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => {
|
||||
call++;
|
||||
if (call === 1) {
|
||||
let xhr = {
|
||||
status: 403,
|
||||
response: '{"turnstile_required":true}',
|
||||
responseText: '{"turnstile_required":true}',
|
||||
channel: null
|
||||
};
|
||||
throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden");
|
||||
}
|
||||
return {
|
||||
status: 200,
|
||||
response: { ok: true },
|
||||
responseText: '{"ok":true}',
|
||||
getAllResponseHeaders: () => ''
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
let utils = makeUtils();
|
||||
let result = await utils.request(url);
|
||||
assert.equal(result.status, 200);
|
||||
assert.deepEqual(result.body, { ok: true });
|
||||
assert.equal(requestStub.callCount, 2);
|
||||
assert.isTrue(clearStub.calledOnce);
|
||||
assert.equal(clearStub.firstCall.args[0], 'https://search.worldcat.org/search?q=bn%3A978-0-585-03015-9');
|
||||
}
|
||||
finally {
|
||||
requestStub.restore();
|
||||
clearStub.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("retries when the 403's XHR is in responseType=json mode (responseText throws)", async function () {
|
||||
let url = 'https://search.worldcat.org/api/search?q=bn%3A978-0-585-03015-9';
|
||||
let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves();
|
||||
let call = 0;
|
||||
let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => {
|
||||
call++;
|
||||
if (call === 1) {
|
||||
let xhr = {
|
||||
status: 403,
|
||||
response: { turnstile_required: true },
|
||||
get responseText() { throw new Error('responseText unavailable'); },
|
||||
channel: null
|
||||
};
|
||||
throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden");
|
||||
}
|
||||
return {
|
||||
status: 200,
|
||||
response: { ok: true },
|
||||
responseText: '{"ok":true}',
|
||||
getAllResponseHeaders: () => ''
|
||||
};
|
||||
});
|
||||
|
||||
try {
|
||||
let utils = makeUtils();
|
||||
let result = await utils.request(url, { responseType: 'json' });
|
||||
assert.equal(result.status, 200);
|
||||
assert.deepEqual(result.body, { ok: true });
|
||||
assert.equal(requestStub.callCount, 2);
|
||||
assert.isTrue(clearStub.calledOnce);
|
||||
}
|
||||
finally {
|
||||
requestStub.restore();
|
||||
clearStub.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry when the URL is not registered", async function () {
|
||||
let url = 'https://example.com/api/search';
|
||||
let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves();
|
||||
let requestStub = sinon.stub(Zotero.HTTP, "request").callsFake(() => {
|
||||
let xhr = { status: 403, response: '', responseText: '', channel: null };
|
||||
throw new Zotero.HTTP.UnexpectedStatusException(xhr, url, "Forbidden");
|
||||
});
|
||||
|
||||
try {
|
||||
let utils = makeUtils();
|
||||
let err;
|
||||
try { await utils.request(url); }
|
||||
catch (e) { err = e; }
|
||||
assert.instanceOf(err, Zotero.HTTP.UnexpectedStatusException);
|
||||
assert.equal(requestStub.callCount, 1);
|
||||
assert.isTrue(clearStub.notCalled);
|
||||
}
|
||||
finally {
|
||||
requestStub.restore();
|
||||
clearStub.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry in browser on a 200 body that matches detectBlock (status must also match)", async function () {
|
||||
let url = 'https://search.worldcat.org/api/search?q=bn%3A1';
|
||||
let clearStub = sinon.stub(Zotero.BrowserRequest, "clearChallenge").resolves();
|
||||
let requestStub = sinon.stub(Zotero.HTTP, "request").returns({
|
||||
status: 200,
|
||||
response: '{"turnstile_required":true}',
|
||||
responseText: '{"turnstile_required":true}',
|
||||
getAllResponseHeaders: () => ''
|
||||
});
|
||||
|
||||
try {
|
||||
let utils = makeUtils();
|
||||
// WorldCat's detectBlock requires status === 403; a 200 with the same
|
||||
// body should not trigger a retry.
|
||||
await utils.request(url, { responseType: 'text' });
|
||||
assert.equal(requestStub.callCount, 1);
|
||||
assert.isTrue(clearStub.notCalled);
|
||||
}
|
||||
finally {
|
||||
requestStub.restore();
|
||||
clearStub.restore();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue