From ead93b6ccc4b336f98e530f2db014a84ff397491 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sat, 22 Jun 2019 05:28:02 -0400 Subject: [PATCH] Stop uploading files on quota error until next manual sync or restart --- .../zotero/xpcom/storage/storageEngine.js | 6 + .../zotero/xpcom/storage/storageLocal.js | 19 +++ chrome/content/zotero/xpcom/storage/zfs.js | 122 ++++++++++++------ chrome/content/zotero/xpcom/zotero.js | 1 + test/tests/zfsTest.js | 19 ++- 5 files changed, 123 insertions(+), 44 deletions(-) diff --git a/chrome/content/zotero/xpcom/storage/storageEngine.js b/chrome/content/zotero/xpcom/storage/storageEngine.js index f987ba3388..59f2627d01 100644 --- a/chrome/content/zotero/xpcom/storage/storageEngine.js +++ b/chrome/content/zotero/xpcom/storage/storageEngine.js @@ -133,6 +133,12 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* var filesEditable = Zotero.Libraries.get(libraryID).filesEditable; this.requestsRemaining = 0; + // Clear over-quota flag on manual sync + if (!this.background && Zotero.Sync.Storage.Local.storageRemainingForLibrary.has(libraryID)) { + Zotero.debug("Clearing over-quota flag for " + this.library.name); + Zotero.Sync.Storage.Local.storageRemainingForLibrary.delete(libraryID) + } + // Check for updated files to upload if (!filesEditable) { Zotero.debug("No file editing access -- skipping file modification check for " diff --git a/chrome/content/zotero/xpcom/storage/storageLocal.js b/chrome/content/zotero/xpcom/storage/storageLocal.js index 06389bcd82..55a8f6749e 100644 --- a/chrome/content/zotero/xpcom/storage/storageLocal.js +++ b/chrome/content/zotero/xpcom/storage/storageLocal.js @@ -11,6 +11,25 @@ Zotero.Sync.Storage.Local = { lastFullFileCheck: {}, uploadCheckFiles: [], + storageRemainingForLibrary: new Map(), + + init: function () { + Zotero.Notifier.registerObserver(this, ['group'], 'storageLocal'); + }, + + notify: async function (action, type, ids, _extraData) { + // Clean up cache on group deletion + if (action == 'delete' && type == 'group') { + for (let libraryID of ids) { + if (this.lastFullFileCheck[libraryID]) { + delete this.lastFullFileCheck[libraryID]; + } + if (this.storageRemainingForLibrary.has(libraryID)) { + this.storageRemainingForLibrary.delete(libraryID); + } + } + } + }, getEnabledForLibrary: function (libraryID) { var libraryType = Zotero.Libraries.get(libraryID).libraryType; diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js index c8f93605df..480ac979be 100644 --- a/chrome/content/zotero/xpcom/storage/zfs.js +++ b/chrome/content/zotero/xpcom/storage/zfs.js @@ -257,7 +257,38 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = { uploadFile: Zotero.Promise.coroutine(function* (request) { var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); - if (yield Zotero.Attachments.hasMultipleFiles(item)) { + var multipleFiles = yield Zotero.Attachments.hasMultipleFiles(item); + + // If we got a quota error for this library, skip upload for all multi-file attachments + // and for single-file attachments that are bigger than the remaining space. This is cleared + // in storageEngine for manual syncs. + var remaining = Zotero.Sync.Storage.Local.storageRemainingForLibrary.get(item.libraryID); + if (remaining !== undefined) { + let skip = false; + if (multipleFiles) { + Zotero.debug("Skipping multi-file upload after quota error"); + skip = true; + } + else { + let size; + try { + // API rounds megabytes to 1 decimal place + size = ((yield OS.File.stat(item.getFilePath())).size / 1024 / 1024).toFixed(1); + } + catch (e) { + Zotero.logError(e); + } + if (size >= remaining) { + Zotero.debug(`Skipping file upload after quota error (${size} >= ${remaining})`); + skip = true; + } + } + if (skip) { + throw yield this._getQuotaError(item); + } + } + + if (multipleFiles) { let created = yield Zotero.Sync.Storage.Utilities.createUploadFile(request); if (!created) { return new Zotero.Sync.Storage.Result; @@ -565,51 +596,16 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = { ); } - let text, buttonText = null, buttonCallback; - let libraryType = item.library.libraryType; + // Store the remaining space so that we can skip files bigger than that until the next + // manual sync + let usage = req.getResponseHeader('Zotero-Storage-Usage'); + let quota = req.getResponseHeader('Zotero-Storage-Quota'); + Zotero.Sync.Storage.Local.storageRemainingForLibrary.set(item.libraryID, quota - usage); - // Group file - if (libraryType == 'group') { - var group = Zotero.Groups.getByLibraryID(item.libraryID); - text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n" - + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2'); - } - // Personal file - else { - text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n" - + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2'); - buttonText = Zotero.getString('sync.storage.openAccountSettings'); - buttonCallback = function () { - var url = "https://www.zotero.org/settings/storage"; - - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var win = wm.getMostRecentWindow("navigator:browser"); - win.ZoteroPane.loadURI(url, { metaKey: true, ctrlKey: true, shiftKey: true }); - } - } - - var filename = item.attachmentFilename; - var fileSize = (yield OS.File.stat(item.getFilePath())).size; - - text += "\n\n" + filename + " (" + Math.round(fileSize / 1024) + "KB)"; - - let e = new Zotero.Error( - text, - "ZFS_OVER_QUOTA", - { - dialogButtonText: buttonText, - dialogButtonCallback: buttonCallback - } - ); - e.errorType = 'warning'; - Zotero.debug(e, 2); - Components.utils.reportError(e); - throw e; + throw yield this._getQuotaError(item); } }), - /** * Given parameters from authorization, upload file to S3 */ @@ -1013,5 +1009,45 @@ Zotero.Sync.Storage.Mode.ZFS.prototype = { return result; } return this._uploadFile(request, item, result); - }) + }), + + + _getQuotaError: async function (item) { + var text, buttonText = null, buttonCallback; + var libraryType = item.library.libraryType; + + // Group file + if (libraryType == 'group') { + let group = Zotero.Groups.getByLibraryID(item.libraryID); + text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n" + + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2'); + } + // Personal file + else { + text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n" + + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2'); + buttonText = Zotero.getString('sync.storage.openAccountSettings'); + buttonCallback = function () { + let url = "https://www.zotero.org/settings/storage"; + let win = Services.wm.getMostRecentWindow("navigator:browser"); + win.ZoteroPane.loadURI(url, { metaKey: true, ctrlKey: true, shiftKey: true }); + } + } + + var filename = item.attachmentFilename; + var fileSize = (await OS.File.stat(item.getFilePath())).size; + + text += "\n\n" + filename + " (" + Math.round(fileSize / 1024) + " KB)"; + + var e = new Zotero.Error( + text, + "ZFS_OVER_QUOTA", + { + dialogButtonText: buttonText, + dialogButtonCallback: buttonCallback + } + ); + e.errorType = 'warning'; + return e; + } } diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index ff79304b6d..bcbda180f2 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -735,6 +735,7 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); yield Zotero.Sync.Data.Local.init(); yield Zotero.Sync.Data.Utilities.init(); + Zotero.Sync.Storage.Local.init(); Zotero.Sync.Runner = new Zotero.Sync.Runner_Module; Zotero.Sync.EventListeners.init(); Zotero.Streamer = new Zotero.Streamer_Module; diff --git a/test/tests/zfsTest.js b/test/tests/zfsTest.js index 48d6013c4c..41bd1d9659 100644 --- a/test/tests/zfsTest.js +++ b/test/tests/zfsTest.js @@ -1001,16 +1001,20 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () { item.synced = true; yield item.saveTx(); + var responses = 0; server.respond(function (req) { if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file` && req.requestBody.indexOf('upload=') == -1 && req.requestHeaders["If-None-Match"] == "*") { + responses++; req.respond( 413, { "Content-Type": "application/json", - "Last-Modified-Version": 10 + "Last-Modified-Version": 10, + "Zotero-Storage-Usage": "300", + "Zotero-Storage-Quota": "300" }, "File would exceed quota (299.7 + 0.5 > 300)" ); @@ -1024,6 +1028,19 @@ describe("Zotero.Sync.Storage.Mode.ZFS", function () { assert.equal(e.errorType, 'warning'); assert.include(e.message, 'test.png'); assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings')); + assert.equal(responses, 1); + + // Try again + var e = yield getPromiseError(zfs.uploadFile({ + name: item.libraryKey + })); + assert.ok(e); + assert.equal(e.errorType, 'warning'); + assert.include(e.message, 'test.png'); + assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings')); + // Shouldn't have been another request. A manual sync resets the flag, but we're not + // testing that here. + assert.equal(responses, 1); }) }) })