From 78b1d2ee35ed56420bfb4f34b73d9971365e5e71 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Fri, 9 Dec 2016 04:36:42 -0500 Subject: [PATCH] Import translation improvements - Don't block the UI with a progress meter during imports. Instead, show a popup in the bottom right when the import is done that shows how many items were saved. - Fix hang when importing some files - Fix various problems with asynchronous operations/transactions - Use the save queue for imports instead of creating concurrent transactions that can time out - Wait for the save to finish before returning from the translate() promise. All save modes now use the save queue, so code that handled the non-save-queue process can probably be removed. - Serialize child attachments instead of running them concurrently. This might make multi-attachment saves a little slower, since they can't download at the same time, but it avoids problems with concurrent transactions. We might be able to improve this to allow concurrent downloads, or allow concurrent saves for a limited number of items (e.g., from web saving) if not for larger imports. - Change collection handling during import, since UI is now active - Select the root collection at the beginning of the import - Assign items and collections to the root during the import instead of at the end - Don't select other collections - Change a few ItemSaver functions to use promises and remove unnecessary callbacks. (This includes some connector code that needs to be tested.) - Change some `parentID` variables in ItemSaver to `parentItemID` for clarity, since collections are now handled in more places To-do: - Save items in smaller batches instead of doing all in the same transaction - Show progress meter in a bottom-right popup during the import --- chrome/content/zotero/fileInterface.js | 50 ++--- .../zotero/xpcom/connector/translate_item.js | 62 +++--- .../content/zotero/xpcom/server_connector.js | 40 ++-- .../zotero/xpcom/translation/translate.js | 126 ++++++----- .../xpcom/translation/translate_item.js | 204 ++++++++++-------- chrome/locale/en-US/zotero/zotero.properties | 3 +- test/tests/translateTest.js | 86 ++++++++ 7 files changed, 327 insertions(+), 244 deletions(-) diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index 4c446ccdf0..4a36066a55 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -320,44 +320,40 @@ var Zotero_File_Interface = new function() { } translation.setTranslator(translators[0]); - translation.setHandler("itemDone", function () { + // TODO: Restore a progress meter + /*translation.setHandler("itemDone", function () { Zotero.updateZoteroPaneProgressMeter(translation.getProgress()); - }); - - // show progress indicator - Zotero_File_Interface.Progress.show( - Zotero.getString("fileInterface.itemsImported") - ); + });*/ yield Zotero.Promise.delay(0); let failed = false; try { - yield translation.translate(libraryID); + yield translation.translate({ + libraryID, + collections: importCollection ? [importCollection.id] : null + }); } catch(e) { Zotero.logError(e); - failed = true; - } - Zotero_File_Interface.Progress.close(); - - // Add items to import collection - if(importCollection) { - yield Zotero.DB.executeTransaction(function* () { - yield importCollection.addItems(translation.newItems.map(item => item.id)); - for(let i=0; i numItems += item.numChildren()); + var progressWin = new Zotero.ProgressWindow(); + progressWin.changeHeadline(Zotero.getString('fileInterface.importComplete')); + if (numItems == 1) { + var icon = translation.newItems[0].getImageSrc(); + } + else { + var icon = 'chrome://zotero/skin/treesource-unfiled' + (Zotero.hiDPI ? "@2x" : "") + '.png'; + } + var title = Zotero.getString(`fileInterface.itemsWereImported`, numItems, numItems); + progressWin.addLines(title, icon) + progressWin.show(); + progressWin.startCloseTimer(); }); /* diff --git a/chrome/content/zotero/xpcom/connector/translate_item.js b/chrome/content/zotero/xpcom/connector/translate_item.js index d9012a494c..a3b9339493 100644 --- a/chrome/content/zotero/xpcom/connector/translate_item.js +++ b/chrome/content/zotero/xpcom/connector/translate_item.js @@ -73,16 +73,12 @@ Zotero.Translate.ItemSaver.prototype = { /** * Saves items to Standalone or the server * @param items Items in Zotero.Item.toArray() format - * @param {Function} callback A callback to be executed when saving is complete. If saving - * succeeded, this callback will be passed true as the first argument and a list of items - * saved as the second. If saving failed, the callback will be passed false as the first - * argument and an error object as the second * @param {Function} [attachmentCallback] A callback that receives information about attachment * save progress. The callback will be called as attachmentCallback(attachment, false, error) * on failure or attachmentCallback(attachment, progressPercent) periodically during saving. */ - "saveItems":function(items, callback, attachmentCallback) { - var me = this; + saveItems: function (items, attachmentCallback) { + var deferred = Zotero.Promise.defer(); // first try to save items via connector var payload = {"items":items}; Zotero.Connector.setCookiesThenSaveItems(payload, function(data, status) { @@ -100,14 +96,15 @@ Zotero.Translate.ItemSaver.prototype = { } } } - callback(true, items); - if(haveAttachments) me._pollForProgress(items, attachmentCallback); + deferred.resolve(items); + if (haveAttachments) this._pollForProgress(items, attachmentCallback); } else if(Zotero.isFx) { - callback(false, new Error("Save via Standalone failed with "+status)); + deferred.reject(new Error("Save via Standalone failed with " + status)); } else { - me._saveToServer(items, callback, attachmentCallback); + deferred.resolve(this._saveToServer(items, attachmentCallback)); } - }); + }.bind(this)); + return deferred.promise; }, /** @@ -169,16 +166,12 @@ Zotero.Translate.ItemSaver.prototype = { /** * Saves items to server * @param items Items in Zotero.Item.toArray() format - * @param {Function} callback A callback to be executed when saving is complete. If saving - * succeeded, this callback will be passed true as the first argument and a list of items - * saved as the second. If saving failed, the callback will be passed false as the first - * argument and an error object as the second * @param {Function} attachmentCallback A callback that receives information about attachment * save progress. The callback will be called as attachmentCallback(attachment, false, error) * on failure or attachmentCallback(attachment, progressPercent) periodically during saving. * attachmentCallback() will be called with all attachments that will be saved */ - "_saveToServer":function(items, callback, attachmentCallback) { + _saveToServer: function (items, attachmentCallback) { var newItems = [], itemIndices = [], typedArraysSupported = false; try { typedArraysSupported = !!(new Uint8Array(1) && new Blob()); @@ -197,41 +190,42 @@ Zotero.Translate.ItemSaver.prototype = { } } - var me = this; + var deferred = Zotero.Promise.defer(); Zotero.API.createItem({"items":newItems}, function(statusCode, response) { if(statusCode !== 200) { - callback(false, new Error("Save to server failed with "+statusCode+" "+response)); + deferred.reject(new Error("Save to server failed with " + statusCode + " " + response)); return; } try { var resp = JSON.parse(response); } catch(e) { - callback(false, new Error("Unexpected response received from server")); + deferred.reject(new Error("Unexpected response received from server")); return; } for(var i in resp.failed) { - callback(false, new Error("Save to server failed with "+statusCode+" "+response)); + deferred.reject(new Error("Save to server failed with " + statusCode + " " + response)); return; } Zotero.debug("Translate: Save to server complete"); - Zotero.Prefs.getCallback(["downloadAssociatedFiles", "automaticSnapshots"], - function(prefs) { - - if(typedArraysSupported) { - for(var i=0; i this.saveQueue = []); return; } this._debug("Translation successful"); @@ -1540,69 +1535,70 @@ Zotero.Translate.Base.prototype = { * Saves items to the database, taking care to defer attachmentProgress notifications * until after save */ - "_saveItems":function(items) { - var me = this, - itemDoneEventsDispatched = false, - deferredProgress = [], - attachmentsWithProgress = []; + _saveItems: function (items) { + var itemDoneEventsDispatched = false; + var deferredProgress = []; + var attachmentsWithProgress = []; this._savingItems++; - this._itemSaver.saveItems(items.slice(), function(returnValue, newItems) { - if(returnValue) { - // Remove attachments not being saved from item.attachments - for(var i=0; i this._saveAttachment(item, parentItemID, attachmentCallback)); + } + + return newItems; }), "saveCollections": Zotero.Promise.coroutine(function* (collections) { var collectionsToProcess = collections.slice(); - var parentIDs = [null]; + // Use first collection passed to translate process as the root + var rootCollectionID = (this._collections && this._collections.length) + ? this._collections[0] : null; + var parentIDs = collections.map(c => null); var topLevelCollections = []; yield Zotero.DB.executeTransaction(function* () { @@ -175,9 +179,13 @@ Zotero.Translate.ItemSaver.prototype = { if (parentID) { newCollection.parentID = parentID; } - yield newCollection.save(); - - if(parentID === null) topLevelCollections.push(newCollection); + else { + newCollection.parentID = rootCollectionID; + topLevelCollections.push(newCollection) + } + yield newCollection.save({ + skipSelect: true + }); var toAdd = []; @@ -222,7 +230,7 @@ Zotero.Translate.ItemSaver.prototype = { * Saves a translator attachment to the database * * @param {Translator Attachment} attachment - * @param {Integer} parentID Item to attach to + * @param {Integer} parentItemID - Item to attach to * @param {Function} attachmentCallback Callback function that takes three * parameters: translator attachment object, percent completion (integer), * and an optional error object @@ -230,7 +238,7 @@ Zotero.Translate.ItemSaver.prototype = { * @return {Zotero.Primise c.id)); + }); + }); + + describe("#_saveAttachment()", function () { + it("should save standalone attachment to collection", function* () { + var collection = yield createDataObject('collection'); + var items = [ + { + itemType: "attachment", + title: "Test", + mimeType: "text/html", + url: "http://example.com" + } + ]; + + var translation = new Zotero.Translate.Import(); + translation.setString(""); + translation.setTranslator(buildDummyTranslator( + "import", + "function detectImport() {}\n" + + "function doImport() {\n" + + " var json = JSON.parse('" + JSON.stringify(items).replace(/['\\]/g, "\\$&") + "');\n" + + " for (var i=0; i