mirror of
https://github.com/zotero/zotero.git
synced 2026-09-06 08:16:02 +00:00
20735c425f
resetData() clears user data and sync state using application APIs instead of doing a full Zotero shutdown/reinit cycle, making it much faster for tests that just need clean state between runs. Also make Tags.init(), Creators.init(), Users.init(), and SyncedSettings.loadAll() properly clear their caches before reloading, so they are safe to call more than once.
1213 lines
36 KiB
JavaScript
1213 lines
36 KiB
JavaScript
"use strict";
|
|
|
|
describe("Zotero.Sync.Storage.Mode.ZFS", function () {
|
|
//
|
|
// Setup
|
|
//
|
|
var apiKey = Zotero.Utilities.randomString(24);
|
|
|
|
var win, server, requestCount, httpd, baseURL;
|
|
var responses = {};
|
|
|
|
function setResponse(response) {
|
|
setHTTPResponse(server, baseURL, response, responses);
|
|
}
|
|
|
|
function resetRequestCount() {
|
|
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
|
|
}
|
|
|
|
function assertRequestCount(count) {
|
|
assert.equal(
|
|
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
|
|
count
|
|
);
|
|
}
|
|
|
|
function parseQueryString(str) {
|
|
var queryStringParams = str.split('&');
|
|
var params = {};
|
|
for (let param of queryStringParams) {
|
|
let [ key, val ] = param.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
return params;
|
|
}
|
|
|
|
function assertAPIKey(request) {
|
|
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
|
|
}
|
|
|
|
//
|
|
// Tests
|
|
//
|
|
beforeEach(async function () {
|
|
await resetData();
|
|
win = await loadZoteroPane();
|
|
|
|
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
|
|
server = sinon.fakeServer.create();
|
|
server.autoRespond = true;
|
|
|
|
var port;
|
|
({ httpd, port } = await startHTTPServer());
|
|
baseURL = `http://localhost:${port}/`;
|
|
|
|
await Zotero.Users.setCurrentUserID(1);
|
|
await Zotero.Users.setCurrentUsername("testuser");
|
|
|
|
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs');
|
|
|
|
// Set download-on-sync by default
|
|
Zotero.Sync.Storage.Local.downloadOnSync(
|
|
Zotero.Libraries.userLibraryID, true
|
|
);
|
|
|
|
resetRequestCount();
|
|
})
|
|
|
|
var setup = async function (options = {}) {
|
|
const { ConcurrentCaller } = ChromeUtils.importESModule("resource://zotero/concurrentCaller.mjs");
|
|
var stopOnError = options.stopOnError !== undefined ? options.stopOnError : true;
|
|
var caller = new ConcurrentCaller(1);
|
|
caller.setLogger(msg => Zotero.debug(msg));
|
|
caller.stopOnError = stopOnError;
|
|
|
|
var client = new Zotero.Sync.APIClient({
|
|
baseURL,
|
|
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
|
|
apiKey,
|
|
caller,
|
|
background: options.background || true
|
|
});
|
|
|
|
var engine = new Zotero.Sync.Storage.Engine({
|
|
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
|
|
controller: new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client,
|
|
maxS3ConsecutiveFailures: 2
|
|
}),
|
|
background: options.background,
|
|
stopOnError
|
|
});
|
|
|
|
return { engine, client, caller };
|
|
}
|
|
|
|
afterEach(function* () {
|
|
Zotero.HTTP.mock = null;
|
|
var defer = Zotero.Promise.defer();
|
|
httpd.stop(() => defer.resolve());
|
|
yield defer.promise;
|
|
win.close();
|
|
Zotero.HTTP.disableErrorRetry = false;
|
|
})
|
|
|
|
after(function* () {
|
|
this.timeout(60000);
|
|
//yield resetDB();
|
|
win.close();
|
|
})
|
|
|
|
|
|
describe("Syncing", function () {
|
|
it("should skip downloads if not marked as needed", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
|
|
var result = await engine.start();
|
|
|
|
assertRequestCount(0);
|
|
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
})
|
|
|
|
it("should ignore download for a remotely missing file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
|
|
httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 404, null);
|
|
}
|
|
}
|
|
);
|
|
var result = await engine.start();
|
|
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
assert.isFalse(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
assert.equal(
|
|
item.attachmentSyncState,
|
|
Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC
|
|
);
|
|
})
|
|
|
|
it("shouldn't update storageVersion if stopped", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var items = [];
|
|
for (let i = 0; i < 5; i++) {
|
|
let item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
var call = 0;
|
|
var stub = sinon.stub(engine.controller, 'downloadFile').callsFake(function () {
|
|
call++;
|
|
if (call == 1) {
|
|
engine.stop();
|
|
}
|
|
return new Zotero.Sync.Storage.Result;
|
|
});
|
|
|
|
var result = await engine.start();
|
|
|
|
stub.restore();
|
|
|
|
assert.equal(library.storageVersion, 0);
|
|
});
|
|
|
|
it("should handle a remotely failing file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
|
|
Zotero.HTTP.disableErrorRetry = true;
|
|
httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 500, null);
|
|
}
|
|
}
|
|
);
|
|
// TODO: In stopOnError mode, this the promise is rejected.
|
|
// This should probably test with stopOnError mode turned off instead.
|
|
var e = await getPromiseError(engine.start());
|
|
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
|
|
|
|
assert.isTrue(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, 0);
|
|
})
|
|
|
|
it("should download a missing file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
// TODO: Test binary data
|
|
var text = Zotero.Utilities.randomString();
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
|
|
var mtime = "1441252524905";
|
|
var md5 = Zotero.Utilities.Internal.md5(text)
|
|
|
|
var s3Path = `pretend-s3/${item.key}`;
|
|
httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
if (!request.hasHeader('Zotero-API-Key')) {
|
|
response.setStatusLine(null, 403, "Forbidden");
|
|
return;
|
|
}
|
|
var key = request.getHeader('Zotero-API-Key');
|
|
if (key != apiKey) {
|
|
response.setStatusLine(null, 403, "Invalid key");
|
|
return;
|
|
}
|
|
response.setStatusLine(null, 302, "Found");
|
|
response.setHeader("Zotero-File-Modification-Time", mtime, false);
|
|
response.setHeader("Zotero-File-MD5", md5, false);
|
|
response.setHeader("Zotero-File-Compressed", "No", false);
|
|
response.setHeader("Location", baseURL + s3Path, false);
|
|
}
|
|
}
|
|
);
|
|
httpd.registerPathHandler(
|
|
"/" + s3Path,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 200, "OK");
|
|
response.write(text);
|
|
}
|
|
}
|
|
);
|
|
var result = await engine.start();
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
var contents = await Zotero.File.getContentsAsync(await item.getFilePathAsync());
|
|
assert.equal(contents, text);
|
|
|
|
assert.isFalse(library.storageDownloadNeeded);
|
|
assert.equal(library.storageVersion, library.libraryVersion);
|
|
})
|
|
|
|
it("should notify when file is downloaded", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var item = new Zotero.Item("attachment");
|
|
item.attachmentLinkMode = 'imported_file';
|
|
item.attachmentPath = 'storage:test.txt';
|
|
var text = Zotero.Utilities.randomString();
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
|
|
var mtime = "1441252524905";
|
|
var md5 = Zotero.Utilities.Internal.md5(text);
|
|
|
|
var s3Path = `pretend-s3/${item.key}`;
|
|
httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 302, "Found");
|
|
response.setHeader("Zotero-File-Modification-Time", mtime, false);
|
|
response.setHeader("Zotero-File-MD5", md5, false);
|
|
response.setHeader("Zotero-File-Compressed", "No", false);
|
|
response.setHeader("Location", baseURL + s3Path, false);
|
|
}
|
|
}
|
|
);
|
|
httpd.registerPathHandler(
|
|
"/" + s3Path,
|
|
{
|
|
handle: function (request, response) {
|
|
response.setStatusLine(null, 200, "OK");
|
|
response.write(text);
|
|
}
|
|
}
|
|
);
|
|
|
|
var deferred = Zotero.Promise.defer();
|
|
var observerID = Zotero.Notifier.registerObserver({
|
|
notify: async function (event, type, ids, extraData) {
|
|
if (event == 'download' && ids[0] == item.id && (await item.getFilePathAsync())) {
|
|
deferred.resolve();
|
|
}
|
|
}
|
|
}, 'file', 'testFileDownload');
|
|
|
|
await engine.start();
|
|
await deferred.promise;
|
|
Zotero.Notifier.unregisterObserver(observerID);
|
|
});
|
|
|
|
it("should upload new files", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
// Single file
|
|
var file1 = getTestDataDirectory();
|
|
file1.append('test.png');
|
|
var item1 = await Zotero.Attachments.importFromFile({ file: file1 });
|
|
var mtime1 = await item1.attachmentModificationTime;
|
|
var hash1 = await item1.attachmentHash;
|
|
var path1 = item1.getFilePath();
|
|
var filename1 = 'test.png';
|
|
var size1 = ((await OS.File.stat(path1))).size;
|
|
var contentType1 = 'image/png';
|
|
var prefix1 = Zotero.Utilities.randomString();
|
|
var suffix1 = Zotero.Utilities.randomString();
|
|
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
let file1Blob = File.createFromFileName ? File.createFromFileName(file1.path) : new File(file1);
|
|
if (file1Blob.then) {
|
|
file1Blob = await file1Blob;
|
|
}
|
|
|
|
// HTML file with auxiliary image
|
|
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
|
|
var parentItem = await createDataObject('item');
|
|
var item2 = await Zotero.Attachments.importSnapshotFromFile({
|
|
file: file2,
|
|
url: 'http://example.com/',
|
|
parentItemID: parentItem.id,
|
|
title: 'Test',
|
|
contentType: 'text/html',
|
|
charset: 'utf-8'
|
|
});
|
|
var mtime2 = await item2.attachmentModificationTime;
|
|
var hash2 = await item2.attachmentHash;
|
|
var path2 = item2.getFilePath();
|
|
var filename2 = 'index.html';
|
|
var size2 = ((await OS.File.stat(path2))).size;
|
|
var contentType2 = 'text/html';
|
|
var charset2 = 'utf-8';
|
|
var prefix2 = Zotero.Utilities.randomString();
|
|
var suffix2 = Zotero.Utilities.randomString();
|
|
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
var deferreds = [];
|
|
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Get upload authorization for single file
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.md5, hash1);
|
|
assert.equal(params.mtime, mtime1);
|
|
assert.equal(params.filename, filename1);
|
|
assert.equal(params.filesize, size1);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/1",
|
|
contentType: contentType1,
|
|
prefix: prefix1,
|
|
suffix: suffix1,
|
|
uploadKey: uploadKey1
|
|
})
|
|
);
|
|
}
|
|
// Get upload authorization for multi-file zip
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
// Verify ZIP hash
|
|
let tmpZipPath = OS.Path.join(
|
|
Zotero.getTempDirectory().path,
|
|
item2.key + '.zip'
|
|
);
|
|
deferreds.push({
|
|
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
|
|
.then(function (md5) {
|
|
assert.equal(params.zipMD5, md5);
|
|
})
|
|
});
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
Zotero.debug(params);
|
|
assert.equal(params.md5, hash2);
|
|
assert.notEqual(params.zipMD5, hash2);
|
|
assert.equal(params.mtime, mtime2);
|
|
assert.equal(params.filename, filename2);
|
|
assert.equal(params.zipFilename, item2.key + ".zip");
|
|
assert.isTrue(parseInt(params.filesize) == params.filesize);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/2",
|
|
contentType: 'application/zip',
|
|
prefix: prefix2,
|
|
suffix: suffix2,
|
|
uploadKey: uploadKey2
|
|
})
|
|
);
|
|
}
|
|
// Upload single file to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
|
|
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
|
|
assert.equal(
|
|
req.requestBody.size,
|
|
(new Blob(
|
|
[
|
|
prefix1,
|
|
file1Blob,
|
|
suffix1
|
|
]
|
|
).size)
|
|
);
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Upload multi-file ZIP to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
|
|
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
|
|
|
|
// Verify uploaded ZIP file
|
|
let tmpZipPath = OS.Path.join(
|
|
Zotero.getTempDirectory().path,
|
|
Zotero.Utilities.randomString() + '.zip'
|
|
);
|
|
|
|
let deferred = Zotero.Promise.defer();
|
|
deferreds.push(deferred);
|
|
var reader = new FileReader();
|
|
reader.addEventListener("loadend", async function () {
|
|
try {
|
|
let contents = new Uint8Array(reader.result);
|
|
contents = contents.slice(prefix2.length, suffix2.length * -1);
|
|
await IOUtils.write(tmpZipPath, contents);
|
|
|
|
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
|
|
.createInstance(Components.interfaces.nsIZipReader);
|
|
zr.open(Zotero.File.pathToFile(tmpZipPath));
|
|
zr.test(null);
|
|
var entries = zr.findEntries('*');
|
|
var entryNames = [];
|
|
while (entries.hasMore()) {
|
|
entryNames.push(entries.getNext());
|
|
}
|
|
assert.equal(entryNames.length, 2);
|
|
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
|
|
assert.equal(zr.getEntry('index.html').realSize, size2);
|
|
assert.equal(zr.getEntry('img.gif').realSize, 42);
|
|
|
|
deferred.resolve();
|
|
}
|
|
catch (e) {
|
|
deferred.reject(e);
|
|
}
|
|
});
|
|
reader.readAsArrayBuffer(req.requestBody);
|
|
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Register single-file upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
|
|
&& req.requestBody.indexOf('upload=') != -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.upload, uploadKey1);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 10
|
|
},
|
|
""
|
|
);
|
|
}
|
|
// Register multi-file upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
|
|
&& req.requestBody.indexOf('upload=') != -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
let parts = req.requestBody.split('&');
|
|
let params = {};
|
|
for (let part of parts) {
|
|
let [key, val] = part.split('=');
|
|
params[key] = decodeURIComponent(val);
|
|
}
|
|
assert.equal(params.upload, uploadKey2);
|
|
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 15
|
|
},
|
|
""
|
|
);
|
|
}
|
|
})
|
|
|
|
// TODO: One-step uploads
|
|
/*// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/json" + fixSinonBug
|
|
);
|
|
|
|
let params = JSON.parse(req.requestBody);
|
|
assert.equal(params.md5, hash);
|
|
assert.equal(params.mtime, mtime);
|
|
assert.equal(params.filename, filename);
|
|
assert.equal(params.size, size);
|
|
assert.equal(params.contentType, contentType);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3",
|
|
headers: {
|
|
"Content-Type": contentType,
|
|
"Content-MD5": hash,
|
|
//"Content-Length": params.size, process but don't return
|
|
//"x-amz-meta-"
|
|
},
|
|
uploadKey
|
|
})
|
|
);
|
|
}
|
|
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
|
|
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
|
|
assert.instanceOf(req.requestBody, File);
|
|
req.respond(201, {}, "");
|
|
}
|
|
})*/
|
|
var result = await engine.start();
|
|
|
|
await Promise.all(deferreds.map(d => d.promise));
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isTrue(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
// Check local objects
|
|
assert.equal(item1.attachmentSyncedModificationTime, mtime1);
|
|
assert.equal(item1.attachmentSyncedHash, hash1);
|
|
assert.equal(item1.version, 10);
|
|
assert.equal(item2.attachmentSyncedModificationTime, mtime2);
|
|
assert.equal(item2.attachmentSyncedHash, hash2);
|
|
assert.equal(item2.version, 15);
|
|
})
|
|
|
|
it("should update local info for remotely updated file that matches local file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var library = Zotero.Libraries.userLibrary;
|
|
library.libraryVersion = 5;
|
|
await library.saveTx();
|
|
library.storageDownloadNeeded = true;
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.txt');
|
|
var item = await Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.attachmentSyncState = "to_download";
|
|
await item.saveTx();
|
|
var path = await item.getFilePathAsync();
|
|
await OS.File.setDates(path, null, new Date() - 100000);
|
|
|
|
var json = item.toJSON();
|
|
await Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
|
|
|
|
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) + "";
|
|
var md5 = Zotero.Utilities.Internal.md5(file)
|
|
|
|
var processDownloadSpy = sinon.spy(Zotero.Sync.Storage.Local, "processDownload");
|
|
|
|
var s3Path = `pretend-s3/${item.key}`;
|
|
httpd.registerPathHandler(
|
|
`/users/1/items/${item.key}/file`,
|
|
{
|
|
handle: function (request, response) {
|
|
if (!request.hasHeader('Zotero-API-Key')) {
|
|
response.setStatusLine(null, 403, "Forbidden");
|
|
return;
|
|
}
|
|
var key = request.getHeader('Zotero-API-Key');
|
|
if (key != apiKey) {
|
|
response.setStatusLine(null, 403, "Invalid key");
|
|
return;
|
|
}
|
|
response.setStatusLine(null, 302, "Found");
|
|
response.setHeader("Zotero-File-Modification-Time", mtime, false);
|
|
response.setHeader("Zotero-File-MD5", md5, false);
|
|
response.setHeader("Zotero-File-Compressed", "No", false);
|
|
response.setHeader("Location", baseURL + s3Path, false);
|
|
}
|
|
}
|
|
);
|
|
var result = await engine.start();
|
|
|
|
assert.equal(item.attachmentSyncedModificationTime, mtime);
|
|
assert.equal(await item.attachmentModificationTime, mtime);
|
|
assert.isTrue(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
assert.isTrue(processDownloadSpy.notCalled);
|
|
|
|
processDownloadSpy.restore();
|
|
})
|
|
|
|
it("should update local info for file that already exists on the server", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file: file });
|
|
item.version = 5;
|
|
await item.saveTx();
|
|
var json = item.toJSON();
|
|
await Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
|
|
|
|
var mtime = await item.attachmentModificationTime;
|
|
var hash = await item.attachmentHash;
|
|
var path = item.getFilePath();
|
|
var filename = 'test.png';
|
|
var size = ((await OS.File.stat(path))).size;
|
|
var contentType = 'image/png';
|
|
|
|
var newVersion = 10;
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Get upload authorization for single file
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1) {
|
|
assertAPIKey(req);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
assert.equal(
|
|
req.requestHeaders["Content-Type"],
|
|
"application/x-www-form-urlencoded" + fixSinonBug
|
|
);
|
|
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": newVersion
|
|
},
|
|
JSON.stringify({
|
|
exists: 1,
|
|
})
|
|
);
|
|
}
|
|
})
|
|
|
|
// TODO: One-step uploads
|
|
var result = await engine.start();
|
|
|
|
assert.isTrue(result.localChanges);
|
|
assert.isTrue(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
|
|
// Check local objects
|
|
assert.equal(item.attachmentSyncedModificationTime, mtime);
|
|
assert.equal(item.attachmentSyncedHash, hash);
|
|
assert.equal(item.version, newVersion);
|
|
})
|
|
|
|
|
|
it("should retry with If-None-Match on 412 with missing remote hash", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
item.attachmentSyncedModificationTime = Date.now();
|
|
item.attachmentSyncedHash = 'bd4c33e03798a7e8bc0b46f8bda74fac'
|
|
await item.saveTx();
|
|
|
|
var contentType = 'image/png';
|
|
var prefix = Zotero.Utilities.randomString();
|
|
var suffix = Zotero.Utilities.randomString();
|
|
var uploadKey = Zotero.Utilities.randomString(32, 'abcdef0123456789');
|
|
|
|
var called = 0;
|
|
// https://github.com/cjohansen/Sinon.JS/issues/607
|
|
let fixSinonBug = ";charset=utf-8";
|
|
server.respond(function (req) {
|
|
// Try with If-Match
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')
|
|
&& req.requestHeaders["If-Match"] == item.attachmentSyncedHash) {
|
|
called++;
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
"If-Match set but file does not exist"
|
|
);
|
|
}
|
|
// Retry with If-None-Match
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
assert.equal(called++, 1);
|
|
req.respond(
|
|
200,
|
|
{
|
|
"Content-Type": "application/json"
|
|
},
|
|
JSON.stringify({
|
|
url: baseURL + "pretend-s3/1",
|
|
contentType: contentType,
|
|
prefix: prefix,
|
|
suffix: suffix,
|
|
uploadKey: uploadKey
|
|
})
|
|
);
|
|
}
|
|
// Upload file to S3
|
|
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
|
|
assert.equal(called++, 2);
|
|
req.respond(201, {}, "");
|
|
}
|
|
// Use If-None-Match when registering upload
|
|
else if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.includes('upload=')) {
|
|
assert.equal(called++, 3);
|
|
assert.equal(req.requestHeaders["If-None-Match"], "*");
|
|
req.respond(
|
|
204,
|
|
{
|
|
"Last-Modified-Version": 10
|
|
},
|
|
""
|
|
);
|
|
}
|
|
});
|
|
|
|
var result = await engine.start();
|
|
assert.equal(called, 4);
|
|
});
|
|
|
|
|
|
it("should stop uploading files on quota error", async function () {
|
|
var { engine, client, caller } = await setup({ stopOnError: false });
|
|
|
|
var numItems = 4;
|
|
var items = [];
|
|
for (let i = 0; i < numItems; i++) {
|
|
let item = await importFileAttachment('test.png');
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
var requests = 0;
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url.startsWith(`${baseURL}users/1/items/`)
|
|
&& req.url.endsWith('/file')
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
requests++;
|
|
req.respond(
|
|
413,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10,
|
|
"Zotero-Storage-Usage": "300",
|
|
"Zotero-Storage-Quota": "300"
|
|
},
|
|
"File would exceed quota (299.7 + 0.5 > 300)"
|
|
);
|
|
}
|
|
})
|
|
|
|
await engine.start();
|
|
assert.equal(requests, Zotero.Prefs.get('sync.storage.maxUploads'));
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.delete(items[0].libraryID);
|
|
});
|
|
|
|
|
|
// If there was a quota error in a previous run and remaining storage was determined to be
|
|
// very low, stop further file uploads for a background sync even when we bail without an
|
|
// HTTP request. A manual sync clears the remaining-storage value.
|
|
it("should stop uploading files for background sync if no storage remaining after previous quota error", async function () {
|
|
var { engine, client, caller } = await setup({ background: true, stopOnError: false });
|
|
|
|
var numItems = 4;
|
|
var items = [];
|
|
for (let i = 0; i < numItems; i++) {
|
|
let item = await importFileAttachment('test.png');
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
items.push(item);
|
|
}
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.set(items[0].libraryID, 0);
|
|
|
|
var spy = sinon.spy(engine.controller, 'uploadFile');
|
|
await engine.start()
|
|
|
|
assert.equal(spy.callCount, Zotero.Prefs.get('sync.storage.maxUploads'));
|
|
spy.restore();
|
|
|
|
Zotero.Sync.Storage.Local.storageRemainingForLibrary.delete(items[0].libraryID);
|
|
});
|
|
})
|
|
|
|
|
|
describe("#uploadFile()", function () {
|
|
it("should compress single-file HTML snapshots", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var dir = await getTempDirectory();
|
|
var file = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
|
|
var file2 = OS.Path.join(dir, 'index.html');
|
|
await OS.File.copy(file, file2);
|
|
file = file2;
|
|
|
|
var parentItem = await createDataObject('item');
|
|
var item = await Zotero.Attachments.importSnapshotFromFile({
|
|
file,
|
|
url: 'http://example.com/',
|
|
parentItemID: parentItem.id,
|
|
title: 'Test',
|
|
contentType: 'text/html',
|
|
charset: 'utf-8'
|
|
});
|
|
|
|
var request = { name: item.libraryKey };
|
|
|
|
var stub = sinon.stub(zfs, '_processUploadFile').returns(Promise.resolve());
|
|
await zfs.uploadFile(request);
|
|
|
|
var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
|
|
// _getUploadFile() should return the ZIP file
|
|
assert.equal((await zfs._getUploadFile(item)).path, zipFile);
|
|
assert.isTrue(await OS.File.exists(zipFile));
|
|
|
|
stub.restore();
|
|
});
|
|
});
|
|
|
|
|
|
describe("#_processUploadFile()", function () {
|
|
it("should handle 404 from upload authorization request", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.mtime = await item.attachmentModificationTime;
|
|
itemJSON.data.md5 = await item.attachmentHash;
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& !req.requestBody.includes('upload=')) {
|
|
req.respond(
|
|
404,
|
|
{
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"Not Found"
|
|
);
|
|
}
|
|
})
|
|
|
|
var result = await zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.isTrue(result.syncRequired);
|
|
});
|
|
|
|
it("should handle 412 with matching version and hash matching local file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.mtime = await item.attachmentModificationTime;
|
|
itemJSON.data.md5 = await item.attachmentHash;
|
|
|
|
// Set saved hash to a different value, which should be overwritten
|
|
//
|
|
// We're also testing cases where a hash isn't set for a file (e.g., if the
|
|
// storage directory was transferred, the mtime doesn't match, but the file was
|
|
// never downloaded), but there's no difference in behavior
|
|
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
item.attachmentSyncedHash = dbHash;
|
|
await item.saveTx({ skipAll: true });
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-Match"] == dbHash) {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"ETag does not match current version of file"
|
|
);
|
|
}
|
|
})
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items?itemKey=${item.key}&includeTrashed=1`,
|
|
status: 200,
|
|
text: JSON.stringify([itemJSON])
|
|
});
|
|
|
|
var result = await zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.equal(item.attachmentSyncedHash, ((await item.attachmentHash)));
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
assert.isFalse(result.fileSyncRequired);
|
|
})
|
|
|
|
it("should handle 412 with matching version and hash not matching local file", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file: filePath });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
|
|
var fileHash = await item.attachmentHash;
|
|
var itemJSON = item.toResponseJSON();
|
|
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 5
|
|
},
|
|
"If-None-Match: * set but file exists"
|
|
);
|
|
}
|
|
})
|
|
setResponse({
|
|
method: "GET",
|
|
url: `users/1/items?itemKey=${item.key}&includeTrashed=1`,
|
|
status: 200,
|
|
text: JSON.stringify([itemJSON])
|
|
});
|
|
|
|
var result = await zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.isNull(item.attachmentSyncedHash);
|
|
assert.equal(item.attachmentSyncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT);
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isFalse(result.syncRequired);
|
|
assert.isTrue(result.fileSyncRequired);
|
|
})
|
|
|
|
it("should handle 412 with greater version", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
req.respond(
|
|
412,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10
|
|
},
|
|
"If-None-Match: * set but file exists"
|
|
);
|
|
}
|
|
})
|
|
|
|
var result = await zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
});
|
|
assert.equal(item.version, 5);
|
|
assert.equal(item.synced, true);
|
|
assert.isFalse(result.localChanges);
|
|
assert.isFalse(result.remoteChanges);
|
|
assert.isTrue(result.syncRequired);
|
|
// Item should be marked for redownloading
|
|
var versions = await Zotero.Sync.Data.Local.getObjectsToTryFromSyncQueue('item', item.libraryID);
|
|
assert.include(versions, item.key);
|
|
});
|
|
|
|
|
|
it("should handle 413 on quota limit", async function () {
|
|
var { engine, client, caller } = await setup();
|
|
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
|
|
apiClient: client
|
|
})
|
|
|
|
var file = getTestDataDirectory();
|
|
file.append('test.png');
|
|
var item = await Zotero.Attachments.importFromFile({ file });
|
|
item.version = 5;
|
|
item.synced = true;
|
|
await item.saveTx();
|
|
|
|
var responses = 0;
|
|
server.respond(function (req) {
|
|
if (req.method == "POST"
|
|
&& req.url == `${baseURL}users/1/items/${item.key}/file`
|
|
&& req.requestBody.indexOf('upload=') == -1
|
|
&& req.requestHeaders["If-None-Match"] == "*") {
|
|
responses++;
|
|
req.respond(
|
|
413,
|
|
{
|
|
"Content-Type": "application/json",
|
|
"Last-Modified-Version": 10,
|
|
"Zotero-Storage-Usage": "300",
|
|
"Zotero-Storage-Quota": "300"
|
|
},
|
|
"File would exceed quota (299.7 + 0.5 > 300)"
|
|
);
|
|
}
|
|
})
|
|
|
|
var e = await getPromiseError(zfs._processUploadFile({
|
|
name: item.libraryKey
|
|
}));
|
|
assert.ok(e);
|
|
assert.equal(e.errorType, 'warning');
|
|
assert.include(e.message, 'test.png');
|
|
assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings'));
|
|
assert.equal(responses, 1);
|
|
|
|
// Try again
|
|
var e = await getPromiseError(zfs.uploadFile({
|
|
name: item.libraryKey,
|
|
engine
|
|
}));
|
|
assert.ok(e);
|
|
assert.equal(e.errorType, 'warning');
|
|
assert.include(e.message, 'test.png');
|
|
assert.equal(e.dialogButtonText, Zotero.getString('sync.storage.openAccountSettings'));
|
|
// Shouldn't have been another request. A manual sync resets the flag, but we're not
|
|
// testing that here.
|
|
assert.equal(responses, 1);
|
|
})
|
|
})
|
|
})
|