scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-08-28 05:25:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Add platform-native file change watcher for storage sync

Browse source 
Use platform-native file change APIs to track which stored-file
attachments have been modified, avoiding expensive full scans of all
attachment files during sync.

Backends:
- macOS: FSEvents (persistent event journal, survives restarts)
- Windows: ReadDirectoryChangesW (live recursive directory watch
  via overlapped I/O polling)
- Linux: inotify (live per-directory watches)

On unsupported platforms or on error, falls back to existing scan logic.
This commit is contained in:
Dan Stillman 2026-02-23 00:58:12 -05:00
parent 2739b29709
commit f21e1b2d32
8 changed files with 1757 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

190
chrome/content/zotero/xpcom/storage/fileChangeWatcher.js Normal file
View file

@ -0,0 +1,190 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2026 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* File change watcher for storage sync
*
* Uses platform-native file change APIs to track which stored-file attachments have been modified,
* avoiding expensive full scans of all attachment files.
*
* Backends (loaded from separate files):
* macOS -- FSEvents (persistent event journal, survives restarts)
* Windows -- ReadDirectoryChangesW (live recursive directory watch)
* Linux -- inotify (live per-directory watches)
*
* On unsupported platforms or on error, falls back to the existing scan logic.
*/
Zotero.Sync.Storage.FileChangeWatcher = {
available: false,
_backend: null,
// Cached storage root path (normalized, with trailing separator)
_storageRoot: null,
// Key validation pattern
_keyPattern: /^[A-Z0-9]{8}$/,
// For live watcher backends (RDCW and inotify), track whether the first call has happened
// (returns null to trigger a full scan) and enforce periodic full scans every
// _MAX_WATCHER_AGE ms
_liveWatcherFirstCall: true,
_liveWatcherLastFullScanTime: 0,
_MAX_WATCHER_AGE: 10800000, // 3 hours -- matches storageEngine.maxCheckAge
init() {
try {
// Ensure the storage directory exists -- on a new installation it may not have
// been created yet, and the backends need it to set up their watches
let storageDir = Zotero.getStorageDirectory();
storageDir.normalize();
let sep = Zotero.isWin ? "\\" : "/";
let path = storageDir.path;
if (!path.endsWith(sep)) {
path += sep;
}
this._storageRoot = path;
}
catch (e) {
Zotero.logError(e);
Zotero.debug("FileChangeWatcher: Could not resolve storage root");
return;
}
let initFn;
let backendName;
if (Zotero.isMac) {
initFn = '_initFSEvents';
backendName = 'fsevents';
}
else if (Zotero.isWin) {
initFn = '_initRDCW';
backendName = 'rdcw';
}
else if (Zotero.isLinux) {
initFn = '_initInotify';
backendName = 'inotify';
}
else {
Zotero.debug("FileChangeWatcher: No backend available for this platform");
return;
}
try {
this[initFn]();
this._backend = backendName;
this.available = true;
Zotero.debug(`FileChangeWatcher: ${backendName} backend initialized`);
}
catch (e) {
Zotero.logError(e);
Zotero.debug("FileChangeWatcher: " + backendName + " init failed -- "
+ "falling back to legacy scanning");
}
},
/**
* Get the set of item keys whose storage files have changed since the last call to this method.
*
* @return {Set|null} Set of 8-char item keys, or null to signal that the caller should fall
* back to a full scan
*/
getChangedItemKeys() {
if (!this.available) {
return null;
}
try {
switch (this._backend) {
case 'fsevents':
return this._getChangedItemKeysFSEvents();
case 'rdcw':
return this._getChangedItemKeysRDCW();
case 'inotify':
return this._getChangedItemKeysInotify();
}
}
catch (e) {
Zotero.logError(e);
Zotero.debug("FileChangeWatcher: getChangedItemKeys() failed -- signaling fallback");
}
return null;
},
close() {
switch (this._backend) {
case 'fsevents':
this._closeFSEvents();
break;
case 'rdcw':
this._closeRDCW();
break;
case 'inotify':
this._closeInotify();
break;
}
this._backend = null;
this.available = false;
},
//
// Common helpers
//
/**
* For live watcher backends (RDCW and inotify), check whether we should return null to
* trigger a full scan (first call or periodic refresh).
*
* @return {boolean} true if getChangedItemKeys should return null
*/
_liveWatcherNeedsFullScan() {
if (this._liveWatcherFirstCall) {
this._liveWatcherFirstCall = false;
this._liveWatcherLastFullScanTime = Date.now();
return true;
}
if (Date.now() - this._liveWatcherLastFullScanTime
> this._MAX_WATCHER_AGE) {
this._liveWatcherLastFullScanTime = Date.now();
return true;
}
return false;
},
/**
* Log changed key count or "no changes" and return the Set.
*
* @param {Set} keys
* @return {Set}
*/
_returnKeys(keys) {
if (keys.size > 0) {
Zotero.debug("FileChangeWatcher: " + keys.size + " changed key(s): "
+ [...keys].join(", "));
}
else {
Zotero.debug("FileChangeWatcher: No changes detected");
}
return keys;
},
};

292
chrome/content/zotero/xpcom/storage/fileChangeWatcher_fsevents.js Normal file
View file

@ -0,0 +1,292 @@
/* eslint-disable */
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2026 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* FSEvents backend for FileChangeWatcher (macOS)
*
* Uses the macOS FSEvents persistent event journal to detect which storage files have changed
* since the last sync. The event ID is saved to a pref so changes that occur while Zotero is
* closed are still detected.
*/
Object.assign(Zotero.Sync.Storage.FileChangeWatcher, {
_ctypes: null,
_objcLib: null,
_coreServicesLib: null,
_cfLib: null,
// Cached ctypes declarations
_FSEventsGetCurrentEventId: null,
_FSEventStreamCreate: null,
_FSEventStreamScheduleWithRunLoop: null,
_FSEventStreamStart: null,
_FSEventStreamFlushSync: null,
_FSEventStreamStop: null,
_FSEventStreamInvalidate: null,
_FSEventStreamRelease: null,
_CFRunLoopGetCurrent: null,
_FSEventStreamCallbackType: null,
// ObjC bridge helpers
_getClass: null,
_regSel: null,
_msg: null,
_msg_id: null,
_msg_ptr: null,
_initFSEvents() {
let { ctypes } = ChromeUtils.importESModule(
"resource://gre/modules/ctypes.sys.mjs"
);
this._ctypes = ctypes;
let id = ctypes.voidptr_t;
let SEL = ctypes.voidptr_t;
this._objcLib = ctypes.open("/usr/lib/libobjc.dylib");
this._coreServicesLib = ctypes.open(
"/System/Library/Frameworks/CoreServices.framework"
+ "/CoreServices"
);
this._cfLib = ctypes.open(
"/System/Library/Frameworks/CoreFoundation.framework"
+ "/CoreFoundation"
);
// ObjC bridge -- just enough for NSString/NSArray creation
this._getClass = this._objcLib.declare(
"objc_getClass", ctypes.default_abi, id, ctypes.char.ptr
);
this._regSel = this._objcLib.declare(
"sel_registerName", ctypes.default_abi, SEL, ctypes.char.ptr
);
this._msg = this._objcLib.declare(
"objc_msgSend", ctypes.default_abi, id, id, SEL
);
this._msg_id = this._objcLib.declare(
"objc_msgSend", ctypes.default_abi, id, id, SEL, id
);
this._msg_ptr = this._objcLib.declare(
"objc_msgSend", ctypes.default_abi, id, id, SEL,
ctypes.char.ptr
);
// FSEvents functions
this._FSEventsGetCurrentEventId = this._coreServicesLib.declare(
"FSEventsGetCurrentEventId",
ctypes.default_abi, ctypes.uint64_t
);
this._FSEventStreamCallbackType = ctypes.FunctionType(
ctypes.default_abi, ctypes.void_t,
[
ctypes.voidptr_t, ctypes.voidptr_t,
ctypes.size_t, ctypes.voidptr_t,
ctypes.voidptr_t, ctypes.voidptr_t,
]
);
this._FSEventStreamCreate = this._coreServicesLib.declare(
"FSEventStreamCreate", ctypes.default_abi,
ctypes.voidptr_t,
ctypes.voidptr_t, this._FSEventStreamCallbackType.ptr,
ctypes.voidptr_t, ctypes.voidptr_t,
ctypes.uint64_t, ctypes.double, ctypes.uint32_t
);
this._FSEventStreamScheduleWithRunLoop
= this._coreServicesLib.declare(
"FSEventStreamScheduleWithRunLoop",
ctypes.default_abi, ctypes.void_t,
ctypes.voidptr_t, ctypes.voidptr_t, ctypes.voidptr_t
);
this._FSEventStreamStart = this._coreServicesLib.declare(
"FSEventStreamStart", ctypes.default_abi,
ctypes.bool, ctypes.voidptr_t
);
this._FSEventStreamFlushSync = this._coreServicesLib.declare(
"FSEventStreamFlushSync", ctypes.default_abi,
ctypes.void_t, ctypes.voidptr_t
);
this._FSEventStreamStop = this._coreServicesLib.declare(
"FSEventStreamStop", ctypes.default_abi,
ctypes.void_t, ctypes.voidptr_t
);
this._FSEventStreamInvalidate = this._coreServicesLib.declare(
"FSEventStreamInvalidate", ctypes.default_abi,
ctypes.void_t, ctypes.voidptr_t
);
this._FSEventStreamRelease = this._coreServicesLib.declare(
"FSEventStreamRelease", ctypes.default_abi,
ctypes.void_t, ctypes.voidptr_t
);
this._CFRunLoopGetCurrent = this._cfLib.declare(
"CFRunLoopGetCurrent", ctypes.default_abi,
ctypes.voidptr_t
);
// Check for data directory change since last run
let savedPath = Zotero.Prefs.get(
"sync.storage.watcher.fsEventsStoragePath"
);
if (savedPath && savedPath !== this._storageRoot) {
Zotero.debug("FileChangeWatcher: Storage path changed -- clearing saved event ID");
Zotero.Prefs.clear("sync.storage.watcher.fsEventsEventID");
}
Zotero.Prefs.set("sync.storage.watcher.fsEventsStoragePath", this._storageRoot);
},
_cls(name) {
return this._getClass(name);
},
_sel(name) {
return this._regSel(name);
},
_nsStr(str) {
return this._msg_ptr(
this._cls("NSString"),
this._sel("stringWithUTF8String:"),
str
);
},
_getChangedItemKeysFSEvents() {
let ctypes = this._ctypes;
// If no saved event ID, record baseline and signal full scan
let savedIdStr = Zotero.Prefs.get(
"sync.storage.watcher.fsEventsEventID"
);
if (!savedIdStr) {
let currentId = this._FSEventsGetCurrentEventId();
Zotero.Prefs.set(
"sync.storage.watcher.fsEventsEventID", currentId.toString()
);
Zotero.debug("FileChangeWatcher: No saved event ID -- recorded baseline "
+ currentId + ", signaling full scan");
return null;
}
let sinceEventId = ctypes.UInt64(savedIdStr);
let changedKeys = new Set();
let storageRoot = this._storageRoot;
let keyPattern = this._keyPattern;
let callbackFn = this._FSEventStreamCallbackType.ptr(function (
_streamRef, _info, numEvents, eventPaths,
_eventFlags, _eventIds
) {
let n = Number(numEvents);
let StringArray = ctypes.ArrayType(ctypes.char.ptr, n);
let paths = ctypes.cast(
eventPaths, StringArray.ptr
).contents;
for (let i = 0; i < n; i++) {
let p = paths[i].readString();
if (!p.startsWith(storageRoot)) continue;
let relative = p.substring(storageRoot.length);
let slashIdx = relative.indexOf("/");
let key = slashIdx > 0
? relative.substring(0, slashIdx)
: relative;
if (keyPattern.test(key)) {
changedKeys.add(key);
}
}
});
let pathNS = this._nsStr(this._storageRoot);
let pathsArray = this._msg_id(
this._cls("NSArray"),
this._sel("arrayWithObject:"),
pathNS
);
// kFSEventStreamCreateFlagNoDefer |
// kFSEventStreamCreateFlagFileEvents
let flags = 0x02 | 0x10;
let stream = this._FSEventStreamCreate(
null, callbackFn, null, pathsArray,
sinceEventId, 0.0, flags
);
if (stream.isNull()) {
Zotero.debug("FileChangeWatcher: FSEventStreamCreate returned null");
return null;
}
try {
let runLoop = this._CFRunLoopGetCurrent();
let runLoopMode = this._nsStr("kCFRunLoopDefaultMode");
this._FSEventStreamScheduleWithRunLoop(
stream, runLoop, runLoopMode
);
let started = this._FSEventStreamStart(stream);
if (!started) {
Zotero.debug("FileChangeWatcher: FSEventStreamStart failed");
return null;
}
this._FSEventStreamFlushSync(stream);
this._FSEventStreamStop(stream);
}
finally {
this._FSEventStreamInvalidate(stream);
this._FSEventStreamRelease(stream);
}
let newEventId = this._FSEventsGetCurrentEventId();
Zotero.Prefs.set(
"sync.storage.watcher.fsEventsEventID", newEventId.toString()
);
return this._returnKeys(changedKeys);
},
_closeFSEvents() {
if (this._objcLib) {
this._objcLib.close();
this._objcLib = null;
}
if (this._coreServicesLib) {
this._coreServicesLib.close();
this._coreServicesLib = null;
}
if (this._cfLib) {
this._cfLib.close();
this._cfLib = null;
}
},
});

305
chrome/content/zotero/xpcom/storage/fileChangeWatcher_inotify.js Normal file
View file

@ -0,0 +1,305 @@
/* eslint-disable */
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2026 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* inotify backend for FileChangeWatcher (Linux)
*
* Like the Windows RDCW backend, inotify is a live watcher -- it only captures events from the
* moment watches are set up. The first getChangedItemKeys() call returns null to trigger a full
* scan. Periodic full scans every 3 hours ensure no changes are missed permanently.
*
* Note: errno values are read from ctypes.errno, which Mozilla ctypes captures immediately after
* each default_abi call. Using an explicit __errno_location() call would be unreliable because
* intervening ctypes machinery can reset errno.
*/
Object.assign(Zotero.Sync.Storage.FileChangeWatcher, {
_libc: null,
_inotifyFd: -1,
_inotifyRootWd: -1,
_inotifyWdToKey: null, // Map<int, string>
_inotifyAccumulatedKeys: null, // Set<string>
_inotifySubdirsSetUp: false,
// ctypes function declarations for inotify
_inotifyInit1: null,
_inotifyAddWatch: null,
_readFn: null,
_closeFn: null,
// inotify constants
_IN_CLOSE_WRITE: 0x00000008,
_IN_CREATE: 0x00000100,
_IN_DELETE: 0x00000200,
_IN_MOVED_TO: 0x00000080,
_IN_ISDIR: 0x40000000,
_IN_Q_OVERFLOW: 0x00004000,
_IN_NONBLOCK: 0x00000800,
_IN_CLOEXEC: 0x00080000,
_EAGAIN: 11,
_EINTR: 4,
_ENOSPC: 28,
_INOTIFY_BUF_SIZE: 8192,
_inotifyBuf: null,
_initInotify() {
let { ctypes } = ChromeUtils.importESModule(
"resource://gre/modules/ctypes.sys.mjs"
);
this._ctypes = ctypes;
this._libc = ctypes.open("libc.so.6");
this._inotifyInit1 = this._libc.declare(
"inotify_init1", ctypes.default_abi,
ctypes.int, ctypes.int
);
this._inotifyAddWatch = this._libc.declare(
"inotify_add_watch", ctypes.default_abi,
ctypes.int,
ctypes.int, ctypes.char.ptr, ctypes.uint32_t
);
this._readFn = this._libc.declare(
"read", ctypes.default_abi, ctypes.ssize_t,
ctypes.int, ctypes.uint8_t.ptr, ctypes.size_t
);
this._closeFn = this._libc.declare(
"close", ctypes.default_abi,
ctypes.int, ctypes.int
);
this._inotifyBuf = new (ctypes.ArrayType(
ctypes.uint8_t, this._INOTIFY_BUF_SIZE
))();
// Create inotify instance (non-blocking, close-on-exec)
let fd = this._inotifyInit1(
this._IN_NONBLOCK | this._IN_CLOEXEC
);
if (fd < 0) {
throw new Error("inotify_init1 failed (errno " + ctypes.errno + ")");
}
this._inotifyFd = fd;
// Watch storage root for new item directories
let rootWd = this._inotifyAddWatch(
fd, this._storageRoot,
this._IN_CREATE | this._IN_MOVED_TO
);
if (rootWd < 0) {
let errno = ctypes.errno;
this._closeFn(fd);
this._inotifyFd = -1;
throw new Error("inotify_add_watch on storage root failed (errno "
+ errno + ")");
}
this._inotifyRootWd = rootWd;
this._inotifyWdToKey = new Map();
this._inotifyAccumulatedKeys = new Set();
this._inotifySubdirsSetUp = false;
this._liveWatcherFirstCall = true;
},
/**
* Add inotify watches for all existing item subdirectories. Called lazily on first
* getChangedItemKeys() to avoid slowing down startup.
*/
_inotifySetupSubdirs() {
let ctypes = this._ctypes;
let dir = Zotero.File.pathToFile(this._storageRoot.slice(0, -1));
let count = 0;
let entries = dir.directoryEntries;
while (entries.hasMoreElements()) {
let entry = entries.getNext().QueryInterface(Ci.nsIFile);
if (!entry.isDirectory()) continue;
let name = entry.leafName;
if (!this._keyPattern.test(name)) continue;
let wd = this._inotifyAddWatch(
this._inotifyFd, entry.path,
this._IN_CLOSE_WRITE | this._IN_MOVED_TO
| this._IN_CREATE | this._IN_DELETE
);
if (wd < 0) {
if (ctypes.errno === this._ENOSPC) {
Zotero.debug("FileChangeWatcher: inotify watch limit reached after "
+ count + " directories -- some changes may be missed");
break;
}
continue;
}
this._inotifyWdToKey.set(wd, name);
count++;
}
this._inotifySubdirsSetUp = true;
Zotero.debug("FileChangeWatcher: Added inotify watches for " + count
+ " storage directories");
},
/**
* Non-blocking drain of all pending inotify events into _inotifyAccumulatedKeys.
*
* @return {boolean} false if an overflow was detected
*/
_inotifyDrainEvents() {
let ctypes = this._ctypes;
let headerSize = 16; // struct inotify_event fixed part
while (true) {
let bytesRead = this._readFn(
this._inotifyFd,
this._inotifyBuf.addressOfElement(0),
this._INOTIFY_BUF_SIZE
);
if (bytesRead <= 0) {
let errno = ctypes.errno;
if (errno === this._EAGAIN
|| errno === this._EINTR) {
break;
}
Zotero.debug("FileChangeWatcher: inotify read error (errno " + errno + ")");
break;
}
let offset = 0;
while (offset + headerSize <= bytesRead) {
let wdPtr = ctypes.cast(
this._inotifyBuf.addressOfElement(offset),
ctypes.int32_t.ptr
);
let maskPtr = ctypes.cast(
this._inotifyBuf.addressOfElement(offset + 4),
ctypes.uint32_t.ptr
);
let lenPtr = ctypes.cast(
this._inotifyBuf.addressOfElement(
offset + 12
),
ctypes.uint32_t.ptr
);
let wd = wdPtr.contents;
let mask = maskPtr.contents;
let nameLen = lenPtr.contents;
if (mask & this._IN_Q_OVERFLOW) {
Zotero.debug("FileChangeWatcher: inotify queue overflow");
return false;
}
let name = "";
if (nameLen > 0) {
let namePtr = ctypes.cast(
this._inotifyBuf.addressOfElement(
offset + headerSize
),
ctypes.char.ptr
);
name = namePtr.readString();
}
if (wd === this._inotifyRootWd) {
if ((mask & (this._IN_CREATE
| this._IN_MOVED_TO))
&& (mask & this._IN_ISDIR)
&& this._keyPattern.test(name)) {
this._inotifyAccumulatedKeys.add(name);
let subPath = this._storageRoot + name;
let subWd = this._inotifyAddWatch(
this._inotifyFd, subPath,
this._IN_CLOSE_WRITE
| this._IN_MOVED_TO
| this._IN_CREATE
| this._IN_DELETE
);
if (subWd >= 0) {
this._inotifyWdToKey.set(
subWd, name
);
}
}
}
else {
let key = this._inotifyWdToKey.get(wd);
if (key) {
this._inotifyAccumulatedKeys.add(key);
}
}
offset += headerSize + nameLen;
}
}
return true;
},
_getChangedItemKeysInotify() {
if (!this._inotifySubdirsSetUp) {
this._inotifySetupSubdirs();
}
let ok = this._inotifyDrainEvents();
// First call or periodic refresh -- signal full scan
if (this._liveWatcherNeedsFullScan()) {
Zotero.debug("FileChangeWatcher: Live watcher signaling full scan"
+ " (first call or periodic refresh)");
this._inotifyAccumulatedKeys.clear();
return null;
}
// Overflow -- signal full scan
if (!ok) {
this._inotifyAccumulatedKeys.clear();
this._liveWatcherLastFullScanTime = Date.now();
return null;
}
let keys = this._inotifyAccumulatedKeys;
this._inotifyAccumulatedKeys = new Set();
return this._returnKeys(keys);
},
_closeInotify() {
if (this._inotifyFd >= 0) {
this._closeFn(this._inotifyFd);
this._inotifyFd = -1;
}
if (this._libc) {
this._libc.close();
this._libc = null;
}
this._inotifyWdToKey = null;
this._inotifyAccumulatedKeys = null;
},
});

409
chrome/content/zotero/xpcom/storage/fileChangeWatcher_rdcw.js Normal file
View file

@ -0,0 +1,409 @@
/* eslint-disable */
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2026 Corporation for Digital Scholarship
Vienna, Virginia, USA
https://www.zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* ReadDirectoryChangesW backend for FileChangeWatcher (Windows)
*
* Uses overlapped I/O with ReadDirectoryChangesW to watch the storage directory recursively.
* Unlike the previous USN Change Journal backend, this works without admin privileges -- it only
* needs read access to the storage directory itself.
*
* This is a live watcher (like inotify on Linux): the first getChangedItemKeys() call returns null
* to trigger a full scan, then accumulates changes between calls. Periodic full scans every 3 hours
* via _liveWatcherNeedsFullScan() ensure no changes are missed permanently.
*
* Approach -- overlapped I/O polling:
* 1. Init: Open storage directory with FILE_FLAG_OVERLAPPED, create an event handle, issue the
* first ReadDirectoryChangesW call (returns immediately, queues the request)
* 2. Poll (getChangedItemKeys): Call GetOverlappedResult(bWait=FALSE) -- non-blocking check.
* If data ready, parse FILE_NOTIFY_INFORMATION records, extract item keys, re-arm.
* If not ready, return accumulated keys (may be empty).
* 3. Close: Close directory handle (auto-cancels pending I/O), close event handle, close kernel32
*
* Note: Win32 error codes are read from ctypes.winLastError, which Mozilla ctypes captures
* immediately after each winapi_abi call. Using an explicit GetLastError() declaration would be
* unreliable because intervening ctypes machinery can reset the thread error.
*/
Object.assign(Zotero.Sync.Storage.FileChangeWatcher, {
_kernel32: null,
_ctypes: null,
_dirHandle: null,
_eventHandle: null,
_overlapped: null,
_rdcwBuf: null,
_rdcwAccumulatedKeys: null,
_rdcwPending: false,
// ctypes function declarations
_CreateFileW: null,
_CloseHandle: null,
_ReadDirectoryChangesW: null,
_GetOverlappedResult: null,
_CreateEventW: null,
// OVERLAPPED struct type
_OVERLAPPED: null,
// Windows constants
_FILE_LIST_DIRECTORY: 0x0001,
_FILE_SHARE_READ: 0x00000001,
_FILE_SHARE_WRITE: 0x00000002,
_FILE_SHARE_DELETE: 0x00000004,
_OPEN_EXISTING: 3,
_FILE_FLAG_BACKUP_SEMANTICS: 0x02000000,
_FILE_FLAG_OVERLAPPED: 0x40000000,
// Notify filter -- watch for file/dir name changes, size changes, and last-write changes
_RDCW_NOTIFY_FILTER: 0x00000001 // FILE_NOTIFY_CHANGE_FILE_NAME
| 0x00000002 // FILE_NOTIFY_CHANGE_DIR_NAME
| 0x00000008 // FILE_NOTIFY_CHANGE_SIZE
| 0x00000010, // FILE_NOTIFY_CHANGE_LAST_WRITE
_RDCW_BUF_SIZE: 65536,
// Expected overlapped I/O error codes
_ERROR_IO_INCOMPLETE: 996,
_ERROR_IO_PENDING: 997,
_ERROR_NOTIFY_ENUM_DIR: 1022,
_initRDCW() {
let { ctypes } = ChromeUtils.importESModule(
"resource://gre/modules/ctypes.sys.mjs"
);
this._ctypes = ctypes;
this._kernel32 = ctypes.open("kernel32.dll");
// ---- Struct types ----
this._OVERLAPPED = new ctypes.StructType("OVERLAPPED", [
{ "Internal": ctypes.uintptr_t },
{ "InternalHigh": ctypes.uintptr_t },
{ "OffsetLow": ctypes.uint32_t },
{ "OffsetHigh": ctypes.uint32_t },
{ "hEvent": ctypes.voidptr_t },
]);
// ---- Function declarations ----
this._CreateFileW = this._kernel32.declare(
"CreateFileW", ctypes.winapi_abi,
ctypes.voidptr_t, // HANDLE
ctypes.char16_t.ptr, // lpFileName
ctypes.uint32_t, // dwDesiredAccess
ctypes.uint32_t, // dwShareMode
ctypes.voidptr_t, // lpSecurityAttributes
ctypes.uint32_t, // dwCreationDisposition
ctypes.uint32_t, // dwFlagsAndAttributes
ctypes.voidptr_t // hTemplateFile
);
this._CloseHandle = this._kernel32.declare(
"CloseHandle", ctypes.winapi_abi,
ctypes.bool, ctypes.voidptr_t
);
this._ReadDirectoryChangesW = this._kernel32.declare(
"ReadDirectoryChangesW", ctypes.winapi_abi,
ctypes.bool,
ctypes.voidptr_t, // hDirectory
ctypes.voidptr_t, // lpBuffer
ctypes.uint32_t, // nBufferLength
ctypes.bool, // bWatchSubtree
ctypes.uint32_t, // dwNotifyFilter
ctypes.uint32_t.ptr, // lpBytesReturned
ctypes.voidptr_t, // lpOverlapped (OVERLAPPED*)
ctypes.voidptr_t // lpCompletionRoutine
);
this._GetOverlappedResult = this._kernel32.declare(
"GetOverlappedResult", ctypes.winapi_abi,
ctypes.bool,
ctypes.voidptr_t, // hFile
ctypes.voidptr_t, // lpOverlapped (OVERLAPPED*)
ctypes.uint32_t.ptr, // lpNumberOfBytesTransferred
ctypes.bool // bWait
);
this._CreateEventW = this._kernel32.declare(
"CreateEventW", ctypes.winapi_abi,
ctypes.voidptr_t, // HANDLE return
ctypes.voidptr_t, // lpEventAttributes
ctypes.bool, // bManualReset
ctypes.bool, // bInitialState
ctypes.voidptr_t // lpName
);
// ---- Open storage directory ----
let dirPath = this._storageRoot.slice(0, -1); // remove trailing backslash
let dirPathBuf = ctypes.char16_t.array()(dirPath);
let handle = this._CreateFileW(
dirPathBuf,
this._FILE_LIST_DIRECTORY,
this._FILE_SHARE_READ | this._FILE_SHARE_WRITE | this._FILE_SHARE_DELETE,
null,
this._OPEN_EXISTING,
this._FILE_FLAG_BACKUP_SEMANTICS | this._FILE_FLAG_OVERLAPPED,
null
);
// INVALID_HANDLE_VALUE is (HANDLE)-1 -- cast to intptr_t to check
if (ctypes.cast(handle, ctypes.intptr_t).value.toString() === "-1") {
throw new Error("Cannot open storage directory for watching (error "
+ ctypes.winLastError + ")");
}
this._dirHandle = handle;
// ---- Create event for overlapped I/O ----
let eventHandle = this._CreateEventW(null, true, false, null);
if (eventHandle.isNull()) {
this._CloseHandle(this._dirHandle);
this._dirHandle = null;
throw new Error("CreateEventW failed (error " + ctypes.winLastError + ")");
}
this._eventHandle = eventHandle;
// ---- Allocate buffer and overlapped struct ----
this._rdcwBuf = new (ctypes.ArrayType(
ctypes.uint8_t, this._RDCW_BUF_SIZE
))();
this._overlapped = new this._OVERLAPPED();
this._overlapped.Internal = 0;
this._overlapped.InternalHigh = 0;
this._overlapped.OffsetLow = 0;
this._overlapped.OffsetHigh = 0;
this._overlapped.hEvent = this._eventHandle;
this._rdcwAccumulatedKeys = new Set();
this._liveWatcherFirstCall = true;
// ---- Issue first ReadDirectoryChangesW call ----
this._rdcwArm();
this._rdcwPending = true;
},
/**
* Issue (or re-issue) an overlapped ReadDirectoryChangesW call.
* Throws on failure.
*/
_rdcwArm() {
let bytesReturned = new this._ctypes.uint32_t(0);
let ok = this._ReadDirectoryChangesW(
this._dirHandle,
this._rdcwBuf.address(),
this._RDCW_BUF_SIZE,
true, // bWatchSubtree -- recursive
this._RDCW_NOTIFY_FILTER,
bytesReturned.address(),
this._overlapped.address(),
null // no completion routine
);
if (!ok) {
let err = this._ctypes.winLastError;
// ERROR_IO_PENDING is expected for overlapped I/O
if (err !== this._ERROR_IO_PENDING) {
throw new Error("ReadDirectoryChangesW failed (error " + err + ")");
}
}
},
/**
* Parse FILE_NOTIFY_INFORMATION records from the buffer and extract item keys.
*
* FILE_NOTIFY_INFORMATION is a variable-length linked list:
* DWORD NextEntryOffset (0 = last entry)
* DWORD Action
* DWORD FileNameLength (in bytes)
* WCHAR FileName[] (UTF-16LE, not null-terminated, relative path with backslashes)
*
* We extract the first path component before the first backslash as the item key.
*
* @param {number} bytesTransferred
*/
_rdcwParseNotifications(bytesTransferred) {
let ctypes = this._ctypes;
let offset = 0;
while (offset < bytesTransferred) {
// Read NextEntryOffset (uint32_t at offset+0)
let nextEntryPtr = ctypes.cast(
this._rdcwBuf.addressOfElement(offset),
ctypes.uint32_t.ptr
);
let nextEntryOffset = nextEntryPtr.contents;
// Read FileNameLength (uint32_t at offset+8, in bytes)
let fileNameLenPtr = ctypes.cast(
this._rdcwBuf.addressOfElement(offset + 8),
ctypes.uint32_t.ptr
);
let fileNameLength = fileNameLenPtr.contents;
// Read FileName (UTF-16LE at offset+12)
let nameChars = fileNameLength / 2;
if (nameChars > 0) {
// Extract the first path component (before first backslash)
let key = "";
let nameStart = offset + 12;
for (let i = 0; i < nameChars; i++) {
let byteOff = nameStart + i * 2;
let lo = this._rdcwBuf[byteOff];
let hi = this._rdcwBuf[byteOff + 1];
let ch = lo | (hi << 8);
if (ch === 0x5C) { // backslash
break;
}
key += String.fromCharCode(ch);
}
if (this._keyPattern.test(key)) {
this._rdcwAccumulatedKeys.add(key);
}
}
if (nextEntryOffset === 0) {
break;
}
offset += nextEntryOffset;
}
},
_getChangedItemKeysRDCW() {
let ctypes = this._ctypes;
// Drain all available notifications
if (this._rdcwPending) {
let maxDrains = 100;
let drains = 0;
while (drains++ < maxDrains) {
let bytesTransferred = new ctypes.uint32_t(0);
let ok = this._GetOverlappedResult(
this._dirHandle,
this._overlapped.address(),
bytesTransferred.address(),
false // bWait=FALSE -- non-blocking
);
if (!ok) {
let err = ctypes.winLastError;
if (err === this._ERROR_IO_INCOMPLETE) {
// No data ready yet -- that's fine
break;
}
if (err === this._ERROR_NOTIFY_ENUM_DIR) {
// Buffer overflow -- signal full scan
Zotero.debug("FileChangeWatcher: RDCW buffer overflow"
+ " -- signaling full scan");
this._rdcwAccumulatedKeys.clear();
this._liveWatcherLastFullScanTime = Date.now();
// Re-arm for future notifications
try {
this._rdcwArm();
}
catch (e) {
Zotero.logError(e);
this._rdcwPending = false;
}
return null;
}
// Other error -- log and signal full scan
Zotero.debug("FileChangeWatcher: GetOverlappedResult failed"
+ " (error " + err + ")");
this._rdcwPending = false;
return null;
}
// Data ready
let bytes = bytesTransferred.value;
if (bytes === 0) {
// Zero bytes means overflow -- signal full scan
Zotero.debug("FileChangeWatcher: RDCW returned 0 bytes"
+ " -- signaling full scan");
this._rdcwAccumulatedKeys.clear();
this._liveWatcherLastFullScanTime = Date.now();
try {
this._rdcwArm();
}
catch (e) {
Zotero.logError(e);
this._rdcwPending = false;
}
return null;
}
this._rdcwParseNotifications(bytes);
// Re-arm and check for more
try {
this._rdcwArm();
}
catch (e) {
Zotero.logError(e);
this._rdcwPending = false;
break;
}
}
}
// First call or periodic refresh -- signal full scan
if (this._liveWatcherNeedsFullScan()) {
Zotero.debug("FileChangeWatcher: Live watcher signaling full scan"
+ " (first call or periodic refresh)");
this._rdcwAccumulatedKeys.clear();
return null;
}
let keys = this._rdcwAccumulatedKeys;
this._rdcwAccumulatedKeys = new Set();
return this._returnKeys(keys);
},
_closeRDCW() {
// Closing the directory handle auto-cancels pending overlapped I/O
if (this._dirHandle && !this._dirHandle.isNull()) {
this._CloseHandle(this._dirHandle);
this._dirHandle = null;
}
if (this._eventHandle && !this._eventHandle.isNull()) {
this._CloseHandle(this._eventHandle);
this._eventHandle = null;
}
if (this._kernel32) {
this._kernel32.close();
this._kernel32 = null;
}
this._overlapped = null;
this._rdcwBuf = null;
this._rdcwAccumulatedKeys = null;
this._rdcwPending = false;
},
});

34
chrome/content/zotero/xpcom/storage/storageEngine.js
View file

@ -146,6 +146,40 @@ Zotero.Sync.Storage.Engine.prototype.start = async function () {
Zotero.debug("No file editing access -- skipping file modification check for "
+ this.library.name);
}
// Use file change watcher to check only files that actually changed on disk
//
// Note: File-sync downloads also generate filesystem events, so recently downloaded files will
// appear here on the next sync cycle, but checkForUpdatedFiles() will see that the mtime/hash
// match the synced values and skip them.
else if (Zotero.Sync.Storage.FileChangeWatcher.available) {
let changedKeys = Zotero.Sync.Storage.FileChangeWatcher.getChangedItemKeys();
if (changedKeys) {
if (changedKeys.size > 0) {
let keysArray = Array.from(changedKeys);
let itemIDs = [];
// Batch to avoid hitting SQLite parameter limit
await Zotero.Utilities.Internal.forEachChunkAsync(
keysArray, 500, async function (chunk) {
let ids = await Zotero.DB.columnQueryAsync(
"SELECT itemID FROM items WHERE libraryID=? AND key IN ("
+ chunk.map(() => '?').join(',') + ")",
[libraryID, ...chunk]
);
itemIDs.push(...ids);
}
);
if (itemIDs.length) {
await this.local.checkForUpdatedFiles(libraryID, itemIDs);
}
}
// else: no changes detected, skip scan entirely
}
else {
// Watcher returned null (first run or error) -- full scan
this.local.lastFullFileCheck[libraryID] = new Date().getTime();
await this.local.checkForUpdatedFiles(libraryID);
}
}
// If this is a background sync, it's not the first sync of the session, the library has had
// at least one full check this session, and it's been less than maxCheckAge since the last
// full check of this library, check only files that were previously modified or opened

1
chrome/content/zotero/xpcom/zotero.js
View file

@ -710,6 +710,7 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte
await Zotero.Sync.Data.Local.init();
await Zotero.Sync.Data.Utilities.init();
Zotero.Sync.Storage.Local.init();
Zotero.Sync.Storage.FileChangeWatcher.init();
Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
Zotero.Sync.EventListeners.init();
Zotero.Streamer = new Zotero.Streamer_Module;

22
chrome/content/zotero/zotero.mjs
View file

@ -147,6 +147,7 @@ const xpcomFilesLocal = [
'storage',
'storage/storageEngine',
'storage/storageLocal',
'storage/fileChangeWatcher',
'storage/storageRequest',
'storage/storageResult',
'storage/storageUtilities',
@ -274,6 +275,27 @@ function makeZoteroContext() {
}
}
// Load platform-specific FileChangeWatcher backend
{
let os = Services.appinfo.OS;
let backendFile = os == "Darwin" ? "storage/fileChangeWatcher_fsevents"
: os == "WINNT" ? "storage/fileChangeWatcher_rdcw"
: os == "Linux" ? "storage/fileChangeWatcher_inotify"
: null;
if (backendFile) {
try {
subscriptLoader.loadSubScript(
"chrome://zotero/content/xpcom/" + backendFile + ".js", zContext, "utf-8"
);
}
catch (e) {
dump("Error loading " + backendFile + ".js\n\n");
dump(e + "\n\n");
Components.utils.reportError("Error loading " + backendFile + ".js");
}
}
}
// Load RDF files into Zotero.RDF.AJAW namespace (easier than modifying all of the references)
const rdfXpcomFiles = [
'rdf/init',

504
test/tests/fileChangeWatcherTest.js Normal file
View file

@ -0,0 +1,504 @@
describe("Zotero.Sync.Storage.FileChangeWatcher", function () {
let watcher = Zotero.Sync.Storage.FileChangeWatcher;
// Pref keys used by the watcher
let PREF_FSEVENTS_EVENT_ID = "sync.storage.watcher.fsEventsEventID";
let PREF_FSEVENTS_STORAGE_PATH = "sync.storage.watcher.fsEventsStoragePath";
function clearWatcherPrefs() {
Zotero.Prefs.clear(PREF_FSEVENTS_EVENT_ID);
Zotero.Prefs.clear(PREF_FSEVENTS_STORAGE_PATH);
}
// The Set returned by getChangedItemKeys() is created in the XPCOM
// context, so instanceof Set doesn't work across contexts. Use
// duck-typing instead.
function assertIsSet(val, msg) {
assert.isNotNull(val, msg);
assert.isFunction(val.has, msg);
assert.isNumber(val.size, msg);
}
describe("FSEvents backend (macOS)", function () {
before(async function () {
if (!Zotero.isMac) {
this.skip();
}
// Ensure the storage directory exists (it may not yet in
// the test environment)
let storageDir = PathUtils.join(
Zotero.DataDirectory.dir, "storage"
);
await IOUtils.makeDirectory(storageDir, {
ignoreExisting: true
});
// Close any watcher initialized at startup
watcher.close();
clearWatcherPrefs();
});
afterEach(function () {
watcher.close();
clearWatcherPrefs();
});
after(function () {
// Re-init the watcher so it's available for normal operation
// after the test suite
watcher.init();
});
it("should initialize successfully", function () {
watcher.init();
assert.isTrue(watcher.available);
assert.equal(watcher._backend, 'fsevents');
assert.ok(watcher._storageRoot, "Storage root should be set");
});
it("should return null on first call with no saved event ID", function () {
watcher.init();
let result = watcher.getChangedItemKeys();
assert.isNull(result, "First call should return null");
// Should have saved a baseline event ID
let savedId = Zotero.Prefs.get(PREF_FSEVENTS_EVENT_ID);
assert.isString(savedId);
assert.notEqual(savedId, "0");
});
it("should return an empty set when no files have changed", function () {
watcher.init();
// Establish baseline
watcher.getChangedItemKeys();
// Second call -- no changes
let result = watcher.getChangedItemKeys();
assertIsSet(result);
assert.equal(result.size, 0);
});
it("should detect a modified attachment file", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
try {
watcher.init();
// Establish baseline
watcher.getChangedItemKeys();
// Modify the file
let path = await item.getFilePathAsync();
await Zotero.File.putContentsAsync(
path, Zotero.Utilities.randomString()
);
// Wait for FSEvents to register
await Zotero.Promise.delay(1000);
let result = watcher.getChangedItemKeys();
assertIsSet(result);
assert.isTrue(
result.has(item.key),
"Should contain the key of the modified item"
);
}
finally {
await item.eraseTx();
}
});
it("should detect changes across multiple items", async function () {
this.timeout(15000);
let item1 = await importFileAttachment('test.png');
let item2 = await importTextAttachment();
let item3 = await importFileAttachment('test.png');
try {
watcher.init();
// Wait for FSEvents from item creation to flush
// before establishing baseline
await Zotero.Promise.delay(1000);
watcher.getChangedItemKeys();
// Modify only items 1 and 3
let path1 = await item1.getFilePathAsync();
await Zotero.File.putContentsAsync(
path1, Zotero.Utilities.randomString()
);
let path3 = await item3.getFilePathAsync();
await Zotero.File.putContentsAsync(
path3, Zotero.Utilities.randomString()
);
await Zotero.Promise.delay(1000);
let result = watcher.getChangedItemKeys();
assertIsSet(result);
assert.isTrue(result.has(item1.key),
"Should contain key of first modified item");
assert.isFalse(result.has(item2.key),
"Should not contain key of unmodified item");
assert.isTrue(result.has(item3.key),
"Should contain key of second modified item");
}
finally {
await item1.eraseTx();
await item2.eraseTx();
await item3.eraseTx();
}
});
it("should not report the same changes twice", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
try {
watcher.init();
watcher.getChangedItemKeys();
// Modify file
let path = await item.getFilePathAsync();
await Zotero.File.putContentsAsync(
path, Zotero.Utilities.randomString()
);
await Zotero.Promise.delay(1000);
// First query picks up the change
let result1 = watcher.getChangedItemKeys();
assertIsSet(result1);
assert.isTrue(result1.has(item.key));
// Second query should be empty
let result2 = watcher.getChangedItemKeys();
assertIsSet(result2);
assert.equal(result2.size, 0,
"Should not report the same change twice");
}
finally {
await item.eraseTx();
}
});
it("should detect a new file added to a storage directory", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
try {
watcher.init();
watcher.getChangedItemKeys();
// Add a new file to the item's storage directory
let storageDir = Zotero.Attachments.getStorageDirectory(item).path;
let newFile = PathUtils.join(storageDir, "extra.txt");
await Zotero.File.putContentsAsync(newFile, "extra content");
await Zotero.Promise.delay(1000);
let result = watcher.getChangedItemKeys();
assertIsSet(result);
assert.isTrue(result.has(item.key),
"Should detect new file in storage directory");
}
finally {
await item.eraseTx();
}
});
it("should return null when storage path has changed", function () {
// Simulate a previous run with a different storage path
Zotero.Prefs.set(PREF_FSEVENTS_STORAGE_PATH, "/some/other/path/");
Zotero.Prefs.set(PREF_FSEVENTS_EVENT_ID, "12345");
watcher.init();
// The saved event ID should have been cleared
let result = watcher.getChangedItemKeys();
assert.isNull(result,
"Should return null after storage path change");
});
it("should persist event ID across close/init cycles", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
try {
watcher.init();
// Establish baseline
watcher.getChangedItemKeys();
let savedId = Zotero.Prefs.get(PREF_FSEVENTS_EVENT_ID);
assert.ok(savedId, "Event ID should be saved");
// Close and re-init (simulating restart)
watcher.close();
watcher.init();
// Should not return null -- saved event ID should be used
let result = watcher.getChangedItemKeys();
assertIsSet(result,
"Should return a Set (not null) after re-init "
+ "with saved event ID");
}
finally {
await item.eraseTx();
}
});
it("should detect changes that happened while watcher was closed", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
try {
watcher.init();
watcher.getChangedItemKeys();
// Close the watcher
watcher.close();
// Modify file while watcher is closed
let path = await item.getFilePathAsync();
await Zotero.File.putContentsAsync(
path, Zotero.Utilities.randomString()
);
await Zotero.Promise.delay(1000);
// Re-init and query -- FSEvents journal should have
// the change even though the watcher was closed
watcher.init();
let result = watcher.getChangedItemKeys();
assertIsSet(result);
assert.isTrue(result.has(item.key),
"Should detect changes that occurred while "
+ "watcher was closed");
}
finally {
await item.eraseTx();
}
});
it("should only report valid 8-char item keys", async function () {
this.timeout(15000);
watcher.init();
watcher.getChangedItemKeys();
// Create a non-standard directory name in the storage root
let storageRoot = watcher._storageRoot;
let invalidDir = PathUtils.join(storageRoot, "not-a-key");
await IOUtils.makeDirectory(invalidDir, { ignoreExisting: true });
let testFile = PathUtils.join(invalidDir, "test.txt");
await IOUtils.writeUTF8(testFile, "test");
await Zotero.Promise.delay(1000);
try {
let result = watcher.getChangedItemKeys();
assertIsSet(result);
// Should not contain the invalid directory name
for (let key of result) {
assert.match(key, /^[A-Z0-9]{8}$/,
"All returned keys should match the 8-char "
+ "pattern");
}
}
finally {
await IOUtils.remove(invalidDir, { recursive: true });
}
});
});
describe("integration with storageEngine", function () {
before(async function () {
if (!Zotero.isMac) {
this.skip();
}
let storageDir = PathUtils.join(
Zotero.DataDirectory.dir, "storage"
);
await IOUtils.makeDirectory(storageDir, {
ignoreExisting: true
});
watcher.close();
clearWatcherPrefs();
});
afterEach(function () {
watcher.close();
clearWatcherPrefs();
});
after(function () {
watcher.init();
});
it("should be checked before falling back to legacy scanning", async function () {
this.timeout(15000);
let item = await importFileAttachment('test.png');
let hash = await item.attachmentHash;
let mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
await OS.File.setDates(
(await item.getFilePathAsync()), null, mtime
);
// Mark as synced
item.attachmentSyncedModificationTime = mtime;
item.attachmentSyncedHash = hash;
item.attachmentSyncState = "in_sync";
await item.saveTx({ skipAll: true });
try {
watcher.init();
// Establish baseline
watcher.getChangedItemKeys();
// Modify the file
let path = await item.getFilePathAsync();
await OS.File.setDates(path);
await Zotero.File.putContentsAsync(
path, Zotero.Utilities.randomString()
);
await Zotero.Promise.delay(1000);
// Verify the watcher reports the change
let changedKeys = watcher.getChangedItemKeys();
assertIsSet(changedKeys);
assert.isTrue(changedKeys.has(item.key));
// Map keys to itemIDs as storageEngine does
let libraryID = Zotero.Libraries.userLibraryID;
let itemIDs = await Zotero.DB.columnQueryAsync(
"SELECT itemID FROM items WHERE libraryID=?"
+ " AND key IN ("
+ Array.from(changedKeys).map(() => '?')
.join(',')
+ ")",
[libraryID, ...changedKeys]
);
assert.include(itemIDs, item.id);
// checkForUpdatedFiles should flag it for upload
let changed = await Zotero.Sync.Storage.Local
.checkForUpdatedFiles(libraryID, itemIDs);
assert.isTrue(changed);
assert.equal(
item.attachmentSyncState,
Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD
);
}
finally {
await item.eraseTx();
}
});
});
describe("ReadDirectoryChangesW backend (Windows)", function () {
before(async function () {
if (!Zotero.isWin) {
this.skip();
}
let storageDir = PathUtils.join(
Zotero.DataDirectory.dir, "storage"
);
await IOUtils.makeDirectory(storageDir, {
ignoreExisting: true
});
watcher.close();
});
afterEach(function () {
watcher.close();
});
after(function () {
watcher.init();
});
it("should initialize successfully");
it("should return null on first call (no persistent journal)");
it("should return an empty set when no files have changed");
it("should detect a modified attachment file");
it("should detect changes across multiple items");
it("should not report the same changes twice");
it("should detect a new file added to a storage directory");
it("should re-arm overlapped I/O after draining notifications");
it("should return null on buffer overflow and re-arm");
it("should return null for periodic full scan after max age");
it("should only report valid 8-char item keys");
});
describe("inotify backend (Linux)", function () {
before(async function () {
if (!Zotero.isLinux) {
this.skip();
}
let storageDir = PathUtils.join(
Zotero.DataDirectory.dir, "storage"
);
await IOUtils.makeDirectory(storageDir, {
ignoreExisting: true
});
watcher.close();
clearWatcherPrefs();
});
afterEach(function () {
watcher.close();
clearWatcherPrefs();
});
after(function () {
watcher.init();
});
it("should initialize successfully");
it("should return null on first call (no persistent journal)");
it("should return an empty set when no files have changed");
it("should detect a modified attachment file");
it("should detect changes across multiple items");
it("should not report the same changes twice");
it("should detect a new file added to a storage directory");
it("should auto-watch newly created storage subdirectories");
it("should return null for periodic full scan after max age");
it("should only report valid 8-char item keys");
});
describe("fallback behavior", function () {
it("should return null from getChangedItemKeys() when unavailable", function () {
let savedAvailable = watcher.available;
let savedBackend = watcher._backend;
watcher.available = false;
watcher._backend = null;
try {
let result = watcher.getChangedItemKeys();
assert.isNull(result);
}
finally {
watcher.available = savedAvailable;
watcher._backend = savedBackend;
}
});
it("should set available to false after close()", function () {
if (!Zotero.isMac) {
this.skip();
}
watcher.init();
assert.isTrue(watcher.available);
watcher.close();
assert.isFalse(watcher.available);
assert.isNull(watcher._backend);
// Re-init for other tests
clearWatcherPrefs();
watcher.init();
});
});
});