diff --git a/chrome/content/zotero/xpcom/storage/fileChangeWatcher.js b/chrome/content/zotero/xpcom/storage/fileChangeWatcher.js new file mode 100644 index 0000000000..e4bdc7a8a1 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/fileChangeWatcher.js @@ -0,0 +1,190 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * File change watcher for storage sync + * + * Uses platform-native file change APIs to track which stored-file attachments have been modified, + * avoiding expensive full scans of all attachment files. + * + * Backends (loaded from separate files): + * macOS -- FSEvents (persistent event journal, survives restarts) + * Windows -- ReadDirectoryChangesW (live recursive directory watch) + * Linux -- inotify (live per-directory watches) + * + * On unsupported platforms or on error, falls back to the existing scan logic. + */ +Zotero.Sync.Storage.FileChangeWatcher = { + available: false, + _backend: null, + + // Cached storage root path (normalized, with trailing separator) + _storageRoot: null, + + // Key validation pattern + _keyPattern: /^[A-Z0-9]{8}$/, + + // For live watcher backends (RDCW and inotify), track whether the first call has happened + // (returns null to trigger a full scan) and enforce periodic full scans every + // _MAX_WATCHER_AGE ms + _liveWatcherFirstCall: true, + _liveWatcherLastFullScanTime: 0, + _MAX_WATCHER_AGE: 10800000, // 3 hours -- matches storageEngine.maxCheckAge + + init() { + try { + // Ensure the storage directory exists -- on a new installation it may not have + // been created yet, and the backends need it to set up their watches + let storageDir = Zotero.getStorageDirectory(); + storageDir.normalize(); + let sep = Zotero.isWin ? "\\" : "/"; + let path = storageDir.path; + if (!path.endsWith(sep)) { + path += sep; + } + this._storageRoot = path; + } + catch (e) { + Zotero.logError(e); + Zotero.debug("FileChangeWatcher: Could not resolve storage root"); + return; + } + + let initFn; + let backendName; + if (Zotero.isMac) { + initFn = '_initFSEvents'; + backendName = 'fsevents'; + } + else if (Zotero.isWin) { + initFn = '_initRDCW'; + backendName = 'rdcw'; + } + else if (Zotero.isLinux) { + initFn = '_initInotify'; + backendName = 'inotify'; + } + else { + Zotero.debug("FileChangeWatcher: No backend available for this platform"); + return; + } + + try { + this[initFn](); + this._backend = backendName; + this.available = true; + Zotero.debug(`FileChangeWatcher: ${backendName} backend initialized`); + } + catch (e) { + Zotero.logError(e); + Zotero.debug("FileChangeWatcher: " + backendName + " init failed -- " + + "falling back to legacy scanning"); + } + }, + + /** + * Get the set of item keys whose storage files have changed since the last call to this method. + * + * @return {Set|null} Set of 8-char item keys, or null to signal that the caller should fall + * back to a full scan + */ + getChangedItemKeys() { + if (!this.available) { + return null; + } + try { + switch (this._backend) { + case 'fsevents': + return this._getChangedItemKeysFSEvents(); + case 'rdcw': + return this._getChangedItemKeysRDCW(); + case 'inotify': + return this._getChangedItemKeysInotify(); + } + } + catch (e) { + Zotero.logError(e); + Zotero.debug("FileChangeWatcher: getChangedItemKeys() failed -- signaling fallback"); + } + return null; + }, + + close() { + switch (this._backend) { + case 'fsevents': + this._closeFSEvents(); + break; + case 'rdcw': + this._closeRDCW(); + break; + case 'inotify': + this._closeInotify(); + break; + } + this._backend = null; + this.available = false; + }, + + // + // Common helpers + // + + /** + * For live watcher backends (RDCW and inotify), check whether we should return null to + * trigger a full scan (first call or periodic refresh). + * + * @return {boolean} true if getChangedItemKeys should return null + */ + _liveWatcherNeedsFullScan() { + if (this._liveWatcherFirstCall) { + this._liveWatcherFirstCall = false; + this._liveWatcherLastFullScanTime = Date.now(); + return true; + } + if (Date.now() - this._liveWatcherLastFullScanTime + > this._MAX_WATCHER_AGE) { + this._liveWatcherLastFullScanTime = Date.now(); + return true; + } + return false; + }, + + /** + * Log changed key count or "no changes" and return the Set. + * + * @param {Set} keys + * @return {Set} + */ + _returnKeys(keys) { + if (keys.size > 0) { + Zotero.debug("FileChangeWatcher: " + keys.size + " changed key(s): " + + [...keys].join(", ")); + } + else { + Zotero.debug("FileChangeWatcher: No changes detected"); + } + return keys; + }, +}; diff --git a/chrome/content/zotero/xpcom/storage/fileChangeWatcher_fsevents.js b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_fsevents.js new file mode 100644 index 0000000000..a1052bfc84 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_fsevents.js @@ -0,0 +1,292 @@ +/* eslint-disable */ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * FSEvents backend for FileChangeWatcher (macOS) + * + * Uses the macOS FSEvents persistent event journal to detect which storage files have changed + * since the last sync. The event ID is saved to a pref so changes that occur while Zotero is + * closed are still detected. + */ +Object.assign(Zotero.Sync.Storage.FileChangeWatcher, { + _ctypes: null, + _objcLib: null, + _coreServicesLib: null, + _cfLib: null, + + // Cached ctypes declarations + _FSEventsGetCurrentEventId: null, + _FSEventStreamCreate: null, + _FSEventStreamScheduleWithRunLoop: null, + _FSEventStreamStart: null, + _FSEventStreamFlushSync: null, + _FSEventStreamStop: null, + _FSEventStreamInvalidate: null, + _FSEventStreamRelease: null, + _CFRunLoopGetCurrent: null, + _FSEventStreamCallbackType: null, + + // ObjC bridge helpers + _getClass: null, + _regSel: null, + _msg: null, + _msg_id: null, + _msg_ptr: null, + + _initFSEvents() { + let { ctypes } = ChromeUtils.importESModule( + "resource://gre/modules/ctypes.sys.mjs" + ); + this._ctypes = ctypes; + + let id = ctypes.voidptr_t; + let SEL = ctypes.voidptr_t; + + this._objcLib = ctypes.open("/usr/lib/libobjc.dylib"); + this._coreServicesLib = ctypes.open( + "/System/Library/Frameworks/CoreServices.framework" + + "/CoreServices" + ); + this._cfLib = ctypes.open( + "/System/Library/Frameworks/CoreFoundation.framework" + + "/CoreFoundation" + ); + + // ObjC bridge -- just enough for NSString/NSArray creation + this._getClass = this._objcLib.declare( + "objc_getClass", ctypes.default_abi, id, ctypes.char.ptr + ); + this._regSel = this._objcLib.declare( + "sel_registerName", ctypes.default_abi, SEL, ctypes.char.ptr + ); + this._msg = this._objcLib.declare( + "objc_msgSend", ctypes.default_abi, id, id, SEL + ); + this._msg_id = this._objcLib.declare( + "objc_msgSend", ctypes.default_abi, id, id, SEL, id + ); + this._msg_ptr = this._objcLib.declare( + "objc_msgSend", ctypes.default_abi, id, id, SEL, + ctypes.char.ptr + ); + + // FSEvents functions + this._FSEventsGetCurrentEventId = this._coreServicesLib.declare( + "FSEventsGetCurrentEventId", + ctypes.default_abi, ctypes.uint64_t + ); + + this._FSEventStreamCallbackType = ctypes.FunctionType( + ctypes.default_abi, ctypes.void_t, + [ + ctypes.voidptr_t, ctypes.voidptr_t, + ctypes.size_t, ctypes.voidptr_t, + ctypes.voidptr_t, ctypes.voidptr_t, + ] + ); + + this._FSEventStreamCreate = this._coreServicesLib.declare( + "FSEventStreamCreate", ctypes.default_abi, + ctypes.voidptr_t, + ctypes.voidptr_t, this._FSEventStreamCallbackType.ptr, + ctypes.voidptr_t, ctypes.voidptr_t, + ctypes.uint64_t, ctypes.double, ctypes.uint32_t + ); + + this._FSEventStreamScheduleWithRunLoop + = this._coreServicesLib.declare( + "FSEventStreamScheduleWithRunLoop", + ctypes.default_abi, ctypes.void_t, + ctypes.voidptr_t, ctypes.voidptr_t, ctypes.voidptr_t + ); + + this._FSEventStreamStart = this._coreServicesLib.declare( + "FSEventStreamStart", ctypes.default_abi, + ctypes.bool, ctypes.voidptr_t + ); + + this._FSEventStreamFlushSync = this._coreServicesLib.declare( + "FSEventStreamFlushSync", ctypes.default_abi, + ctypes.void_t, ctypes.voidptr_t + ); + + this._FSEventStreamStop = this._coreServicesLib.declare( + "FSEventStreamStop", ctypes.default_abi, + ctypes.void_t, ctypes.voidptr_t + ); + + this._FSEventStreamInvalidate = this._coreServicesLib.declare( + "FSEventStreamInvalidate", ctypes.default_abi, + ctypes.void_t, ctypes.voidptr_t + ); + + this._FSEventStreamRelease = this._coreServicesLib.declare( + "FSEventStreamRelease", ctypes.default_abi, + ctypes.void_t, ctypes.voidptr_t + ); + + this._CFRunLoopGetCurrent = this._cfLib.declare( + "CFRunLoopGetCurrent", ctypes.default_abi, + ctypes.voidptr_t + ); + + // Check for data directory change since last run + let savedPath = Zotero.Prefs.get( + "sync.storage.watcher.fsEventsStoragePath" + ); + if (savedPath && savedPath !== this._storageRoot) { + Zotero.debug("FileChangeWatcher: Storage path changed -- clearing saved event ID"); + Zotero.Prefs.clear("sync.storage.watcher.fsEventsEventID"); + } + Zotero.Prefs.set("sync.storage.watcher.fsEventsStoragePath", this._storageRoot); + }, + + _cls(name) { + return this._getClass(name); + }, + + _sel(name) { + return this._regSel(name); + }, + + _nsStr(str) { + return this._msg_ptr( + this._cls("NSString"), + this._sel("stringWithUTF8String:"), + str + ); + }, + + _getChangedItemKeysFSEvents() { + let ctypes = this._ctypes; + + // If no saved event ID, record baseline and signal full scan + let savedIdStr = Zotero.Prefs.get( + "sync.storage.watcher.fsEventsEventID" + ); + if (!savedIdStr) { + let currentId = this._FSEventsGetCurrentEventId(); + Zotero.Prefs.set( + "sync.storage.watcher.fsEventsEventID", currentId.toString() + ); + Zotero.debug("FileChangeWatcher: No saved event ID -- recorded baseline " + + currentId + ", signaling full scan"); + return null; + } + let sinceEventId = ctypes.UInt64(savedIdStr); + + let changedKeys = new Set(); + let storageRoot = this._storageRoot; + let keyPattern = this._keyPattern; + + let callbackFn = this._FSEventStreamCallbackType.ptr(function ( + _streamRef, _info, numEvents, eventPaths, + _eventFlags, _eventIds + ) { + let n = Number(numEvents); + let StringArray = ctypes.ArrayType(ctypes.char.ptr, n); + let paths = ctypes.cast( + eventPaths, StringArray.ptr + ).contents; + for (let i = 0; i < n; i++) { + let p = paths[i].readString(); + if (!p.startsWith(storageRoot)) continue; + let relative = p.substring(storageRoot.length); + let slashIdx = relative.indexOf("/"); + let key = slashIdx > 0 + ? relative.substring(0, slashIdx) + : relative; + if (keyPattern.test(key)) { + changedKeys.add(key); + } + } + }); + + let pathNS = this._nsStr(this._storageRoot); + let pathsArray = this._msg_id( + this._cls("NSArray"), + this._sel("arrayWithObject:"), + pathNS + ); + + // kFSEventStreamCreateFlagNoDefer | + // kFSEventStreamCreateFlagFileEvents + let flags = 0x02 | 0x10; + + let stream = this._FSEventStreamCreate( + null, callbackFn, null, pathsArray, + sinceEventId, 0.0, flags + ); + + if (stream.isNull()) { + Zotero.debug("FileChangeWatcher: FSEventStreamCreate returned null"); + return null; + } + + try { + let runLoop = this._CFRunLoopGetCurrent(); + let runLoopMode = this._nsStr("kCFRunLoopDefaultMode"); + this._FSEventStreamScheduleWithRunLoop( + stream, runLoop, runLoopMode + ); + + let started = this._FSEventStreamStart(stream); + if (!started) { + Zotero.debug("FileChangeWatcher: FSEventStreamStart failed"); + return null; + } + + this._FSEventStreamFlushSync(stream); + this._FSEventStreamStop(stream); + } + finally { + this._FSEventStreamInvalidate(stream); + this._FSEventStreamRelease(stream); + } + + let newEventId = this._FSEventsGetCurrentEventId(); + Zotero.Prefs.set( + "sync.storage.watcher.fsEventsEventID", newEventId.toString() + ); + + return this._returnKeys(changedKeys); + }, + + _closeFSEvents() { + if (this._objcLib) { + this._objcLib.close(); + this._objcLib = null; + } + if (this._coreServicesLib) { + this._coreServicesLib.close(); + this._coreServicesLib = null; + } + if (this._cfLib) { + this._cfLib.close(); + this._cfLib = null; + } + }, +}); diff --git a/chrome/content/zotero/xpcom/storage/fileChangeWatcher_inotify.js b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_inotify.js new file mode 100644 index 0000000000..4f0e0ff076 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_inotify.js @@ -0,0 +1,305 @@ +/* eslint-disable */ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * inotify backend for FileChangeWatcher (Linux) + * + * Like the Windows RDCW backend, inotify is a live watcher -- it only captures events from the + * moment watches are set up. The first getChangedItemKeys() call returns null to trigger a full + * scan. Periodic full scans every 3 hours ensure no changes are missed permanently. + * + * Note: errno values are read from ctypes.errno, which Mozilla ctypes captures immediately after + * each default_abi call. Using an explicit __errno_location() call would be unreliable because + * intervening ctypes machinery can reset errno. + */ +Object.assign(Zotero.Sync.Storage.FileChangeWatcher, { + _libc: null, + _inotifyFd: -1, + _inotifyRootWd: -1, + _inotifyWdToKey: null, // Map + _inotifyAccumulatedKeys: null, // Set + _inotifySubdirsSetUp: false, + + // ctypes function declarations for inotify + _inotifyInit1: null, + _inotifyAddWatch: null, + _readFn: null, + _closeFn: null, + + // inotify constants + _IN_CLOSE_WRITE: 0x00000008, + _IN_CREATE: 0x00000100, + _IN_DELETE: 0x00000200, + _IN_MOVED_TO: 0x00000080, + _IN_ISDIR: 0x40000000, + _IN_Q_OVERFLOW: 0x00004000, + _IN_NONBLOCK: 0x00000800, + _IN_CLOEXEC: 0x00080000, + _EAGAIN: 11, + _EINTR: 4, + _ENOSPC: 28, + + _INOTIFY_BUF_SIZE: 8192, + _inotifyBuf: null, + + _initInotify() { + let { ctypes } = ChromeUtils.importESModule( + "resource://gre/modules/ctypes.sys.mjs" + ); + this._ctypes = ctypes; + + this._libc = ctypes.open("libc.so.6"); + + this._inotifyInit1 = this._libc.declare( + "inotify_init1", ctypes.default_abi, + ctypes.int, ctypes.int + ); + + this._inotifyAddWatch = this._libc.declare( + "inotify_add_watch", ctypes.default_abi, + ctypes.int, + ctypes.int, ctypes.char.ptr, ctypes.uint32_t + ); + + this._readFn = this._libc.declare( + "read", ctypes.default_abi, ctypes.ssize_t, + ctypes.int, ctypes.uint8_t.ptr, ctypes.size_t + ); + + this._closeFn = this._libc.declare( + "close", ctypes.default_abi, + ctypes.int, ctypes.int + ); + + this._inotifyBuf = new (ctypes.ArrayType( + ctypes.uint8_t, this._INOTIFY_BUF_SIZE + ))(); + + // Create inotify instance (non-blocking, close-on-exec) + let fd = this._inotifyInit1( + this._IN_NONBLOCK | this._IN_CLOEXEC + ); + if (fd < 0) { + throw new Error("inotify_init1 failed (errno " + ctypes.errno + ")"); + } + this._inotifyFd = fd; + + // Watch storage root for new item directories + let rootWd = this._inotifyAddWatch( + fd, this._storageRoot, + this._IN_CREATE | this._IN_MOVED_TO + ); + if (rootWd < 0) { + let errno = ctypes.errno; + this._closeFn(fd); + this._inotifyFd = -1; + throw new Error("inotify_add_watch on storage root failed (errno " + + errno + ")"); + } + this._inotifyRootWd = rootWd; + + this._inotifyWdToKey = new Map(); + this._inotifyAccumulatedKeys = new Set(); + this._inotifySubdirsSetUp = false; + this._liveWatcherFirstCall = true; + }, + + /** + * Add inotify watches for all existing item subdirectories. Called lazily on first + * getChangedItemKeys() to avoid slowing down startup. + */ + _inotifySetupSubdirs() { + let ctypes = this._ctypes; + let dir = Zotero.File.pathToFile(this._storageRoot.slice(0, -1)); + + let count = 0; + let entries = dir.directoryEntries; + while (entries.hasMoreElements()) { + let entry = entries.getNext().QueryInterface(Ci.nsIFile); + if (!entry.isDirectory()) continue; + let name = entry.leafName; + if (!this._keyPattern.test(name)) continue; + + let wd = this._inotifyAddWatch( + this._inotifyFd, entry.path, + this._IN_CLOSE_WRITE | this._IN_MOVED_TO + | this._IN_CREATE | this._IN_DELETE + ); + if (wd < 0) { + if (ctypes.errno === this._ENOSPC) { + Zotero.debug("FileChangeWatcher: inotify watch limit reached after " + + count + " directories -- some changes may be missed"); + break; + } + continue; + } + this._inotifyWdToKey.set(wd, name); + count++; + } + + this._inotifySubdirsSetUp = true; + Zotero.debug("FileChangeWatcher: Added inotify watches for " + count + + " storage directories"); + }, + + /** + * Non-blocking drain of all pending inotify events into _inotifyAccumulatedKeys. + * + * @return {boolean} false if an overflow was detected + */ + _inotifyDrainEvents() { + let ctypes = this._ctypes; + let headerSize = 16; // struct inotify_event fixed part + + while (true) { + let bytesRead = this._readFn( + this._inotifyFd, + this._inotifyBuf.addressOfElement(0), + this._INOTIFY_BUF_SIZE + ); + + if (bytesRead <= 0) { + let errno = ctypes.errno; + if (errno === this._EAGAIN + || errno === this._EINTR) { + break; + } + Zotero.debug("FileChangeWatcher: inotify read error (errno " + errno + ")"); + break; + } + + let offset = 0; + while (offset + headerSize <= bytesRead) { + let wdPtr = ctypes.cast( + this._inotifyBuf.addressOfElement(offset), + ctypes.int32_t.ptr + ); + let maskPtr = ctypes.cast( + this._inotifyBuf.addressOfElement(offset + 4), + ctypes.uint32_t.ptr + ); + let lenPtr = ctypes.cast( + this._inotifyBuf.addressOfElement( + offset + 12 + ), + ctypes.uint32_t.ptr + ); + + let wd = wdPtr.contents; + let mask = maskPtr.contents; + let nameLen = lenPtr.contents; + + if (mask & this._IN_Q_OVERFLOW) { + Zotero.debug("FileChangeWatcher: inotify queue overflow"); + return false; + } + + let name = ""; + if (nameLen > 0) { + let namePtr = ctypes.cast( + this._inotifyBuf.addressOfElement( + offset + headerSize + ), + ctypes.char.ptr + ); + name = namePtr.readString(); + } + + if (wd === this._inotifyRootWd) { + if ((mask & (this._IN_CREATE + | this._IN_MOVED_TO)) + && (mask & this._IN_ISDIR) + && this._keyPattern.test(name)) { + this._inotifyAccumulatedKeys.add(name); + let subPath = this._storageRoot + name; + let subWd = this._inotifyAddWatch( + this._inotifyFd, subPath, + this._IN_CLOSE_WRITE + | this._IN_MOVED_TO + | this._IN_CREATE + | this._IN_DELETE + ); + if (subWd >= 0) { + this._inotifyWdToKey.set( + subWd, name + ); + } + } + } + else { + let key = this._inotifyWdToKey.get(wd); + if (key) { + this._inotifyAccumulatedKeys.add(key); + } + } + + offset += headerSize + nameLen; + } + } + + return true; + }, + + _getChangedItemKeysInotify() { + if (!this._inotifySubdirsSetUp) { + this._inotifySetupSubdirs(); + } + + let ok = this._inotifyDrainEvents(); + + // First call or periodic refresh -- signal full scan + if (this._liveWatcherNeedsFullScan()) { + Zotero.debug("FileChangeWatcher: Live watcher signaling full scan" + + " (first call or periodic refresh)"); + this._inotifyAccumulatedKeys.clear(); + return null; + } + + // Overflow -- signal full scan + if (!ok) { + this._inotifyAccumulatedKeys.clear(); + this._liveWatcherLastFullScanTime = Date.now(); + return null; + } + + let keys = this._inotifyAccumulatedKeys; + this._inotifyAccumulatedKeys = new Set(); + return this._returnKeys(keys); + }, + + _closeInotify() { + if (this._inotifyFd >= 0) { + this._closeFn(this._inotifyFd); + this._inotifyFd = -1; + } + if (this._libc) { + this._libc.close(); + this._libc = null; + } + this._inotifyWdToKey = null; + this._inotifyAccumulatedKeys = null; + }, +}); diff --git a/chrome/content/zotero/xpcom/storage/fileChangeWatcher_rdcw.js b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_rdcw.js new file mode 100644 index 0000000000..97e2a13fe9 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/fileChangeWatcher_rdcw.js @@ -0,0 +1,409 @@ +/* eslint-disable */ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * ReadDirectoryChangesW backend for FileChangeWatcher (Windows) + * + * Uses overlapped I/O with ReadDirectoryChangesW to watch the storage directory recursively. + * Unlike the previous USN Change Journal backend, this works without admin privileges -- it only + * needs read access to the storage directory itself. + * + * This is a live watcher (like inotify on Linux): the first getChangedItemKeys() call returns null + * to trigger a full scan, then accumulates changes between calls. Periodic full scans every 3 hours + * via _liveWatcherNeedsFullScan() ensure no changes are missed permanently. + * + * Approach -- overlapped I/O polling: + * 1. Init: Open storage directory with FILE_FLAG_OVERLAPPED, create an event handle, issue the + * first ReadDirectoryChangesW call (returns immediately, queues the request) + * 2. Poll (getChangedItemKeys): Call GetOverlappedResult(bWait=FALSE) -- non-blocking check. + * If data ready, parse FILE_NOTIFY_INFORMATION records, extract item keys, re-arm. + * If not ready, return accumulated keys (may be empty). + * 3. Close: Close directory handle (auto-cancels pending I/O), close event handle, close kernel32 + * + * Note: Win32 error codes are read from ctypes.winLastError, which Mozilla ctypes captures + * immediately after each winapi_abi call. Using an explicit GetLastError() declaration would be + * unreliable because intervening ctypes machinery can reset the thread error. + */ +Object.assign(Zotero.Sync.Storage.FileChangeWatcher, { + _kernel32: null, + _ctypes: null, + _dirHandle: null, + _eventHandle: null, + _overlapped: null, + _rdcwBuf: null, + _rdcwAccumulatedKeys: null, + _rdcwPending: false, + + // ctypes function declarations + _CreateFileW: null, + _CloseHandle: null, + _ReadDirectoryChangesW: null, + _GetOverlappedResult: null, + _CreateEventW: null, + + // OVERLAPPED struct type + _OVERLAPPED: null, + + // Windows constants + _FILE_LIST_DIRECTORY: 0x0001, + _FILE_SHARE_READ: 0x00000001, + _FILE_SHARE_WRITE: 0x00000002, + _FILE_SHARE_DELETE: 0x00000004, + _OPEN_EXISTING: 3, + _FILE_FLAG_BACKUP_SEMANTICS: 0x02000000, + _FILE_FLAG_OVERLAPPED: 0x40000000, + + // Notify filter -- watch for file/dir name changes, size changes, and last-write changes + _RDCW_NOTIFY_FILTER: 0x00000001 // FILE_NOTIFY_CHANGE_FILE_NAME + | 0x00000002 // FILE_NOTIFY_CHANGE_DIR_NAME + | 0x00000008 // FILE_NOTIFY_CHANGE_SIZE + | 0x00000010, // FILE_NOTIFY_CHANGE_LAST_WRITE + + _RDCW_BUF_SIZE: 65536, + + // Expected overlapped I/O error codes + _ERROR_IO_INCOMPLETE: 996, + _ERROR_IO_PENDING: 997, + _ERROR_NOTIFY_ENUM_DIR: 1022, + + _initRDCW() { + let { ctypes } = ChromeUtils.importESModule( + "resource://gre/modules/ctypes.sys.mjs" + ); + this._ctypes = ctypes; + + this._kernel32 = ctypes.open("kernel32.dll"); + + // ---- Struct types ---- + + this._OVERLAPPED = new ctypes.StructType("OVERLAPPED", [ + { "Internal": ctypes.uintptr_t }, + { "InternalHigh": ctypes.uintptr_t }, + { "OffsetLow": ctypes.uint32_t }, + { "OffsetHigh": ctypes.uint32_t }, + { "hEvent": ctypes.voidptr_t }, + ]); + + // ---- Function declarations ---- + + this._CreateFileW = this._kernel32.declare( + "CreateFileW", ctypes.winapi_abi, + ctypes.voidptr_t, // HANDLE + ctypes.char16_t.ptr, // lpFileName + ctypes.uint32_t, // dwDesiredAccess + ctypes.uint32_t, // dwShareMode + ctypes.voidptr_t, // lpSecurityAttributes + ctypes.uint32_t, // dwCreationDisposition + ctypes.uint32_t, // dwFlagsAndAttributes + ctypes.voidptr_t // hTemplateFile + ); + + this._CloseHandle = this._kernel32.declare( + "CloseHandle", ctypes.winapi_abi, + ctypes.bool, ctypes.voidptr_t + ); + + this._ReadDirectoryChangesW = this._kernel32.declare( + "ReadDirectoryChangesW", ctypes.winapi_abi, + ctypes.bool, + ctypes.voidptr_t, // hDirectory + ctypes.voidptr_t, // lpBuffer + ctypes.uint32_t, // nBufferLength + ctypes.bool, // bWatchSubtree + ctypes.uint32_t, // dwNotifyFilter + ctypes.uint32_t.ptr, // lpBytesReturned + ctypes.voidptr_t, // lpOverlapped (OVERLAPPED*) + ctypes.voidptr_t // lpCompletionRoutine + ); + + this._GetOverlappedResult = this._kernel32.declare( + "GetOverlappedResult", ctypes.winapi_abi, + ctypes.bool, + ctypes.voidptr_t, // hFile + ctypes.voidptr_t, // lpOverlapped (OVERLAPPED*) + ctypes.uint32_t.ptr, // lpNumberOfBytesTransferred + ctypes.bool // bWait + ); + + this._CreateEventW = this._kernel32.declare( + "CreateEventW", ctypes.winapi_abi, + ctypes.voidptr_t, // HANDLE return + ctypes.voidptr_t, // lpEventAttributes + ctypes.bool, // bManualReset + ctypes.bool, // bInitialState + ctypes.voidptr_t // lpName + ); + + // ---- Open storage directory ---- + + let dirPath = this._storageRoot.slice(0, -1); // remove trailing backslash + let dirPathBuf = ctypes.char16_t.array()(dirPath); + + let handle = this._CreateFileW( + dirPathBuf, + this._FILE_LIST_DIRECTORY, + this._FILE_SHARE_READ | this._FILE_SHARE_WRITE | this._FILE_SHARE_DELETE, + null, + this._OPEN_EXISTING, + this._FILE_FLAG_BACKUP_SEMANTICS | this._FILE_FLAG_OVERLAPPED, + null + ); + // INVALID_HANDLE_VALUE is (HANDLE)-1 -- cast to intptr_t to check + if (ctypes.cast(handle, ctypes.intptr_t).value.toString() === "-1") { + throw new Error("Cannot open storage directory for watching (error " + + ctypes.winLastError + ")"); + } + this._dirHandle = handle; + + // ---- Create event for overlapped I/O ---- + + let eventHandle = this._CreateEventW(null, true, false, null); + if (eventHandle.isNull()) { + this._CloseHandle(this._dirHandle); + this._dirHandle = null; + throw new Error("CreateEventW failed (error " + ctypes.winLastError + ")"); + } + this._eventHandle = eventHandle; + + // ---- Allocate buffer and overlapped struct ---- + + this._rdcwBuf = new (ctypes.ArrayType( + ctypes.uint8_t, this._RDCW_BUF_SIZE + ))(); + + this._overlapped = new this._OVERLAPPED(); + this._overlapped.Internal = 0; + this._overlapped.InternalHigh = 0; + this._overlapped.OffsetLow = 0; + this._overlapped.OffsetHigh = 0; + this._overlapped.hEvent = this._eventHandle; + + this._rdcwAccumulatedKeys = new Set(); + this._liveWatcherFirstCall = true; + + // ---- Issue first ReadDirectoryChangesW call ---- + + this._rdcwArm(); + this._rdcwPending = true; + }, + + /** + * Issue (or re-issue) an overlapped ReadDirectoryChangesW call. + * Throws on failure. + */ + _rdcwArm() { + let bytesReturned = new this._ctypes.uint32_t(0); + let ok = this._ReadDirectoryChangesW( + this._dirHandle, + this._rdcwBuf.address(), + this._RDCW_BUF_SIZE, + true, // bWatchSubtree -- recursive + this._RDCW_NOTIFY_FILTER, + bytesReturned.address(), + this._overlapped.address(), + null // no completion routine + ); + if (!ok) { + let err = this._ctypes.winLastError; + // ERROR_IO_PENDING is expected for overlapped I/O + if (err !== this._ERROR_IO_PENDING) { + throw new Error("ReadDirectoryChangesW failed (error " + err + ")"); + } + } + }, + + /** + * Parse FILE_NOTIFY_INFORMATION records from the buffer and extract item keys. + * + * FILE_NOTIFY_INFORMATION is a variable-length linked list: + * DWORD NextEntryOffset (0 = last entry) + * DWORD Action + * DWORD FileNameLength (in bytes) + * WCHAR FileName[] (UTF-16LE, not null-terminated, relative path with backslashes) + * + * We extract the first path component before the first backslash as the item key. + * + * @param {number} bytesTransferred + */ + _rdcwParseNotifications(bytesTransferred) { + let ctypes = this._ctypes; + let offset = 0; + + while (offset < bytesTransferred) { + // Read NextEntryOffset (uint32_t at offset+0) + let nextEntryPtr = ctypes.cast( + this._rdcwBuf.addressOfElement(offset), + ctypes.uint32_t.ptr + ); + let nextEntryOffset = nextEntryPtr.contents; + + // Read FileNameLength (uint32_t at offset+8, in bytes) + let fileNameLenPtr = ctypes.cast( + this._rdcwBuf.addressOfElement(offset + 8), + ctypes.uint32_t.ptr + ); + let fileNameLength = fileNameLenPtr.contents; + + // Read FileName (UTF-16LE at offset+12) + let nameChars = fileNameLength / 2; + if (nameChars > 0) { + // Extract the first path component (before first backslash) + let key = ""; + let nameStart = offset + 12; + for (let i = 0; i < nameChars; i++) { + let byteOff = nameStart + i * 2; + let lo = this._rdcwBuf[byteOff]; + let hi = this._rdcwBuf[byteOff + 1]; + let ch = lo | (hi << 8); + if (ch === 0x5C) { // backslash + break; + } + key += String.fromCharCode(ch); + } + + if (this._keyPattern.test(key)) { + this._rdcwAccumulatedKeys.add(key); + } + } + + if (nextEntryOffset === 0) { + break; + } + offset += nextEntryOffset; + } + }, + + _getChangedItemKeysRDCW() { + let ctypes = this._ctypes; + + // Drain all available notifications + if (this._rdcwPending) { + let maxDrains = 100; + let drains = 0; + + while (drains++ < maxDrains) { + let bytesTransferred = new ctypes.uint32_t(0); + let ok = this._GetOverlappedResult( + this._dirHandle, + this._overlapped.address(), + bytesTransferred.address(), + false // bWait=FALSE -- non-blocking + ); + + if (!ok) { + let err = ctypes.winLastError; + if (err === this._ERROR_IO_INCOMPLETE) { + // No data ready yet -- that's fine + break; + } + if (err === this._ERROR_NOTIFY_ENUM_DIR) { + // Buffer overflow -- signal full scan + Zotero.debug("FileChangeWatcher: RDCW buffer overflow" + + " -- signaling full scan"); + this._rdcwAccumulatedKeys.clear(); + this._liveWatcherLastFullScanTime = Date.now(); + // Re-arm for future notifications + try { + this._rdcwArm(); + } + catch (e) { + Zotero.logError(e); + this._rdcwPending = false; + } + return null; + } + // Other error -- log and signal full scan + Zotero.debug("FileChangeWatcher: GetOverlappedResult failed" + + " (error " + err + ")"); + this._rdcwPending = false; + return null; + } + + // Data ready + let bytes = bytesTransferred.value; + if (bytes === 0) { + // Zero bytes means overflow -- signal full scan + Zotero.debug("FileChangeWatcher: RDCW returned 0 bytes" + + " -- signaling full scan"); + this._rdcwAccumulatedKeys.clear(); + this._liveWatcherLastFullScanTime = Date.now(); + try { + this._rdcwArm(); + } + catch (e) { + Zotero.logError(e); + this._rdcwPending = false; + } + return null; + } + + this._rdcwParseNotifications(bytes); + + // Re-arm and check for more + try { + this._rdcwArm(); + } + catch (e) { + Zotero.logError(e); + this._rdcwPending = false; + break; + } + } + } + + // First call or periodic refresh -- signal full scan + if (this._liveWatcherNeedsFullScan()) { + Zotero.debug("FileChangeWatcher: Live watcher signaling full scan" + + " (first call or periodic refresh)"); + this._rdcwAccumulatedKeys.clear(); + return null; + } + + let keys = this._rdcwAccumulatedKeys; + this._rdcwAccumulatedKeys = new Set(); + return this._returnKeys(keys); + }, + + _closeRDCW() { + // Closing the directory handle auto-cancels pending overlapped I/O + if (this._dirHandle && !this._dirHandle.isNull()) { + this._CloseHandle(this._dirHandle); + this._dirHandle = null; + } + if (this._eventHandle && !this._eventHandle.isNull()) { + this._CloseHandle(this._eventHandle); + this._eventHandle = null; + } + if (this._kernel32) { + this._kernel32.close(); + this._kernel32 = null; + } + this._overlapped = null; + this._rdcwBuf = null; + this._rdcwAccumulatedKeys = null; + this._rdcwPending = false; + }, +}); diff --git a/chrome/content/zotero/xpcom/storage/storageEngine.js b/chrome/content/zotero/xpcom/storage/storageEngine.js index 4064f78fbb..28b19ab021 100644 --- a/chrome/content/zotero/xpcom/storage/storageEngine.js +++ b/chrome/content/zotero/xpcom/storage/storageEngine.js @@ -146,6 +146,40 @@ Zotero.Sync.Storage.Engine.prototype.start = async function () { Zotero.debug("No file editing access -- skipping file modification check for " + this.library.name); } + // Use file change watcher to check only files that actually changed on disk + // + // Note: File-sync downloads also generate filesystem events, so recently downloaded files will + // appear here on the next sync cycle, but checkForUpdatedFiles() will see that the mtime/hash + // match the synced values and skip them. + else if (Zotero.Sync.Storage.FileChangeWatcher.available) { + let changedKeys = Zotero.Sync.Storage.FileChangeWatcher.getChangedItemKeys(); + if (changedKeys) { + if (changedKeys.size > 0) { + let keysArray = Array.from(changedKeys); + let itemIDs = []; + // Batch to avoid hitting SQLite parameter limit + await Zotero.Utilities.Internal.forEachChunkAsync( + keysArray, 500, async function (chunk) { + let ids = await Zotero.DB.columnQueryAsync( + "SELECT itemID FROM items WHERE libraryID=? AND key IN (" + + chunk.map(() => '?').join(',') + ")", + [libraryID, ...chunk] + ); + itemIDs.push(...ids); + } + ); + if (itemIDs.length) { + await this.local.checkForUpdatedFiles(libraryID, itemIDs); + } + } + // else: no changes detected, skip scan entirely + } + else { + // Watcher returned null (first run or error) -- full scan + this.local.lastFullFileCheck[libraryID] = new Date().getTime(); + await this.local.checkForUpdatedFiles(libraryID); + } + } // If this is a background sync, it's not the first sync of the session, the library has had // at least one full check this session, and it's been less than maxCheckAge since the last // full check of this library, check only files that were previously modified or opened diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index e59dee7a33..bf01e745d2 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -710,6 +710,7 @@ const { CommandLineOptions } = ChromeUtils.importESModule("chrome://zotero/conte await Zotero.Sync.Data.Local.init(); await Zotero.Sync.Data.Utilities.init(); Zotero.Sync.Storage.Local.init(); + Zotero.Sync.Storage.FileChangeWatcher.init(); Zotero.Sync.Runner = new Zotero.Sync.Runner_Module; Zotero.Sync.EventListeners.init(); Zotero.Streamer = new Zotero.Streamer_Module; diff --git a/chrome/content/zotero/zotero.mjs b/chrome/content/zotero/zotero.mjs index 915d3e2e57..47a2a0f520 100644 --- a/chrome/content/zotero/zotero.mjs +++ b/chrome/content/zotero/zotero.mjs @@ -147,6 +147,7 @@ const xpcomFilesLocal = [ 'storage', 'storage/storageEngine', 'storage/storageLocal', + 'storage/fileChangeWatcher', 'storage/storageRequest', 'storage/storageResult', 'storage/storageUtilities', @@ -274,6 +275,27 @@ function makeZoteroContext() { } } + // Load platform-specific FileChangeWatcher backend + { + let os = Services.appinfo.OS; + let backendFile = os == "Darwin" ? "storage/fileChangeWatcher_fsevents" + : os == "WINNT" ? "storage/fileChangeWatcher_rdcw" + : os == "Linux" ? "storage/fileChangeWatcher_inotify" + : null; + if (backendFile) { + try { + subscriptLoader.loadSubScript( + "chrome://zotero/content/xpcom/" + backendFile + ".js", zContext, "utf-8" + ); + } + catch (e) { + dump("Error loading " + backendFile + ".js\n\n"); + dump(e + "\n\n"); + Components.utils.reportError("Error loading " + backendFile + ".js"); + } + } + } + // Load RDF files into Zotero.RDF.AJAW namespace (easier than modifying all of the references) const rdfXpcomFiles = [ 'rdf/init', diff --git a/test/tests/fileChangeWatcherTest.js b/test/tests/fileChangeWatcherTest.js new file mode 100644 index 0000000000..30383abb2a --- /dev/null +++ b/test/tests/fileChangeWatcherTest.js @@ -0,0 +1,504 @@ +describe("Zotero.Sync.Storage.FileChangeWatcher", function () { + let watcher = Zotero.Sync.Storage.FileChangeWatcher; + + // Pref keys used by the watcher + let PREF_FSEVENTS_EVENT_ID = "sync.storage.watcher.fsEventsEventID"; + let PREF_FSEVENTS_STORAGE_PATH = "sync.storage.watcher.fsEventsStoragePath"; + + function clearWatcherPrefs() { + Zotero.Prefs.clear(PREF_FSEVENTS_EVENT_ID); + Zotero.Prefs.clear(PREF_FSEVENTS_STORAGE_PATH); + } + + // The Set returned by getChangedItemKeys() is created in the XPCOM + // context, so instanceof Set doesn't work across contexts. Use + // duck-typing instead. + function assertIsSet(val, msg) { + assert.isNotNull(val, msg); + assert.isFunction(val.has, msg); + assert.isNumber(val.size, msg); + } + + describe("FSEvents backend (macOS)", function () { + before(async function () { + if (!Zotero.isMac) { + this.skip(); + } + // Ensure the storage directory exists (it may not yet in + // the test environment) + let storageDir = PathUtils.join( + Zotero.DataDirectory.dir, "storage" + ); + await IOUtils.makeDirectory(storageDir, { + ignoreExisting: true + }); + // Close any watcher initialized at startup + watcher.close(); + clearWatcherPrefs(); + }); + + afterEach(function () { + watcher.close(); + clearWatcherPrefs(); + }); + + after(function () { + // Re-init the watcher so it's available for normal operation + // after the test suite + watcher.init(); + }); + + it("should initialize successfully", function () { + watcher.init(); + assert.isTrue(watcher.available); + assert.equal(watcher._backend, 'fsevents'); + assert.ok(watcher._storageRoot, "Storage root should be set"); + }); + + it("should return null on first call with no saved event ID", function () { + watcher.init(); + let result = watcher.getChangedItemKeys(); + assert.isNull(result, "First call should return null"); + // Should have saved a baseline event ID + let savedId = Zotero.Prefs.get(PREF_FSEVENTS_EVENT_ID); + assert.isString(savedId); + assert.notEqual(savedId, "0"); + }); + + it("should return an empty set when no files have changed", function () { + watcher.init(); + // Establish baseline + watcher.getChangedItemKeys(); + + // Second call -- no changes + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + assert.equal(result.size, 0); + }); + + it("should detect a modified attachment file", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + + try { + watcher.init(); + // Establish baseline + watcher.getChangedItemKeys(); + + // Modify the file + let path = await item.getFilePathAsync(); + await Zotero.File.putContentsAsync( + path, Zotero.Utilities.randomString() + ); + + // Wait for FSEvents to register + await Zotero.Promise.delay(1000); + + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + assert.isTrue( + result.has(item.key), + "Should contain the key of the modified item" + ); + } + finally { + await item.eraseTx(); + } + }); + + it("should detect changes across multiple items", async function () { + this.timeout(15000); + + let item1 = await importFileAttachment('test.png'); + let item2 = await importTextAttachment(); + let item3 = await importFileAttachment('test.png'); + + try { + watcher.init(); + // Wait for FSEvents from item creation to flush + // before establishing baseline + await Zotero.Promise.delay(1000); + watcher.getChangedItemKeys(); + + // Modify only items 1 and 3 + let path1 = await item1.getFilePathAsync(); + await Zotero.File.putContentsAsync( + path1, Zotero.Utilities.randomString() + ); + let path3 = await item3.getFilePathAsync(); + await Zotero.File.putContentsAsync( + path3, Zotero.Utilities.randomString() + ); + + await Zotero.Promise.delay(1000); + + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + assert.isTrue(result.has(item1.key), + "Should contain key of first modified item"); + assert.isFalse(result.has(item2.key), + "Should not contain key of unmodified item"); + assert.isTrue(result.has(item3.key), + "Should contain key of second modified item"); + } + finally { + await item1.eraseTx(); + await item2.eraseTx(); + await item3.eraseTx(); + } + }); + + it("should not report the same changes twice", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + + try { + watcher.init(); + watcher.getChangedItemKeys(); + + // Modify file + let path = await item.getFilePathAsync(); + await Zotero.File.putContentsAsync( + path, Zotero.Utilities.randomString() + ); + + await Zotero.Promise.delay(1000); + + // First query picks up the change + let result1 = watcher.getChangedItemKeys(); + assertIsSet(result1); + assert.isTrue(result1.has(item.key)); + + // Second query should be empty + let result2 = watcher.getChangedItemKeys(); + assertIsSet(result2); + assert.equal(result2.size, 0, + "Should not report the same change twice"); + } + finally { + await item.eraseTx(); + } + }); + + it("should detect a new file added to a storage directory", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + + try { + watcher.init(); + watcher.getChangedItemKeys(); + + // Add a new file to the item's storage directory + let storageDir = Zotero.Attachments.getStorageDirectory(item).path; + let newFile = PathUtils.join(storageDir, "extra.txt"); + await Zotero.File.putContentsAsync(newFile, "extra content"); + + await Zotero.Promise.delay(1000); + + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + assert.isTrue(result.has(item.key), + "Should detect new file in storage directory"); + } + finally { + await item.eraseTx(); + } + }); + + it("should return null when storage path has changed", function () { + // Simulate a previous run with a different storage path + Zotero.Prefs.set(PREF_FSEVENTS_STORAGE_PATH, "/some/other/path/"); + Zotero.Prefs.set(PREF_FSEVENTS_EVENT_ID, "12345"); + + watcher.init(); + + // The saved event ID should have been cleared + let result = watcher.getChangedItemKeys(); + assert.isNull(result, + "Should return null after storage path change"); + }); + + it("should persist event ID across close/init cycles", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + + try { + watcher.init(); + // Establish baseline + watcher.getChangedItemKeys(); + + let savedId = Zotero.Prefs.get(PREF_FSEVENTS_EVENT_ID); + assert.ok(savedId, "Event ID should be saved"); + + // Close and re-init (simulating restart) + watcher.close(); + watcher.init(); + + // Should not return null -- saved event ID should be used + let result = watcher.getChangedItemKeys(); + assertIsSet(result, + "Should return a Set (not null) after re-init " + + "with saved event ID"); + } + finally { + await item.eraseTx(); + } + }); + + it("should detect changes that happened while watcher was closed", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + + try { + watcher.init(); + watcher.getChangedItemKeys(); + + // Close the watcher + watcher.close(); + + // Modify file while watcher is closed + let path = await item.getFilePathAsync(); + await Zotero.File.putContentsAsync( + path, Zotero.Utilities.randomString() + ); + + await Zotero.Promise.delay(1000); + + // Re-init and query -- FSEvents journal should have + // the change even though the watcher was closed + watcher.init(); + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + assert.isTrue(result.has(item.key), + "Should detect changes that occurred while " + + "watcher was closed"); + } + finally { + await item.eraseTx(); + } + }); + + it("should only report valid 8-char item keys", async function () { + this.timeout(15000); + + watcher.init(); + watcher.getChangedItemKeys(); + + // Create a non-standard directory name in the storage root + let storageRoot = watcher._storageRoot; + let invalidDir = PathUtils.join(storageRoot, "not-a-key"); + await IOUtils.makeDirectory(invalidDir, { ignoreExisting: true }); + let testFile = PathUtils.join(invalidDir, "test.txt"); + await IOUtils.writeUTF8(testFile, "test"); + + await Zotero.Promise.delay(1000); + + try { + let result = watcher.getChangedItemKeys(); + assertIsSet(result); + // Should not contain the invalid directory name + for (let key of result) { + assert.match(key, /^[A-Z0-9]{8}$/, + "All returned keys should match the 8-char " + + "pattern"); + } + } + finally { + await IOUtils.remove(invalidDir, { recursive: true }); + } + }); + }); + + describe("integration with storageEngine", function () { + before(async function () { + if (!Zotero.isMac) { + this.skip(); + } + let storageDir = PathUtils.join( + Zotero.DataDirectory.dir, "storage" + ); + await IOUtils.makeDirectory(storageDir, { + ignoreExisting: true + }); + watcher.close(); + clearWatcherPrefs(); + }); + + afterEach(function () { + watcher.close(); + clearWatcherPrefs(); + }); + + after(function () { + watcher.init(); + }); + + it("should be checked before falling back to legacy scanning", async function () { + this.timeout(15000); + + let item = await importFileAttachment('test.png'); + let hash = await item.attachmentHash; + let mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000; + await OS.File.setDates( + (await item.getFilePathAsync()), null, mtime + ); + + // Mark as synced + item.attachmentSyncedModificationTime = mtime; + item.attachmentSyncedHash = hash; + item.attachmentSyncState = "in_sync"; + await item.saveTx({ skipAll: true }); + + try { + watcher.init(); + // Establish baseline + watcher.getChangedItemKeys(); + + // Modify the file + let path = await item.getFilePathAsync(); + await OS.File.setDates(path); + await Zotero.File.putContentsAsync( + path, Zotero.Utilities.randomString() + ); + + await Zotero.Promise.delay(1000); + + // Verify the watcher reports the change + let changedKeys = watcher.getChangedItemKeys(); + assertIsSet(changedKeys); + assert.isTrue(changedKeys.has(item.key)); + + // Map keys to itemIDs as storageEngine does + let libraryID = Zotero.Libraries.userLibraryID; + let itemIDs = await Zotero.DB.columnQueryAsync( + "SELECT itemID FROM items WHERE libraryID=?" + + " AND key IN (" + + Array.from(changedKeys).map(() => '?') + .join(',') + + ")", + [libraryID, ...changedKeys] + ); + assert.include(itemIDs, item.id); + + // checkForUpdatedFiles should flag it for upload + let changed = await Zotero.Sync.Storage.Local + .checkForUpdatedFiles(libraryID, itemIDs); + assert.isTrue(changed); + assert.equal( + item.attachmentSyncState, + Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD + ); + } + finally { + await item.eraseTx(); + } + }); + }); + + describe("ReadDirectoryChangesW backend (Windows)", function () { + before(async function () { + if (!Zotero.isWin) { + this.skip(); + } + let storageDir = PathUtils.join( + Zotero.DataDirectory.dir, "storage" + ); + await IOUtils.makeDirectory(storageDir, { + ignoreExisting: true + }); + watcher.close(); + }); + + afterEach(function () { + watcher.close(); + }); + + after(function () { + watcher.init(); + }); + + it("should initialize successfully"); + it("should return null on first call (no persistent journal)"); + it("should return an empty set when no files have changed"); + it("should detect a modified attachment file"); + it("should detect changes across multiple items"); + it("should not report the same changes twice"); + it("should detect a new file added to a storage directory"); + it("should re-arm overlapped I/O after draining notifications"); + it("should return null on buffer overflow and re-arm"); + it("should return null for periodic full scan after max age"); + it("should only report valid 8-char item keys"); + }); + + describe("inotify backend (Linux)", function () { + before(async function () { + if (!Zotero.isLinux) { + this.skip(); + } + let storageDir = PathUtils.join( + Zotero.DataDirectory.dir, "storage" + ); + await IOUtils.makeDirectory(storageDir, { + ignoreExisting: true + }); + watcher.close(); + clearWatcherPrefs(); + }); + + afterEach(function () { + watcher.close(); + clearWatcherPrefs(); + }); + + after(function () { + watcher.init(); + }); + + it("should initialize successfully"); + it("should return null on first call (no persistent journal)"); + it("should return an empty set when no files have changed"); + it("should detect a modified attachment file"); + it("should detect changes across multiple items"); + it("should not report the same changes twice"); + it("should detect a new file added to a storage directory"); + it("should auto-watch newly created storage subdirectories"); + it("should return null for periodic full scan after max age"); + it("should only report valid 8-char item keys"); + }); + + describe("fallback behavior", function () { + it("should return null from getChangedItemKeys() when unavailable", function () { + let savedAvailable = watcher.available; + let savedBackend = watcher._backend; + watcher.available = false; + watcher._backend = null; + try { + let result = watcher.getChangedItemKeys(); + assert.isNull(result); + } + finally { + watcher.available = savedAvailable; + watcher._backend = savedBackend; + } + }); + + it("should set available to false after close()", function () { + if (!Zotero.isMac) { + this.skip(); + } + watcher.init(); + assert.isTrue(watcher.available); + watcher.close(); + assert.isFalse(watcher.available); + assert.isNull(watcher._backend); + // Re-init for other tests + clearWatcherPrefs(); + watcher.init(); + }); + }); +});