diff --git a/.gitattributes b/.gitattributes
index 4fb3d77151..c0373931d8 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3,3 +3,4 @@ app/win/zotero.exe.tar.xz filter=lfs diff=lfs merge=lfs -text
app/mac/updater.tar.xz filter=lfs diff=lfs merge=lfs -text
app/win/updater.exe.tar.xz filter=lfs diff=lfs merge=lfs -text
app/linux/updater.tar.xz filter=lfs diff=lfs merge=lfs -text
+resource/embeddings/*.wasm filter=lfs diff=lfs merge=lfs -text
diff --git a/chrome/content/zotero/collectionViewItemTree.jsx b/chrome/content/zotero/collectionViewItemTree.jsx
index 30ad42f7c8..7ae2f48629 100644
--- a/chrome/content/zotero/collectionViewItemTree.jsx
+++ b/chrome/content/zotero/collectionViewItemTree.jsx
@@ -770,6 +770,13 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
refresh = true;
madeChanges = true;
}
+ // Under an active similarity quick search, handle removals with a full refresh too
+ else if (['remove', 'delete', 'trash'].includes(action)
+ && collectionTreeRows.some(row => row.isSimilaritySearch())) {
+ this.itemTree.invalidateRowCache(ids);
+ refresh = true;
+ madeChanges = true;
+ }
else if ((action == 'remove' && !collectionTreeRows.some(row => row.isLibrary(true)))
|| action == 'delete' || action == 'trash'
|| (action == 'removeDuplicatesMaster' && this.viewMode == 'duplicates')) {
diff --git a/chrome/content/zotero/elements/quickSearchTextbox.js b/chrome/content/zotero/elements/quickSearchTextbox.js
index 79f51869e1..0095132a0a 100644
--- a/chrome/content/zotero/elements/quickSearchTextbox.js
+++ b/chrome/content/zotero/elements/quickSearchTextbox.js
@@ -45,11 +45,17 @@
`, ['chrome://zotero/locale/zotero.dtd']);
}
- _searchModes = {
- titleCreatorYear: Zotero.getString('quickSearch.mode.titleCreatorYear'),
- fields: Zotero.getString('quickSearch.mode.fieldsAndTags'),
- everything: Zotero.getString('quickSearch.mode.everything')
- };
+ get _searchModes() {
+ let modes = {
+ titleCreatorYear: Zotero.getString('quickSearch.mode.titleCreatorYear'),
+ fields: Zotero.getString('quickSearch.mode.fieldsAndTags'),
+ everything: Zotero.getString('quickSearch.mode.everything')
+ };
+ if (Zotero.Embeddings.isEnabled()) {
+ modes.similarity = Zotero.getString('quickSearch-mode-similarity');
+ }
+ return modes;
+ }
_searchModePopup = null;
@@ -122,8 +128,36 @@
}
});
wrapper.appendChild(advancedButton);
+ this._advancedButton = advancedButton;
}
-
+
+ // Dropdown selecting how many results the similarity mode keeps;
+ // shown in place of the Advanced Search button
+ let topKList = document.createXULElement('menulist');
+ topKList.id = 'zotero-tb-search-topk';
+ topKList.hidden = true;
+ document.l10n.setAttributes(topKList, 'quicksearch-semantic-topk');
+ let topKPopup = document.createXULElement('menupopup');
+ for (let n of [5, 10, 25, 50, 100]) {
+ let item = document.createXULElement('menuitem');
+ item.label = String(n);
+ item.value = String(n);
+ topKPopup.append(item);
+ }
+ topKList.append(topKPopup);
+ topKList.value = String(Zotero.Prefs.get('search.quicksearch-semantic-topK'));
+ topKList.addEventListener('command', (event) => {
+ // Don't trigger a quick search via the oncommand handler
+ event.stopPropagation();
+ Zotero.Prefs.set('search.quicksearch-semantic-topK', parseInt(topKList.value));
+ // Re-run the current search with the new top-K
+ if (this.value) {
+ this.dispatchEvent(new Event('command'));
+ }
+ });
+ wrapper.appendChild(topKList);
+ this._topKList = topKList;
+
this.deck = this.firstElementChild;
this.querySelector('.advanced-collapse-button').addEventListener('command', (event) => {
@@ -158,6 +192,15 @@
popup.id = "search-mode-popup";
popup.toggleAttribute("needsgutter", true);
+ this._populateSearchModePopup(popup);
+ // Rebuild the menu if the available modes changed since it was built
+ // (e.g. semantic search was enabled or disabled in the preferences)
+ popup.addEventListener('popupshowing', () => this._syncSearchModePopup());
+
+ return this._searchModePopup = popup;
+ }
+
+ _populateSearchModePopup(popup) {
for (let [mode, label] of Object.entries(this._searchModes)) {
let item = document.createXULElement('menuitem');
item.setAttribute('type', 'radio');
@@ -175,8 +218,22 @@
popup.append(item);
}
-
- return this._searchModePopup = popup;
+ }
+
+ _syncSearchModePopup() {
+ let popup = this._searchModePopup;
+ if (!popup) {
+ return;
+ }
+ let modes = Object.keys(this._searchModes);
+ let current = [...popup.children].map(item => item.value);
+ if (current.length === modes.length && current.every((mode, i) => mode === modes[i])) {
+ return;
+ }
+ popup.replaceChildren();
+ this._populateSearchModePopup(popup);
+ let active = Zotero.Prefs.get('search.quicksearch-mode');
+ popup.querySelector(`menuitem[value="${active}"]`)?.setAttribute('checked', 'true');
}
onCollectionSelected() {
@@ -192,10 +249,19 @@
mode = 'fields';
}
+ this._syncSearchModePopup();
this.searchModePopup.querySelector(`menuitem[value="${mode}"]`)
.setAttribute('checked', 'true');
document.l10n.setAttributes(this.searchTextbox.inputField, "quicksearch-input", { placeholder: this._searchModes[mode] });
+ // Advanced Search doesn't apply to semantic search, so swap its
+ // button for the similarity result-count dropdown
+ let isSimilarity = mode === 'similarity';
+ if (this._advancedButton) {
+ this._advancedButton.hidden = isSimilarity;
+ }
+ this._topKList.hidden = !isSimilarity;
+
let advancedSearchDeck = document.getElementById('zotero-advanced-search-pane-deck');
if (advancedSearchDeck) {
let state = advancedSearchDeck.state;
diff --git a/chrome/content/zotero/preferences/preferences_advanced.js b/chrome/content/zotero/preferences/preferences_advanced.js
index 1021c26d27..05ae78d5aa 100644
--- a/chrome/content/zotero/preferences/preferences_advanced.js
+++ b/chrome/content/zotero/preferences/preferences_advanced.js
@@ -57,6 +57,92 @@ Zotero_Preferences.Advanced = {
document.getElementById('zotero-prefpane-advanced-enable-local-api').addEventListener('synctopreference', () => {
this.updateLocalAPIUI();
});
+
+ this.initSemanticSearch();
+ },
+
+
+ initSemanticSearch: function () {
+ // Populate the model menu from the model registry. The preferences
+ // code watches bound menulists for added items and selects the one
+ // matching the current pref value.
+ let modelPopup = document.querySelector('#semantic-search-model > menupopup');
+ for (let { name, l10nID } of Zotero.Embeddings.getAvailableModels()) {
+ let menuitem = document.createXULElement('menuitem');
+ menuitem.setAttribute('value', name);
+ document.l10n.setAttributes(menuitem, l10nID);
+ modelPopup.append(menuitem);
+ }
+
+ // Live progress updates from the background indexer
+ this._semanticSearchListener = status => this.updateSemanticSearchUI(status);
+ Zotero.Embeddings.Indexing.addProgressListener(this._semanticSearchListener);
+
+ document.getElementById('zotero-prefpane-advanced').addEventListener('unload', () => {
+ Zotero.Embeddings.Indexing.removeProgressListener(this._semanticSearchListener);
+ });
+
+ document.getElementById('semantic-search-resume').addEventListener('command', () => {
+ Zotero.Embeddings.Indexing.startIndexing();
+ });
+
+ document.getElementById('semantic-search-stop').addEventListener('command', () => {
+ Zotero.Embeddings.Indexing.stopIndexing();
+ });
+
+ // Render current state, then compute up-to-date per-library counts
+ this.updateSemanticSearchUI(Zotero.Embeddings.Indexing.getStatus());
+ Zotero.Embeddings.Indexing.refreshStatus();
+ },
+
+
+ updateSemanticSearchUI: function (status) {
+ let statusBox = document.getElementById('semantic-search-status');
+ statusBox.hidden = !status.enabled;
+ if (!status.enabled) {
+ return;
+ }
+
+ // Phase / status message
+ let phaseLabel = document.getElementById('semantic-search-phase');
+ let hasRemaining = status.libraries.some(lib => lib.indexed < lib.eligible);
+ if (status.error) {
+ document.l10n.setAttributes(phaseLabel, 'preferences-advanced-semantic-search-error', { error: status.error });
+ }
+ else if (status.phase === 'downloading') {
+ document.l10n.setAttributes(phaseLabel, 'preferences-advanced-semantic-search-downloading');
+ }
+ else if (status.phase === 'indexing') {
+ document.l10n.setAttributes(phaseLabel, 'preferences-advanced-semantic-search-indexing');
+ }
+ else {
+ document.l10n.setAttributes(phaseLabel,
+ (status.paused || hasRemaining)
+ ? 'preferences-advanced-semantic-search-paused'
+ : 'preferences-advanced-semantic-search-idle');
+ }
+
+ // Offer a manual restart when enabled but not currently indexing and
+ // indexing is stopped or there's outstanding work (or the last run
+ // errored out)
+ document.getElementById('semantic-search-resume').hidden
+ = status.indexing || !(status.error || status.paused || hasRemaining);
+
+ // Offer to stop indexing while it's running
+ document.getElementById('semantic-search-stop').hidden = !status.indexing;
+
+ // Per-library "indexed / total" counts
+ let grid = document.getElementById('semantic-search-libraries');
+ if (grid.childElementCount !== status.libraries.length * 2) {
+ grid.textContent = '';
+ for (let i = 0; i < status.libraries.length; i++) {
+ grid.append(document.createXULElement('label'), document.createXULElement('label'));
+ }
+ }
+ status.libraries.forEach((lib, i) => {
+ grid.children[i * 2].setAttribute('value', lib.name);
+ grid.children[i * 2 + 1].setAttribute('value', `${lib.indexed} / ${lib.eligible}`);
+ });
},
diff --git a/chrome/content/zotero/preferences/preferences_advanced.xhtml b/chrome/content/zotero/preferences/preferences_advanced.xhtml
index 0140a75edb..03eb77b692 100644
--- a/chrome/content/zotero/preferences/preferences_advanced.xhtml
+++ b/chrome/content/zotero/preferences/preferences_advanced.xhtml
@@ -292,7 +292,34 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/chrome/content/zotero/xpcom/collectionTreeRow.js b/chrome/content/zotero/xpcom/collectionTreeRow.js
index 62e39fc7d1..1e78304fde 100644
--- a/chrome/content/zotero/xpcom/collectionTreeRow.js
+++ b/chrome/content/zotero/xpcom/collectionTreeRow.js
@@ -377,7 +377,20 @@ Zotero.CollectionTreeRow.prototype.getSearchResults = async function (asTempTabl
if (!this._cachedResults) {
let s = await this.getSearchObject();
try {
- this._cachedResults = await s.search();
+ let results = await s.search();
+ // Similarity (semantic) quick search: re-rank the scoped results by
+ // similarity to the query and keep the top K (the
+ // search.quicksearch-semantic-topK pref). Assign the cache only once
+ // ranking succeeds, so a ranking failure (e.g. model not yet
+ // downloaded) doesn't leave the full unranked scope cached as the
+ // search result.
+ if (this.isSimilaritySearch()) {
+ results = await Zotero.Embeddings.rankItemIDs(
+ this.searchText, results,
+ { limit: Zotero.Prefs.get('search.quicksearch-semantic-topK') }
+ );
+ }
+ this._cachedResults = results;
}
catch (e) {
Zotero.logError(e);
@@ -493,9 +506,12 @@ Zotero.CollectionTreeRow.prototype.getSearchObject = async function (options = {
if (!options.unfiltered) {
// Add Quick Search unless advanced search is enabled
if (this.searchText && !this.advancedSearch) {
- let cond = 'quicksearch-'
- + (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode'));
- s2.addCondition(cond, 'contains', this.searchText);
+ let mode = this.searchMode || Zotero.Prefs.get('search.quicksearch-mode');
+ // The similarity mode isn't a SQL condition -- the base search returns
+ // the full scope and getSearchResults() re-ranks it semantically.
+ if (mode !== 'similarity') {
+ s2.addCondition('quicksearch-' + mode, 'contains', this.searchText);
+ }
}
if (this.tags) {
@@ -616,14 +632,26 @@ Zotero.CollectionTreeRow.prototype.clearCache = function () {
};
Zotero.CollectionTreeRow.prototype.setSearch = function (searchText, mode = null) {
- if (this.searchText === searchText && this.searchMode === mode) {
+ // Callers usually pass no mode (the active mode lives in the pref), so compare
+ // the effective mode from the last call: switching modes with unchanged text
+ // has to trigger a re-run -- as does changing the similarity top-K, which
+ // lives in its own pref. With no search text, neither matters.
+ let effectiveMode = mode || Zotero.Prefs.get('search.quicksearch-mode');
+ let topK = effectiveMode === 'similarity'
+ ? Zotero.Prefs.get('search.quicksearch-semantic-topK')
+ : null;
+ if (this.searchText === searchText
+ && (!searchText
+ || (this._effectiveSearchMode === effectiveMode && this._searchTopK === topK))) {
return false;
}
this.clearCache();
this.searchText = searchText;
this.searchMode = mode;
+ this._effectiveSearchMode = effectiveMode;
+ this._searchTopK = topK;
return true;
-}
+};
Zotero.CollectionTreeRow.prototype.setAdvancedSearch = function (advancedSearch) {
// Clearing an already-clear filter is a no-op. A passed search is always
@@ -692,6 +720,15 @@ Zotero.CollectionTreeRow.prototype.isSearchMode = function () {
}
}
+/**
+ * Whether an active quick search on this row is in the semantic similarity
+ * mode -- i.e. the results are a ranked top-K set rather than a plain filter
+ */
+Zotero.CollectionTreeRow.prototype.isSimilaritySearch = function () {
+ return !!this.searchText && !this.advancedSearch
+ && (this.searchMode || Zotero.Prefs.get('search.quicksearch-mode')) === 'similarity';
+};
+
Zotero.CollectionTreeRow.prototype.isSortable = function () {
return !this.isFeedsOrFeed() && !this.isRecentlyRead();
}
diff --git a/chrome/content/zotero/xpcom/embeddings.js b/chrome/content/zotero/xpcom/embeddings.js
new file mode 100644
index 0000000000..3372997091
--- /dev/null
+++ b/chrome/content/zotero/xpcom/embeddings.js
@@ -0,0 +1,1031 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2026 Corporation for Digital Scholarship
+ Vienna, Virginia, USA
+ https://www.zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see .
+
+ ***** END LICENSE BLOCK *****
+*/
+
+/**
+ *
+ * Zotero.Embeddings -- the embedding engine and its public face: model
+ * config + download, the inference worker (bundled transformers.js + ONNX
+ * Runtime, run off the main thread), embed*(), and rankItemIDs() for the
+ * search path.
+ *
+ * Zotero.Embeddings.Indexing -- everything that decides what gets embedded
+ * and keeps the itemEmbeddings table filled.
+ *
+ */
+Zotero.Embeddings = new function () {
+ // Key order is the display order in the preferences model menu
+ const MODELS = {
+ 'bge-small-en-v1.5': {
+ revision: 1,
+ // HF repo id -- also the transformers.js pipeline id and download URL basis.
+ modelId: 'Xenova/bge-small-en-v1.5',
+ // int8 dynamic quantization ('q8' -> onnx/model_quantized.onnx)
+ dtype: 'q8',
+ pooling: 'cls',
+ // bge prepends a retrieval instruction to queries; passages get none.
+ queryPrefix: 'Represent this sentence for searching relevant passages: ',
+ passagePrefix: '',
+ l10nID: 'preferences-advanced-semantic-search-english',
+ files: [
+ 'config.json',
+ 'tokenizer.json',
+ 'tokenizer_config.json',
+ 'special_tokens_map.json',
+ 'onnx/model_quantized.onnx',
+ ]
+ },
+ 'multilingual-e5-small': {
+ revision: 1,
+ modelId: 'Xenova/multilingual-e5-small',
+ dtype: 'q8',
+ pooling: 'mean',
+ queryPrefix: 'query: ',
+ passagePrefix: 'passage: ',
+ l10nID: 'preferences-advanced-semantic-search-multilingual',
+ files: [
+ 'config.json',
+ 'tokenizer.json',
+ 'tokenizer_config.json',
+ 'special_tokens_map.json',
+ 'onnx/model_quantized.onnx',
+ ]
+ }
+ };
+
+ // Written into the model directory once every file has been downloaded, so
+ // that isDownloaded() doesn't have to re-check each file on every call.
+ const MARKER_FILE = '.download-complete.json';
+
+ const SUBDIR = 'embeddings';
+
+ /**
+ * Name of the active model, from the global preference. Empty string means
+ * semantic search is disabled.
+ * @return {String}
+ */
+ this.getModelName = function () {
+ return Zotero.Prefs.get('embeddings.model') || '';
+ };
+
+ /**
+ * Whether semantic search is enabled (a known model is selected).
+ * @return {Boolean}
+ */
+ this.isEnabled = function () {
+ return Object.prototype.hasOwnProperty.call(MODELS, this.getModelName());
+ };
+
+ /**
+ * Available models, in display order, for the preferences UI.
+ *
+ * @return {Object[]} - [{ name, l10nID }]
+ */
+ this.getAvailableModels = function () {
+ return Object.entries(MODELS).map(([name, model]) => ({ name, l10nID: model.l10nID }));
+ };
+
+ /**
+ * Identity of the active embedding function: model name plus its revision.
+ * Any code change that alters the vectors a model produces (dtype, upstream
+ * weights, prefixes, pooling) must bump that model's `revision`, so that
+ * stored embeddings are detected as stale and reindexed (see
+ * Indexing._ensureIndexMatchesModel()).
+ * @return {String}
+ */
+ this.getModelVersion = function () {
+ return this.getModelName() + '/' + _getModel().revision;
+ };
+
+ /**
+ * Absolute path to the directory where the active model's files live.
+ * @return {String}
+ */
+ this.getModelDirectory = function () {
+ return PathUtils.join(Zotero.DataDirectory.dir, SUBDIR, this.getModelName());
+ };
+
+ function _getModel() {
+ let name = Zotero.Embeddings.getModelName();
+ let model = MODELS[name];
+ if (!model) {
+ throw new Error(`Unknown embeddings model '${name}'`);
+ }
+ return model;
+ }
+
+ /**
+ * Whether the active model has been fully downloaded and is ready to use.
+ *
+ * @return {Promise}
+ */
+ this.isDownloaded = async function () {
+ let model = _getModel();
+ let dir = this.getModelDirectory();
+ let markerPath = PathUtils.join(dir, MARKER_FILE);
+
+ let marker;
+ try {
+ marker = JSON.parse(await Zotero.File.getContentsAsync(markerPath));
+ }
+ catch {
+ return false;
+ }
+ // A revision bump invalidates a previously downloaded model
+ if (!marker || marker.revision !== model.revision) {
+ return false;
+ }
+ // Make sure the files are actually still there
+ for (let file of model.files) {
+ if (!(await IOUtils.exists(PathUtils.join(dir, ...file.split('/'))))) {
+ return false;
+ }
+ }
+ return true;
+ };
+
+ /**
+ * Download the active model's files into its directory. Files that already
+ * exist are skipped, so this can be safely called again to resume after a
+ * failure. On success a completion marker is written.
+ *
+ * @return {Promise} - Path to the model directory
+ */
+ this.download = async function () {
+ let model = _getModel();
+ let baseURL = `https://huggingface.co/${model.modelId}/resolve/main/`;
+ let dir = this.getModelDirectory();
+
+ if (await this.isDownloaded()) {
+ Zotero.debug(`Embeddings: model '${this.getModelName()}' already downloaded`);
+ return dir;
+ }
+
+ Zotero.debug(`Embeddings: downloading model '${this.getModelName()}' from ${baseURL}`);
+ await Zotero.File.createDirectoryIfMissingAsync(dir, { from: Zotero.DataDirectory.dir });
+
+ for (let file of model.files) {
+ await _downloadFile(baseURL + file, dir, file);
+ }
+
+ // TODO: verify file integrity against sha256 hashes from a signed
+ // manifest once the model is mirrored on Zotero infrastructure
+
+ await Zotero.File.putContentsAsync(
+ PathUtils.join(dir, MARKER_FILE),
+ JSON.stringify({
+ model: this.getModelName(),
+ revision: model.revision,
+ files: model.files
+ })
+ );
+
+ Zotero.debug(`Embeddings: model '${this.getModelName()}' downloaded to ${dir}`);
+ return dir;
+ };
+
+ /**
+ * Delete downloaded model directories other than the active one (all of them
+ * when disabled), freeing disk space after a model switch.
+ *
+ * @return {Promise}
+ */
+ this.pruneModels = async function () {
+ let root = PathUtils.join(Zotero.DataDirectory.dir, SUBDIR);
+ let keep = this.isEnabled() ? this.getModelName() : null;
+ let children;
+ try {
+ children = await IOUtils.getChildren(root);
+ }
+ catch {
+ return; // directory doesn't exist yet
+ }
+ for (let path of children) {
+ if (PathUtils.filename(path) === keep) {
+ continue;
+ }
+ let stat = await IOUtils.stat(path);
+ if (stat.type === 'directory') {
+ await IOUtils.remove(path, { recursive: true, ignoreAbsent: true });
+ }
+ }
+ };
+
+ //
+ // Embedding generation, via a bundled transformers.js + ONNX Runtime
+ // (resource://zotero/embeddings/) running in a worker. The model files are
+ // read from the data directory (see download()) and handed to the worker,
+ // which serves them to transformers.js through an in-memory custom cache.
+ //
+
+ const WORKER_URL = 'resource://zotero/embeddings/worker.js';
+ const RESOURCE_DIR = 'resource://zotero/embeddings/';
+ const WASM_FILE = 'ort-wasm-simd-threaded.jsep.wasm';
+
+ let _worker = null;
+ let _workerReady = null;
+ let _requestID = 0;
+ let _pending = new Map();
+
+ function _ensureWorker() {
+ if (_worker) {
+ return;
+ }
+ _worker = new Worker(WORKER_URL, { type: 'module' });
+ _worker.addEventListener('message', (event) => {
+ let { id, result, error } = event.data;
+ let promise = _pending.get(id);
+ if (!promise) {
+ return;
+ }
+ _pending.delete(id);
+ if (error) {
+ let err = new Error(error.message || error);
+ if (error.name) {
+ err.name = error.name;
+ }
+ if (error.stack) {
+ // Preserve the worker-side stack for debugging
+ err.stack = error.stack;
+ }
+ promise.reject(err);
+ }
+ else {
+ promise.resolve(result);
+ }
+ });
+ // A worker-level error (e.g. a crash or OOM while loading the model) won't
+ // produce a per-request reply, so fail any in-flight requests instead of
+ // letting them hang forever, and drop the dead worker so the next call
+ // rebuilds it.
+ _worker.addEventListener('error', (event) => {
+ let err = new Error(`Embeddings worker error: ${event.message || 'unknown (worker may have crashed)'}`);
+ Zotero.logError(err);
+ _failPending(err);
+ _worker = null;
+ _workerReady = null;
+ });
+ _worker.addEventListener('messageerror', () => {
+ _failPending(new Error('Embeddings worker message could not be deserialized'));
+ });
+ }
+
+ function _failPending(err) {
+ for (let { reject } of _pending.values()) {
+ reject(err);
+ }
+ _pending.clear();
+ }
+
+ function _post(action, data, transfer = []) {
+ let id = ++_requestID;
+ return new Promise((resolve, reject) => {
+ _pending.set(id, { resolve, reject });
+ _worker.postMessage({ id, action, data }, transfer);
+ });
+ }
+
+ // Create the worker if needed and initialize it with the model files (read
+ // from the data directory) and the bundled ORT wasm binary. The model must
+ // already be downloaded.
+ async function _getWorker() {
+ _ensureWorker();
+ if (!_workerReady) {
+ _workerReady = (async () => {
+ let model = _getModel();
+ if (!(await Zotero.Embeddings.isDownloaded())) {
+ throw new Error(`Embeddings model '${Zotero.Embeddings.getModelName()}' is not downloaded`);
+ }
+ let dir = Zotero.Embeddings.getModelDirectory();
+
+ // Read the model files from the data directory, transferring the
+ // buffers to the worker to avoid copies
+ let files = [];
+ let transfer = [];
+ for (let rel of model.files) {
+ let path = PathUtils.join(dir, ...rel.split('/'));
+ let bytes = await IOUtils.read(path);
+ files.push([rel, bytes.buffer]);
+ transfer.push(bytes.buffer);
+ }
+
+ // Read the bundled ONNX runtime wasm binary
+ let wasm = await Zotero.HTTP.request('GET', RESOURCE_DIR + WASM_FILE, {
+ responseType: 'arraybuffer'
+ });
+ transfer.push(wasm.response);
+
+ Zotero.debug(`Embeddings: initializing worker for '${model.modelId}' (dtype ${model.dtype}, pooling ${model.pooling})`);
+ await _post('init', {
+ modelId: model.modelId,
+ dtype: model.dtype,
+ pooling: model.pooling,
+ files,
+ wasmPaths: RESOURCE_DIR,
+ wasmBinary: wasm.response
+ }, transfer);
+ Zotero.debug('Embeddings: worker initialized');
+ })();
+ }
+ try {
+ await _workerReady;
+ }
+ catch (e) {
+ // Allow a later retry after a failed initialization
+ _workerReady = null;
+ throw e;
+ }
+ return _worker;
+ }
+
+ /**
+ * Prepare embeddings ahead of time: download the model if needed, then spin
+ * up and initialize the worker so the first embed() is fast.
+ *
+ * @return {Promise} Resolves when the worker is ready to embed
+ */
+ this.preloadModel = async function () {
+ await this.download();
+ await _getWorker();
+ };
+
+ /**
+ * Shut down the embeddings worker, rejecting any in-flight requests.
+ */
+ this.shutdownEngine = function () {
+ if (_worker) {
+ _worker.terminate();
+ _worker = null;
+ }
+ _workerReady = null;
+ _failPending(new Error('Embeddings worker shut down'));
+ };
+
+ /**
+ * Embed an arbitrary string, returning the model's vector. Task prefixes
+ * ("query: ", "passage: ") are NOT added here -- add them at the call site
+ * if the model expects them.
+ *
+ * @param {String} text
+ * @return {Promise}
+ */
+ this.embed = async function (text) {
+ let vectors = await this.embedMany([text]);
+ return vectors[0];
+ };
+
+ /**
+ * Embed multiple strings in a single engine call.
+ *
+ * @param {String[]} texts
+ * @return {Promise}
+ */
+ this.embedMany = async function (texts) {
+ if (!texts.length) {
+ return [];
+ }
+ await _getWorker();
+ Zotero.debug(`Embeddings: embedding batch of ${texts.length}`);
+ let { data, dims } = await _post('embed', { texts });
+ Zotero.debug(`Embeddings: batch of ${texts.length} done`);
+ return _toVectors(data, dims);
+ };
+
+ /**
+ * Embed a search query string, applying the active model's query prefix.
+ *
+ * @param {String} text
+ * @return {Promise}
+ */
+ this.embedQuery = function (text) {
+ return this.embed(_getModel().queryPrefix + text);
+ };
+
+ /**
+ * Embed passages (item texts), applying the active model's passage prefix.
+ *
+ * @param {String[]} texts
+ * @return {Promise}
+ */
+ this.embedPassages = function (texts) {
+ let passagePrefix = _getModel().passagePrefix;
+ return this.embedMany(texts.map(text => passagePrefix + text));
+ };
+
+ // Reshape the worker's flat Float32 output (a tensor of shape
+ // [count, dimensions]) into one Float32Array per input text.
+ function _toVectors(data, dims) {
+ let dim = dims[dims.length - 1];
+ let count = data.length / dim;
+ let vectors = [];
+ for (let i = 0; i < count; i++) {
+ vectors.push(data.slice(i * dim, (i + 1) * dim));
+ }
+ return vectors;
+ }
+
+
+ // mozStorage returns a BLOB as an array of byte values; reinterpret those
+ // bytes as the stored Float32 embedding vector.
+ function _blobToVector(blob) {
+ let bytes = Uint8Array.from(blob);
+ return new Float32Array(bytes.buffer);
+ }
+
+ /**
+ * Rank a given set of items by similarity to a query, returning the most
+ * similar item IDs (highest first). Items without a stored embedding are
+ * dropped. Used to apply semantic ranking within an existing result scope
+ * (e.g. the current collection) rather than the whole library.
+ *
+ * @param {String} queryText
+ * @param {Number[]} itemIDs - Candidate item IDs to rank
+ * @param {Object} [options]
+ * @param {Number} [options.limit] - Keep only this many top results
+ * @return {Promise} - Ranked item IDs, most similar first
+ */
+ this.rankItemIDs = async function (queryText, itemIDs, { limit } = {}) {
+ if (!itemIDs.length || !this.isEnabled()) {
+ return [];
+ }
+ let query = await this.embedQuery(queryText);
+ let dim = query.length;
+
+ // Load embeddings for the candidates in chunks (avoids the SQLite bound-
+ // parameter limit for large collections), scoring each as we go.
+ let scored = [];
+ let chunkSize = 500;
+ for (let i = 0; i < itemIDs.length; i += chunkSize) {
+ let chunk = itemIDs.slice(i, i + chunkSize);
+ let rows = await Zotero.DB.queryAsync(
+ "SELECT itemID, embedding FROM itemEmbeddings WHERE itemID IN ("
+ + chunk.map(() => '?').join(',') + ")",
+ chunk
+ );
+ for (let row of rows) {
+ let vec = _blobToVector(row.embedding);
+ let dot = 0;
+ for (let d = 0; d < dim; d++) {
+ dot += query[d] * vec[d];
+ }
+ scored.push({ itemID: row.itemID, score: dot });
+ }
+ }
+
+ scored.sort((a, b) => b.score - a.score);
+ if (limit !== undefined) {
+ scored = scored.slice(0, limit);
+ }
+ return scored.map(s => s.itemID);
+ };
+
+ /**
+ * Download a single model file to a temp path and move it into place
+ * atomically, so an interrupted download never looks like a complete file.
+ * Files that already exist are skipped.
+ */
+ async function _downloadFile(url, dir, file) {
+ let parts = file.split('/');
+ let destPath = PathUtils.join(dir, ...parts);
+
+ // Already have it from a previous (interrupted) run
+ if (await IOUtils.exists(destPath)) {
+ return;
+ }
+
+ // Create the file's parent directory (e.g. onnx/) if needed
+ if (parts.length > 1) {
+ await Zotero.File.createDirectoryIfMissingAsync(PathUtils.parent(destPath), { from: dir });
+ }
+
+ let tmpPath = destPath + '.tmp';
+ await Zotero.HTTP.download(url, tmpPath);
+ await IOUtils.move(tmpPath, destPath);
+ }
+};
+
+
+/**
+ * Background indexing flow for semantic search: a single queue of itemIDs
+ * drained by one consumer loop.
+ * Inference runs in the worker; batches yield between DB writes.
+ * Also switches models when the extensions.zotero.embeddings.model preference
+ * changes. Progress is reported per library via listeners.
+ */
+Zotero.Embeddings.Indexing = new function () {
+ let _initialized = false;
+ let _indexing = false;
+ let _indexingPromise = null;
+ let _stopping = false;
+ let _phase = 'idle'; // 'idle' | 'downloading' | 'indexing'
+ let _lastError = null;
+ let _status = new Map(); // libraryID -> { name, indexed, eligible }
+ let _progressListeners = new Set();
+ let _lastStatusRefresh = 0;
+
+ // The indexing queue. Producers (the item notifier, startIndexing()) only
+ // add itemIDs here; _run() is the single consumer that embeds them.
+ let _queue = new Set();
+ let _kickTimer = null;
+ // Debounce before starting the consumer, so a burst of changes (e.g. an
+ // import) is picked up in one pass
+ const KICK_DELAY = 3000;
+ // itemIDs pulled off the queue per consumer iteration. Large enough that
+ // sorting by text length within the chunk (see indexItems()) produces
+ // well-packed batches
+ const CHUNK_SIZE = 256;
+
+ // Serialize model switches so rapid preference changes don't run their
+ // clear/prune/re-index steps concurrently.
+ let _switchChain = Promise.resolve();
+
+ /**
+ * Wire up the background indexer. Guarded so multiple windows don't
+ * double-initialize.
+ */
+ this.init = function () {
+ if (_initialized) {
+ return;
+ }
+ _initialized = true;
+
+ // Changing the model preference (including to/from "" = Disabled) switches
+ // models: clear old embeddings + files and re-index with the new one.
+ Zotero.Prefs.registerObserver('embeddings.model', () => {
+ _switchModel();
+ });
+
+ Zotero.Notifier.registerObserver({
+ notify: (event, type, ids) => {
+ if (type !== 'item' || !Zotero.Embeddings.isEnabled()
+ || Zotero.Embeddings.Indexing.isPaused()) {
+ return;
+ }
+ if (event === 'add' || event === 'modify') {
+ for (let id of ids) {
+ _queue.add(id);
+ }
+ _scheduleKick();
+ }
+ }
+ }, ['item'], 'embeddings');
+
+ // Resume indexing if a model is selected and indexing wasn't explicitly
+ // stopped, after a short delay so we don't compete with window setup.
+ if (Zotero.Embeddings.isEnabled() && !this.isPaused()) {
+ Zotero.Promise.delay(5000).then(() => {
+ if (Zotero.Embeddings.isEnabled() && !Zotero.Embeddings.Indexing.isPaused()) {
+ Zotero.Embeddings.Indexing.startIndexing();
+ }
+ });
+ }
+ };
+
+ /**
+ * Whether indexing has been explicitly stopped. While paused, nothing is
+ * indexed at all -- item changes aren't even enqueued -- until
+ * startIndexing() is called again. Persisted so a stop survives a restart.
+ * @return {Boolean}
+ */
+ this.isPaused = function () {
+ return !!Zotero.Prefs.get('embeddings.indexingPaused');
+ };
+
+ function _switchModel() {
+ _switchChain = _switchChain.then(() => _doSwitchModel()).catch(e => Zotero.logError(e));
+ return _switchChain;
+ }
+
+ async function _doSwitchModel() {
+ // Stop any in-progress indexing and wait for it to actually finish before
+ // touching the stored vectors, so we never mix models/dimensions.
+ Zotero.Embeddings.Indexing.stopIndexing();
+ if (_indexingPromise) {
+ try {
+ await _indexingPromise;
+ }
+ catch (e) {
+ Zotero.logError(e);
+ }
+ }
+ Zotero.Embeddings.shutdownEngine();
+
+ // Different models produce different-dimension vectors, so all stored
+ // embeddings are wiped on any switch -- including disabling, which drops
+ // the index and the downloaded files entirely. Then drop every downloaded
+ // model except the newly-selected one.
+ await _clearEmbeddings();
+ await Zotero.Embeddings.pruneModels();
+ _status.clear();
+
+ if (Zotero.Embeddings.isEnabled()) {
+ Zotero.Embeddings.Indexing.startIndexing();
+ }
+ else {
+ _emitProgress();
+ }
+ }
+
+ // Debounce before starting the consumer, so a burst of changes (e.g. an
+ // import) is picked up in one pass. Enqueued ids just sit in the queue
+ // until the consumer runs.
+ function _scheduleKick() {
+ if (_kickTimer) {
+ clearTimeout(_kickTimer);
+ }
+ _kickTimer = setTimeout(() => {
+ _kickTimer = null;
+ _startConsumer();
+ }, KICK_DELAY);
+ }
+
+ // Start the consumer loop if there's work and it isn't already running
+ function _startConsumer() {
+ if (_indexing) {
+ return _indexingPromise;
+ }
+ if (!_queue.size || !Zotero.Embeddings.isEnabled()
+ || Zotero.Embeddings.Indexing.isPaused()) {
+ return Promise.resolve();
+ }
+ _indexingPromise = _run();
+ return _indexingPromise;
+ }
+
+ // The single definition of an eligible item: a regular item with a
+ // non-empty title (including type-specific title fields -- caseName,
+ // subject, nameOfAct) or abstract, in any library.
+ // Everything that needs to know what's eligible -- enqueueing, pruning,
+ // progress counts -- works from this result.
+ //
+ // @return {Promise