diff --git a/chrome/content/zotero/collectionViewItemTree.jsx b/chrome/content/zotero/collectionViewItemTree.jsx
index 1415859e66..b7432fe5cf 100644
--- a/chrome/content/zotero/collectionViewItemTree.jsx
+++ b/chrome/content/zotero/collectionViewItemTree.jsx
@@ -206,8 +206,13 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
);
let libraries = status.libraries
.filter(lib => !libraryIDs.size || libraryIDs.has(lib.libraryID));
- let indexed = libraries.reduce((sum, lib) => sum + lib.indexed, 0);
- let total = libraries.reduce((sum, lib) => sum + lib.eligible, 0);
+ // Coverage is coverage: attachment fulltext is reported separately
+ // in the preferences, but an incomplete index is incomplete
+ // whichever part of it is still filling in
+ let indexed = libraries.reduce(
+ (sum, lib) => sum + lib.indexed + lib.indexedAttachments, 0);
+ let total = libraries.reduce(
+ (sum, lib) => sum + lib.eligible + lib.eligibleAttachments, 0);
if (indexed >= total) {
return null;
}
diff --git a/chrome/content/zotero/customElements.js b/chrome/content/zotero/customElements.js
index 59b5b319bc..03f2108a53 100644
--- a/chrome/content/zotero/customElements.js
+++ b/chrome/content/zotero/customElements.js
@@ -74,6 +74,8 @@ Services.scriptloader.loadSubScript('chrome://zotero/content/elements/itemTreeMe
['attachment-row', 'chrome://zotero/content/elements/attachmentRow.js'],
['attachment-annotations-box', 'chrome://zotero/content/elements/attachmentAnnotationsBox.js'],
['annotation-row', 'chrome://zotero/content/elements/annotationRow.js'],
+ ['search-results-box', 'chrome://zotero/content/elements/searchResultsBox.js'],
+ ['search-result-row', 'chrome://zotero/content/elements/searchResultRow.js'],
['annotation-items-pane', 'chrome://zotero/content/elements/annotationItemsPane.js'],
['context-notes-list', 'chrome://zotero/content/elements/contextNotesList.js'],
['note-row', 'chrome://zotero/content/elements/noteRow.js'],
diff --git a/chrome/content/zotero/elements/itemDetails.js b/chrome/content/zotero/elements/itemDetails.js
index 2e9a705f0a..4fbcd62137 100644
--- a/chrome/content/zotero/elements/itemDetails.js
+++ b/chrome/content/zotero/elements/itemDetails.js
@@ -55,6 +55,8 @@
+
+
diff --git a/chrome/content/zotero/elements/itemPaneSidenav.js b/chrome/content/zotero/elements/itemPaneSidenav.js
index 8da724e7da..da0b3539e3 100644
--- a/chrome/content/zotero/elements/itemPaneSidenav.js
+++ b/chrome/content/zotero/elements/itemPaneSidenav.js
@@ -102,7 +102,7 @@
}
get _builtInPanes() {
- return ["info", "abstract", "attachments", "notes", "note-info", "attachment-info", "attachment-annotations", "libraries-collections", "tags", "related"];
+ return ["info", "abstract", "attachments", "notes", "note-info", "attachment-info", "attachment-annotations", "libraries-collections", "tags", "related", "search-results"];
}
get container() {
diff --git a/chrome/content/zotero/elements/searchResultRow.js b/chrome/content/zotero/elements/searchResultRow.js
new file mode 100644
index 0000000000..9cb6a5d778
--- /dev/null
+++ b/chrome/content/zotero/elements/searchResultRow.js
@@ -0,0 +1,131 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2026 Corporation for Digital Scholarship
+ Vienna, Virginia, USA
+ https://www.zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see .
+
+ ***** END LICENSE BLOCK *****
+*/
+
+"use strict";
+
+{
+ // A best-match search result card: one fulltext chunk of an attachment
+ // (see Zotero.Embeddings.getMatchingChunks()), presented like an
+ // annotation-row -- where the chunk sits in the document as the head,
+ // its text as the quote. The two share their styling (see
+ // scss/elements/_annotationRow.scss).
+ class SearchResultRow extends XULElementBase {
+ content = MozXULElement.parseXULToFragment(`
+
+
+
+
+
+
+
+
+
+
+
+ `);
+
+ _chunk = null;
+
+ get chunk() {
+ return this._chunk;
+ }
+
+ set chunk(chunk) {
+ this._chunk = chunk;
+ this.render();
+ }
+
+ init() {
+ this._path = this.querySelector('.path');
+ this._part = this.querySelector('.part');
+ this._location = this.querySelector('.location');
+ this._quote = this.querySelector('.quote');
+ this._showMore = this.querySelector('.show-more');
+ this._showMore.addEventListener('click', (event) => {
+ // The card's activation (open the attachment) shouldn't fire
+ // for the toggle
+ event.stopPropagation();
+ this._toggleExpanded();
+ });
+ this.render();
+ }
+
+ render() {
+ if (!this.initialized || !this._chunk) return;
+
+ // The chunk's outline path says where in the document it came
+ // from; a chunk from a document without an outline falls back to
+ // a generic label
+ if (this._chunk.outlinePath) {
+ this._path.removeAttribute('data-l10n-id');
+ this._path.textContent = this._chunk.outlinePath;
+ }
+ else {
+ document.l10n.setAttributes(this._path, 'search-result-row-fulltext');
+ }
+ // Which piece of a split section this is, so a match reads as
+ // coming from the middle or the end of its section
+ let parts = this._chunk.sectionParts;
+ this._part.hidden = !(parts > 1);
+ if (parts > 1) {
+ this._part.textContent = `${this._chunk.sectionPart}/${parts}`;
+ }
+ // The page the chunk's section starts on, labeled the way
+ // annotation rows label theirs
+ this._location.hidden = !this._chunk.pageLabel;
+ if (this._chunk.pageLabel) {
+ this._location.textContent
+ = Zotero.getString('pdfReader.page') + ' ' + this._chunk.pageLabel;
+ }
+
+ this._quote.textContent = this._chunk.text || '';
+
+ // Offer "Show More" only when the quote is actually clamped,
+ // which is only measurable once the card has a layout
+ this.classList.remove('expanded');
+ this._showMore.hidden = true;
+ requestAnimationFrame(() => {
+ this._showMore.hidden
+ = this._quote.scrollHeight <= this._quote.clientHeight;
+ });
+
+ // A11y - make focusable and describe the card
+ this.setAttribute('tabindex', 0);
+ this.setAttribute('aria-label', [
+ this._chunk.outlinePath,
+ this._location.hidden ? '' : this._location.textContent,
+ this._chunk.text
+ ].filter(Boolean).join('. '));
+ }
+
+ _toggleExpanded() {
+ let expanded = this.classList.toggle('expanded');
+ document.l10n.setAttributes(this._showMore,
+ expanded ? 'search-result-row-show-less' : 'search-result-row-show-more');
+ }
+ }
+
+ customElements.define('search-result-row', SearchResultRow);
+}
diff --git a/chrome/content/zotero/elements/searchResultsBox.js b/chrome/content/zotero/elements/searchResultsBox.js
new file mode 100644
index 0000000000..0afa3fde28
--- /dev/null
+++ b/chrome/content/zotero/elements/searchResultsBox.js
@@ -0,0 +1,192 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2026 Corporation for Digital Scholarship
+ Vienna, Virginia, USA
+ https://www.zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see .
+
+ ***** END LICENSE BLOCK *****
+*/
+
+{
+ const { ItemPaneSectionElementBase } = ChromeUtils.importESModule(
+ "chrome://zotero/content/elements/itemPaneSectionElementBase.mjs",
+ { global: "current" }
+ );
+
+ // Most matching chunks shown for an attachment
+ const MAX_RESULTS = 5;
+
+ // Why an attachment matched the active best-match search: cards with the
+ // fulltext chunks most similar to the query, so they can be previewed
+ // without opening the file. Shown only while a best-match search is
+ // active, for attachments with matching indexed chunks.
+ class SearchResultsBox extends ItemPaneSectionElementBase {
+ content = MozXULElement.parseXULToFragment(`
+
+
+
+
+ `);
+
+ get item() {
+ return this._item;
+ }
+
+ set item(item) {
+ super.item = (item instanceof Zotero.Item && item.isFileAttachment()) ? item : null;
+ // A new item's emptiness isn't known until asyncRender scores it
+ this._count = undefined;
+ }
+
+ get collectionTreeRows() {
+ return super.collectionTreeRows;
+ }
+
+ // The item pane sets collectionTreeRows after item, so this is where
+ // everything visibility depends on is finally known
+ set collectionTreeRows(collectionTreeRows) {
+ super.collectionTreeRows = collectionTreeRows;
+ this._updateHidden();
+ }
+
+ init() {
+ this.initCollapsibleSection();
+ this._body = this.querySelector('.body');
+ // The header's count placeholder needs a value before the first
+ // async render fills in the real one
+ this._section.setCount(0);
+ // Double-click (or Enter on a focused card) opens the attachment
+ // at the chunk
+ this._body.addEventListener('dblclick', this._handleActivate);
+ this._body.addEventListener('keydown', (event) => {
+ if (event.key == 'Enter') {
+ this._handleActivate(event);
+ }
+ });
+ }
+
+ // The query the selected collection rows are ranked by, or false when
+ // no best-match search is active. Rows can be duck-typed stand-ins
+ // (e.g. the citation dialog's), which implement only part of the row
+ // API.
+ get _query() {
+ // Quick-search state (setSearch()) lives on collection tree row
+ // *instances*, and the ones passed down the item pane are
+ // re-fetched from the collections view at item-selection time --
+ // which can have rebuilt its rows since the items view got its
+ // own set. Only the items view's instances are guaranteed to
+ // carry the active search, so prefer those; the passed rows are
+ // the fallback for hosts without an items view.
+ let itemsView = this.closest('item-pane')?.itemsView;
+ let rows = itemsView?.collectionTreeRows?.length
+ ? itemsView.collectionTreeRows
+ : this.collectionTreeRows;
+ for (let row of rows || []) {
+ if (typeof row.getBestMatchQuery == 'function') {
+ let query = row.getBestMatchQuery();
+ if (query) {
+ return query;
+ }
+ }
+ }
+ return false;
+ }
+
+ // A query change re-renders even when the item didn't change
+ get _renderDependencies() {
+ return [...super._renderDependencies, this._query];
+ }
+
+ render() {}
+
+ async asyncRender() {
+ if (!this.initialized) return;
+ if (this._isAlreadyRendered("async")) return;
+
+ let item = this.item;
+ let query = this._query;
+ this._body.replaceChildren();
+ if (!item || !query) {
+ return;
+ }
+
+ let chunks = [];
+ try {
+ chunks = await Zotero.Embeddings.getMatchingChunks(query, item.id,
+ { limit: MAX_RESULTS });
+ }
+ catch (e) {
+ // Nothing to show while the model is still downloading or the
+ // index is being rebuilt
+ if (!(e instanceof Zotero.Embeddings.IndexNotReadyError)) {
+ Zotero.logError(e);
+ }
+ }
+ // Only fulltext chunks carry their own text; anything else about
+ // the item is already visible in the pane
+ chunks = chunks.filter(chunk => chunk.text);
+ // The selection may have moved on while scoring
+ if (this.item !== item) {
+ return;
+ }
+ this._count = chunks.length;
+ this._section.setCount(chunks.length);
+ this._updateHidden();
+ // Left in the order getMatchingChunks() returns them, strongest
+ // match first: with only a handful of cards shown, the best one
+ // earning the top slot matters more than reading them in
+ // document order
+ for (let chunk of chunks) {
+ let row = document.createXULElement('search-result-row');
+ row.chunk = chunk;
+ this._body.append(row);
+ }
+ }
+
+ // Open the attachment where the activated card's chunk is: for a PDF,
+ // scrolled to and highlighting the chunk's section; without a stored
+ // position (EPUB, snapshot, flat-text fallback), just open it
+ _handleActivate = (event) => {
+ let row = event.target.closest('search-result-row');
+ // The Show More toggle isn't an activation
+ if (!row || !this.item || event.target.closest('.show-more')) {
+ return;
+ }
+ if (typeof ZoteroPane == 'undefined') {
+ return;
+ }
+ let position = row.chunk?.position;
+ ZoteroPane.viewAttachment(this.item.id, null, false,
+ position ? { location: { position } } : undefined)
+ .catch(e => Zotero.logError(e));
+ };
+
+ _updateHidden() {
+ // Visible only for a file attachment during a best-match search;
+ // asyncRender hides it again when nothing matched. Deciding
+ // emptiness needs the async scoring, so unlike the annotations
+ // section this one can't know its final state synchronously --
+ // it appears, then empties out, rather than flickering in late.
+ this.hidden = !this.item || !this._query || this.tabType == 'reader'
+ || this._count === 0;
+ }
+ }
+
+ customElements.define("search-results-box", SearchResultsBox);
+}
diff --git a/chrome/content/zotero/preferences/preferences_advanced.js b/chrome/content/zotero/preferences/preferences_advanced.js
index 69d53c69f4..23097cf7c2 100644
--- a/chrome/content/zotero/preferences/preferences_advanced.js
+++ b/chrome/content/zotero/preferences/preferences_advanced.js
@@ -152,13 +152,17 @@ Zotero_Preferences.Advanced = {
updateSemanticSearchUI: function (status) {
let statusBox = document.getElementById('semantic-search-status');
statusBox.hidden = !status.enabled;
+ // Fulltext indexing only means something with a model selected
+ document.getElementById('semantic-search-index-fulltext').disabled = !status.enabled;
if (!status.enabled) {
return;
}
// Phase / status message
let phaseLabel = document.getElementById('semantic-search-phase');
- let hasRemaining = status.libraries.some(lib => lib.indexed < lib.eligible);
+ let hasRemaining = status.libraries.some(
+ lib => lib.indexed < lib.eligible
+ || lib.indexedAttachments < lib.eligibleAttachments);
if (status.error) {
document.l10n.setAttributes(phaseLabel, 'preferences-advanced-semantic-search-error', { error: status.error });
}
@@ -214,10 +218,18 @@ Zotero_Preferences.Advanced = {
'value',
Zotero.Utilities.Internal.stringWithColon(lib.name)
);
- grid.children[i * 2 + 1].setAttribute(
- 'value',
- `${lib.indexed.toLocaleString()} / ${lib.eligible.toLocaleString()}`
- );
+ // Attachment fulltext is reported on its own, since it's a much
+ // larger and much slower job than the rest -- one combined count
+ // would look stalled. With fulltext indexing off, none are
+ // eligible and only the item count is shown.
+ let counts = `${lib.indexed.toLocaleString()} / ${lib.eligible.toLocaleString()}`;
+ if (lib.eligibleAttachments) {
+ counts += ` ${Zotero.getString('general.and')} `
+ + `${lib.indexedAttachments.toLocaleString()} / `
+ + `${lib.eligibleAttachments.toLocaleString()} `
+ + Zotero.getString('itemTypes.attachment');
+ }
+ grid.children[i * 2 + 1].setAttribute('value', counts);
});
},
diff --git a/chrome/content/zotero/preferences/preferences_advanced.xhtml b/chrome/content/zotero/preferences/preferences_advanced.xhtml
index 7a66fa47d1..3617a6bab2 100644
--- a/chrome/content/zotero/preferences/preferences_advanced.xhtml
+++ b/chrome/content/zotero/preferences/preferences_advanced.xhtml
@@ -307,6 +307,10 @@
+
diff --git a/chrome/content/zotero/xpcom/embeddings.js b/chrome/content/zotero/xpcom/embeddings.js
index 6100019982..097871facc 100644
--- a/chrome/content/zotero/xpcom/embeddings.js
+++ b/chrome/content/zotero/xpcom/embeddings.js
@@ -378,12 +378,34 @@ Zotero.Embeddings = new function () {
// Zotero.Embeddings.Chunking);
// every chunk row carries the hash of the item's full source text,
// and scoring takes the item's best chunk.
+ // An attachment fulltext chunk also records what and where it
+ // came from, so a match can be previewed and located without
+ // re-deriving the chunking: its text, its section's outline path,
+ // the top-level block range it covers, the page it starts on
+ // (pageLabel), a reader-navigable position (navPosition, JSON --
+ // see Zotero.SDT.getSections()), and which piece of a split
+ // section it is (sectionPart of sectionParts). All NULL for
+ // chunks of other item types, which are their own preview and
+ // location.
+ // An attachment that yields no text at all (missing file,
+ // password-protected, no text layer) gets a single row with a
+ // NULL embedding: a record that it was processed, so progress
+ // counts it and later passes skip it (via sourceHash) until the
+ // file changes. Scoring reads only rows with an embedding.
await Zotero.DB.queryAsync(
"CREATE TABLE embeddings.itemEmbeddings (\n"
+ " itemID INTEGER NOT NULL,\n"
+ " chunkIndex INTEGER NOT NULL,\n"
- + " embedding BLOB NOT NULL,\n"
+ + " embedding BLOB,\n"
+ " sourceHash TEXT NOT NULL,\n"
+ + " chunkText TEXT,\n"
+ + " outlinePath TEXT,\n"
+ + " startBlock INTEGER,\n"
+ + " endBlock INTEGER,\n"
+ + " pageLabel TEXT,\n"
+ + " navPosition TEXT,\n"
+ + " sectionPart INTEGER,\n"
+ + " sectionParts INTEGER,\n"
+ " PRIMARY KEY (itemID, chunkIndex)\n"
+ ")"
);
@@ -1095,6 +1117,39 @@ Zotero.Embeddings = new function () {
return new Float32Array(bytes.buffer);
}
+ // The shared guards of the scoring paths: wait out any in-progress model
+ // switch, so a query isn't embedded with one model and compared against
+ // another's vectors, and confirm the stored vectors were produced by the
+ // active model -- during a switch, or a reindex after a revision bump,
+ // the database isn't stamped for the new model until the indexer starts
+ // filling it. Returns the model's calibration: indexing calibrates the
+ // model before it writes a single vector, so a database stamped for this
+ // model always has one to go with it -- but the numbers still have to be
+ // read into memory, since getScoreFraction() reads them synchronously
+ // while rendering.
+ async function _requireReadyIndex() {
+ await Zotero.Embeddings.Indexing.waitForPendingModelSwitch();
+ await Zotero.Embeddings.initDB();
+ let modelVersion = Zotero.Embeddings.getModelVersion();
+ let indexedVersion = await Zotero.DB.valueQueryAsync(
+ "SELECT value FROM embeddings.itemEmbeddingsMeta WHERE key='modelVersion'"
+ );
+ if (indexedVersion !== modelVersion) {
+ throw new Zotero.Embeddings.IndexNotReadyError(
+ `Embeddings index is for '${indexedVersion || 'no model'}', `
+ + `but the active model is '${modelVersion}'`
+ );
+ }
+ let calibration = await Zotero.Embeddings.loadCalibration();
+ if (!calibration) {
+ throw new Zotero.Embeddings.IndexNotReadyError(
+ `Embeddings index is stamped for '${modelVersion}' but the model `
+ + `has no calibration`
+ );
+ }
+ return calibration;
+ }
+
/**
* Score a given set of items by similarity to a query. Items scoring below
* the model's measured minimum aren't matches and aren't returned (see
@@ -1120,34 +1175,7 @@ Zotero.Embeddings = new function () {
if (!itemIDs.length || !this.isEnabled()) {
return scores;
}
- // Wait out any in-progress model switch, so the query isn't embedded
- // with one model and compared against another's vectors
- await Zotero.Embeddings.Indexing.waitForPendingModelSwitch();
- await this.initDB();
- // The stored vectors must have been produced by the active model.
- // During a switch, or a reindex after a revision bump, the database
- // isn't stamped for the new model until the indexer starts filling it.
- let modelVersion = this.getModelVersion();
- let indexedVersion = await Zotero.DB.valueQueryAsync(
- "SELECT value FROM embeddings.itemEmbeddingsMeta WHERE key='modelVersion'"
- );
- if (indexedVersion !== modelVersion) {
- throw new this.IndexNotReadyError(
- `Embeddings index is for '${indexedVersion || 'no model'}', `
- + `but the active model is '${modelVersion}'`
- );
- }
- // Indexing calibrates the model before it writes a single vector, so a
- // database stamped for this model always has a calibration to go with
- // it -- but the numbers still have to be read into memory, since
- // getScoreFraction() reads them synchronously while rendering.
- let calibration = await this.loadCalibration();
- if (!calibration) {
- throw new this.IndexNotReadyError(
- `Embeddings index is stamped for '${modelVersion}' but the model `
- + `has no calibration`
- );
- }
+ let calibration = await _requireReadyIndex();
let generation = _modelGeneration;
let query = _center(await this.embedQuery(queryText));
let minScore = calibration.minScore;
@@ -1169,9 +1197,11 @@ Zotero.Embeddings = new function () {
throw new this.IndexNotReadyError('Model changed during scoring');
}
let chunk = itemIDs.slice(i, i + chunkSize);
+ // Rows without an embedding are processed-but-empty markers (see
+ // _setUpDB()), with nothing to score
let rows = await Zotero.DB.queryAsync(
"SELECT itemID, embedding FROM embeddings.itemEmbeddings WHERE itemID IN ("
- + chunk.map(() => '?').join(',') + ")",
+ + chunk.map(() => '?').join(',') + ") AND embedding IS NOT NULL",
chunk
);
for (let row of rows) {
@@ -1207,6 +1237,55 @@ Zotero.Embeddings = new function () {
}
return scores;
};
+
+ /**
+ * The chunks of a single item most similar to a query, each with where in
+ * the item it came from -- for surfacing why an item matched (e.g. which
+ * section of an attachment's full text). Chunks scoring below the model's
+ * measured minimum aren't matches and aren't returned.
+ *
+ * The text and location fields describe attachment fulltext chunks (see
+ * the itemEmbeddings table); for other item types they're null, and the
+ * item itself is the preview and the location.
+ *
+ * @param {String} queryText
+ * @param {Number} itemID
+ * @param {Object} [options]
+ * @param {Number} [options.limit=3] - Most chunks to return
+ * @return {Promise} - [{ chunkIndex, score, text, outlinePath,
+ * startBlock, endBlock, pageLabel, position, sectionPart,
+ * sectionParts }], best first, ties broken by position in the text.
+ * chunkIndex is document order, for callers that want to re-sort.
+ */
+ this.getMatchingChunks = async function (queryText, itemID, { limit = 3 } = {}) {
+ if (!this.isEnabled()) {
+ return [];
+ }
+ let calibration = await _requireReadyIndex();
+ let query = _center(await this.embedQuery(queryText));
+ let rows = await Zotero.DB.queryAsync(
+ "SELECT chunkIndex, embedding, chunkText, outlinePath, startBlock, endBlock, "
+ + "pageLabel, navPosition, sectionPart, sectionParts "
+ + "FROM embeddings.itemEmbeddings WHERE itemID=? AND embedding IS NOT NULL",
+ itemID
+ );
+ return rows
+ .map(row => ({
+ chunkIndex: row.chunkIndex,
+ score: this.dot(query, _center(_blobToVector(row.embedding))),
+ text: row.chunkText,
+ outlinePath: row.outlinePath,
+ startBlock: row.startBlock,
+ endBlock: row.endBlock,
+ pageLabel: row.pageLabel,
+ position: row.navPosition ? JSON.parse(row.navPosition) : null,
+ sectionPart: row.sectionPart,
+ sectionParts: row.sectionParts
+ }))
+ .filter(chunk => chunk.score >= calibration.minScore)
+ .sort((a, b) => (b.score - a.score) || (a.chunkIndex - b.chunkIndex))
+ .slice(0, limit);
+ };
};
@@ -1214,9 +1293,9 @@ Zotero.Embeddings = new function () {
* Zotero.Embeddings.Chunking -- splitting a text into passages small enough to
* each be embedded on their own, so a long text's later paragraphs are
* searchable instead of being averaged into one vector or truncated away by the
- * pipeline. The indexer applies this only to notes. Chunk size is bounded by
- * CHUNK_MAX_TOKENS, capped by the model's window, and counted in the model's
- * own tokens.
+ * pipeline. The indexer applies this to notes (chunkText()) and to attachment
+ * full text (chunkSections()). Chunk size is bounded by CHUNK_MAX_TOKENS,
+ * capped by the model's window, and counted in the model's own tokens.
*/
Zotero.Embeddings.Chunking = new function () {
// Tokens carried over from the end of one chunk into the start of the
@@ -1234,6 +1313,14 @@ Zotero.Embeddings.Chunking = new function () {
// out paragraph-sized, so this decides only how long a text has to be before
// it's split at all, and how far a single oversized paragraph is split.
const CHUNK_MAX_TOKENS = 768;
+ // Most characters to feed the tokenizer in one encode() call. The
+ // multilingual models' SentencePiece Unigram tokenizer is quadratic in
+ // input length -- its Metaspace pre-tokenizer doesn't split at whitespace,
+ // so the whole input goes through the Viterbi lattice as one string, and a
+ // single long encode can take seconds. Segments this size keep every call
+ // in the tokenizer's linear regime, and token counts are additive across
+ // whitespace boundaries, so the segmented measurement is exact.
+ const TOKENIZER_SEGMENT_CHARS = 1000;
// Tokenizer instances by model name. Failures aren't cached, so a later
// indexing run retries the load.
@@ -1288,31 +1375,94 @@ Zotero.Embeddings.Chunking = new function () {
// chunk's own text may use.
//
// Counts leave out the special tokens the tokenizer wraps every input in, so
- // that the counts of several sentences add up to the count of those sentences
- // joined. The even split in _splitBlockEvenly() depends on that: counting
- // them per sentence would inflate a piece's measured size in proportion to
- // how many sentences it holds, while a block is measured in one call, and the
- // pieces would come out lopsided.
+ // that the counts of several pieces of text add up to the count of those
+ // pieces joined. That additivity is what the whole chunking flow leans on:
+ // a text is tokenized exactly once, at paragraph granularity
+ // (_measureParagraphs()), and every level above -- blocks, sections, groups
+ // of sections -- is sized by summing those counts (_sumTokens()) instead of
+ // re-tokenizing the same text. `joinTokens` is what one '\n\n' join adds
+ // once paragraphs are put back together, measured rather than assumed
+ // (a SentencePiece model can spend a token on collapsed whitespace), so
+ // sums charge it per join.
//
// Those special tokens come off the window instead, together with the
// passage prefix embedPassages() prepends -- neither is part of the text this
// code sees, but both take up room once the chunk is embedded.
function _getMetrics(tokenizer) {
let specialTokens = tokenizer.encode('').length;
- let count = text => tokenizer.encode(text).length - specialTokens;
+ // Measured in segments (see _splitForTokenizer()), so a long paragraph
+ // never hits the tokenizer's quadratic regime in a single call
+ let count = text => _splitForTokenizer(text).reduce(
+ (sum, segment) => sum + tokenizer.encode(segment).length - specialTokens,
+ 0
+ );
let prefix = Zotero.Embeddings.getPassagePrefix();
return {
count,
+ joinTokens: Math.max(0, count('a\n\na') - 2 * count('a')),
budget: Math.min(CHUNK_MAX_TOKENS, Zotero.Embeddings.getModelMaxTokens())
- specialTokens - (prefix ? count(prefix) : 0)
};
}
+ // The paragraphs of a text, each counted exactly once -- the single place
+ // chunking pays for tokenization
+ function _measureParagraphs(text, count) {
+ return text.split(/\n+/)
+ .map(paragraph => paragraph.trim())
+ .filter(Boolean)
+ .map(paragraph => ({ text: paragraph, tokens: count(paragraph) }));
+ }
+
+ // Tokens of the given paragraphs once joined back together, charging
+ // joinTokens per join (see _getMetrics())
+ function _sumTokens(paragraphs, joinTokens) {
+ if (!paragraphs.length) {
+ return 0;
+ }
+ return paragraphs.reduce((sum, paragraph) => sum + paragraph.tokens, 0)
+ + joinTokens * (paragraphs.length - 1);
+ }
+
+ // A text in segments of at most TOKENIZER_SEGMENT_CHARS, each ending
+ // right before a whitespace character, so the next segment carries it and
+ // the tokenizer sees every word with its leading space. Text with no
+ // whitespace in a whole window (e.g., unsegmented CJK) is cut mid-run,
+ // which can miscount by a token per boundary -- noise against the budget.
+ function _splitForTokenizer(text) {
+ if (text.length <= TOKENIZER_SEGMENT_CHARS) {
+ return [text];
+ }
+ let segments = [];
+ let start = 0;
+ while (start < text.length) {
+ let end = Math.min(start + TOKENIZER_SEGMENT_CHARS, text.length);
+ if (end < text.length) {
+ for (let i = end; i > start; i--) {
+ if (/\s/.test(text[i])) {
+ end = i;
+ break;
+ }
+ }
+ }
+ segments.push(text.slice(start, end));
+ start = end;
+ }
+ return segments;
+ }
+
// Last resort for text with no sentence boundary inside the budget: window
// it by tokens.
function _hardSplit(text, budget, tokenizer) {
let pieces = [];
- let ids = tokenizer.encode(text);
+ // Encoded in segments for the same reason count() measures in them
+ // (see _splitForTokenizer()). Each segment keeps its special-token
+ // wrapper -- stripping it would mean knowing how the tokenizer splits
+ // it between start and end -- so a window can hold a few interior
+ // special tokens; decode skips them, and a piece just lands a few
+ // tokens under the budget.
+ let ids = _splitForTokenizer(text)
+ .flatMap(segment => [...tokenizer.encode(segment)]);
let step = Math.max(1, budget - CHUNK_OVERLAP_TOKENS);
for (let start = 0; start < ids.length; start += step) {
let piece = tokenizer
@@ -1435,17 +1585,24 @@ Zotero.Embeddings.Chunking = new function () {
*/
this.chunkText = async function (text) {
let tokenizer = await this.getTokenizer();
- let { count, budget } = _getMetrics(tokenizer);
+ let metrics = _getMetrics(tokenizer);
+ let paragraphs = _measureParagraphs(text, metrics.count);
+ return _chunkParagraphs(text, paragraphs, metrics.budget, metrics, tokenizer);
+ };
+
+ // The paragraph-level chunker behind chunkText(), over paragraphs already
+ // measured by _measureParagraphs() -- everything here is sized by summing
+ // their counts, so no text is tokenized a second time. The budget is the
+ // caller's, so chunkSections() can reserve part of the window for a
+ // section's outline-path prefix. `text` is what a fitting result returns
+ // as its single chunk, joins and all.
+ function _chunkParagraphs(text, paragraphs, budget, metrics, tokenizer) {
+ let { count, joinTokens } = metrics;
// Text that fits the window as it stands, which is most of it
- if (count(text) <= budget) {
+ if (_sumTokens(paragraphs, joinTokens) <= budget) {
return [text];
}
- let paragraphs = text.split(/\n+/)
- .map(paragraph => paragraph.trim())
- .filter(Boolean)
- .map(paragraph => ({ text: paragraph, tokens: count(paragraph) }));
-
// Group paragraphs into blocks, combining any that are too small to
// stand alone with the paragraphs that follow them. A paragraph that
// reaches the minimum on its own becomes its own block, which is what
@@ -1454,7 +1611,7 @@ Zotero.Embeddings.Chunking = new function () {
let pending = [];
let pendingTokens = 0;
for (let paragraph of paragraphs) {
- pending.push(paragraph.text);
+ pending.push(paragraph);
pendingTokens += paragraph.tokens;
if (pendingTokens >= CHUNK_MIN_TOKENS) {
groups.push(pending);
@@ -1476,10 +1633,10 @@ Zotero.Embeddings.Chunking = new function () {
let chunks = [];
for (let group of groups) {
- // Recount rather than summing the paragraphs, so the joins between
- // them are part of the block's size
- let blockText = group.join('\n\n');
- let block = { text: blockText, tokens: count(blockText) };
+ let block = {
+ text: group.map(paragraph => paragraph.text).join('\n\n'),
+ tokens: _sumTokens(group, joinTokens)
+ };
if (block.tokens <= budget) {
chunks.push(block.text);
}
@@ -1488,6 +1645,137 @@ Zotero.Embeddings.Chunking = new function () {
}
}
return chunks.length ? chunks : [text];
+ }
+
+ /**
+ * Split a document's outline sections (see Zotero.SDT.getSections()) into
+ * chunks that each fit the active model's context window. Sections play
+ * the role paragraphs play in chunkText(), one level up: a section is the
+ * topic unit, so two of them never share a chunk unless one was too small
+ * to embed on its own, and a section over the window is split by the
+ * paragraph machinery. Each chunk points back at the section blocks it
+ * covers, so a match can be located in the document later.
+ *
+ * Sections marked `auxiliary` (captions, image descriptions) are exempt
+ * from the too-small merging in both directions: each becomes exactly
+ * one chunk (flagged `auxiliary`), so it can surface on its own without
+ * ever mixing into the running text.
+ *
+ * A chunk's `embedText` -- what actually gets embedded -- is its text
+ * prefixed with its section's outline path (e.g. "Methods >
+ * Participants"), giving a section fragment the context of its headings;
+ * the prefix comes out of the chunk's token budget. `text` stays the
+ * plain piece, for display.
+ *
+ * Each chunk carries its section's location (pageIndex, pageLabel,
+ * position -- see Zotero.SDT.getSections()), and its place within the
+ * section: sectionPart / sectionParts say which piece of a split section
+ * this is, so a match can be read as coming from the middle or the end
+ * of its section. Splitting happens at sentence granularity, blind to
+ * blocks, so the pieces of one section share its location.
+ *
+ * @param {Object[]} sections - [{ text, outlinePath, startBlock, endBlock,
+ * pageIndex, pageLabel, position }]
+ * @return {Promise} - [{ text, embedText, outlinePath,
+ * startBlock, endBlock, pageIndex, pageLabel, position,
+ * sectionPart, sectionParts }]
+ */
+ this.chunkSections = async function (sections) {
+ let tokenizer = await this.getTokenizer();
+ let metrics = _getMetrics(tokenizer);
+ let { count, joinTokens, budget } = metrics;
+
+ // Group sections into chunk-worthy units, combining any too small to
+ // stand alone with the sections that follow them -- same policy as
+ // chunkText()'s paragraph grouping. Each section is measured once, at
+ // paragraph granularity, and carries its paragraphs forward, so no
+ // later stage tokenizes the same text again.
+ let groups = [];
+ let pending = null;
+ // Index of the last body group, for the trailing merge below
+ let lastBodyGroup = -1;
+ for (let section of sections) {
+ if (!section.text) {
+ continue;
+ }
+ let paragraphs = _measureParagraphs(section.text, count);
+ let tokens = _sumTokens(paragraphs, joinTokens);
+ // An auxiliary section (caption, image description) is its own
+ // chunk no matter how small: indexed and searchable on its own,
+ // never mixed into the running text. The body sections around it
+ // still merge with each other -- `pending` just carries across.
+ if (section.auxiliary) {
+ groups.push({ sections: [section], paragraphs, auxiliary: true });
+ continue;
+ }
+ if (pending) {
+ pending.sections.push(section);
+ pending.paragraphs.push(...paragraphs);
+ pending.tokens += tokens;
+ }
+ else {
+ pending = { sections: [section], paragraphs, tokens };
+ }
+ if (pending.tokens >= CHUNK_MIN_TOKENS) {
+ lastBodyGroup = groups.length;
+ groups.push(pending);
+ pending = null;
+ }
+ }
+ // A trailing group still under the minimum joins the previous body
+ // group rather than standing alone as a runt chunk -- never an
+ // auxiliary group, which stays exactly its own text
+ if (pending) {
+ if (lastBodyGroup >= 0) {
+ let last = groups[lastBodyGroup];
+ last.sections.push(...pending.sections);
+ last.paragraphs.push(...pending.paragraphs);
+ }
+ else {
+ groups.push(pending);
+ }
+ }
+ // Body sections merge across the auxiliary sections between them, so
+ // a group can close after an auxiliary group whose blocks it
+ // precedes. Chunk indexes are document order (the UI sorts by them),
+ // so restore it here.
+ groups.sort((a, b) => (a.sections[0].startBlock ?? 0) - (b.sections[0].startBlock ?? 0));
+
+ let chunks = [];
+ for (let group of groups) {
+ // A merged group takes its first section's outline path -- the
+ // heading its text starts under
+ let outlinePath = group.sections[0].outlinePath || '';
+ let startBlock = group.sections[0].startBlock;
+ let endBlock = group.sections[group.sections.length - 1].endBlock;
+ let prefix = outlinePath ? outlinePath + '\n\n' : '';
+ let prefixTokens = prefix ? count(prefix) : 0;
+ // A pathological outline path that would eat a real share of the
+ // window hurts more than it helps
+ if (prefixTokens > budget / 4) {
+ prefix = '';
+ prefixTokens = 0;
+ }
+ let text = group.sections.map(section => section.text).join('\n\n');
+ let first = group.sections[0];
+ let pieces = _chunkParagraphs(text, group.paragraphs, budget - prefixTokens, metrics, tokenizer);
+ for (let i = 0; i < pieces.length; i++) {
+ chunks.push({
+ text: pieces[i],
+ embedText: prefix + pieces[i],
+ outlinePath,
+ startBlock,
+ endBlock,
+ pageIndex: first.pageIndex ?? null,
+ pageLabel: first.pageLabel ?? null,
+ position: first.position ?? null,
+ sectionPart: i + 1,
+ sectionParts: pieces.length,
+ auxiliary: !!group.auxiliary
+ });
+ }
+ }
+ return chunks;
};
};
@@ -1532,6 +1820,9 @@ Zotero.Embeddings.Indexing = new function () {
// Wait longer than the usual debounce before retrying a run that was held
// off for memory
const LOW_MEMORY_RETRY_DELAY = 5 * 60 * 1000;
+ // Most often the per-library status counts are recomputed during a run,
+ // since each refresh is a pass over the database
+ const STATUS_REFRESH_INTERVAL = 5000;
// Debounce before starting the consumer, so a burst of changes (e.g. an
// import) is picked up in one pass
const KICK_DELAY = 3000;
@@ -1566,6 +1857,14 @@ Zotero.Embeddings.Indexing = new function () {
_switchModel();
});
+ // Toggling fulltext indexing changes what's eligible: on, the newly
+ // eligible attachments get indexed; off, their stored chunks are
+ // pruned. Unlike a model switch, nothing already stored goes stale,
+ // so the rest of the index is left alone.
+ Zotero.Prefs.registerObserver('embeddings.indexFulltext', () => {
+ _onIndexFulltextChange().catch(e => Zotero.logError(e));
+ });
+
Zotero.Notifier.registerObserver({
notify: async (event, type, ids) => {
if (type !== 'item' || !Zotero.Embeddings.isEnabled()) {
@@ -1658,6 +1957,24 @@ Zotero.Embeddings.Indexing = new function () {
}
}
+ async function _onIndexFulltextChange() {
+ if (!Zotero.Embeddings.isEnabled()) {
+ return;
+ }
+ if (_indexFulltextEnabled()) {
+ // Turning fulltext indexing on is a request to index the
+ // attachments, so it also resumes a stopped indexer
+ await Zotero.Embeddings.Indexing.startIndexing();
+ }
+ else {
+ // The attachments are no longer eligible, so the ordinary orphan
+ // pruning drops their chunks -- without restarting a paused indexer
+ await Zotero.Embeddings.initDB();
+ await _pruneOrphanedEmbeddings(await _getEligibleItemIDs());
+ await Zotero.Embeddings.Indexing.refreshStatus();
+ }
+ }
+
// Debounce before starting the consumer, so a burst of changes (e.g. an
// import) is picked up in one pass. Enqueued ids just sit in the queue
// until the consumer runs.
@@ -1745,27 +2062,37 @@ Zotero.Embeddings.Indexing = new function () {
// - a note with at least three words in its plain text
// - an annotation with at least three words across the passage it marks
// and its comment
+ // - with fulltext indexing enabled, a PDF/EPUB/snapshot attachment. Its
+ // text lives in a file, so there's no cheap word test here: one that
+ // yields no text at indexing time is recorded as processed instead
+ // (see _indexItems()), so the progress counts still converge.
// Everything that needs to know what's eligible -- enqueueing, pruning,
// progress counts -- works from this result, and _indexItems() skips items
// by the same per-type tests (see _getIndexableText()), so the counts and
// the index stay in agreement.
//
- // @return {Promise