mirror of
https://github.com/zotero/zotero.git
synced 2026-09-11 22:51:15 +00:00
Semantic search on fulltext of attachments
Add optional pref to index and search fulltext of attachments. When enabled, attachment IDs are enqueue after regular items, notes, and annotations. Added helpers to extract outline and sections from structured text module. During indexing, the sections of the attachment are extracted, large sections are broken into chunks, and small sections are combined to fit into the context window of the model. Then, each chunk is embedded with its outline path as the prefix and added to embeddings table. Each row now contains full text of the chunk and it's path - a good amount of duplication needed to ensure that we can reliably connect the embedding of the chunk to its text for a preview. On search in Best Match mode, attachment rows get the score of the highest ranking chunk, so if there is a very relevant chunk in an attachment, the regular item with a non-relevant abstract will still rank highly. When the attachment row is selected, top 5 matching chunks appear in the new search results collapsible-section of the item pane, so one can examine matching chunks without opening the actual reader.
This commit is contained in:
Bogdan Abaev
parent
0d52d7098c
commit
2568b5ccc4
19 changed files with 2397 additions and 115 deletions
|
|
@ -206,8 +206,13 @@ class CollectionViewItemTreeRowProvider extends ItemTreeRowProvider {
|
|||
);
|
||||
let libraries = status.libraries
|
||||
.filter(lib => !libraryIDs.size || libraryIDs.has(lib.libraryID));
|
||||
let indexed = libraries.reduce((sum, lib) => sum + lib.indexed, 0);
|
||||
let total = libraries.reduce((sum, lib) => sum + lib.eligible, 0);
|
||||
// Coverage is coverage: attachment fulltext is reported separately
|
||||
// in the preferences, but an incomplete index is incomplete
|
||||
// whichever part of it is still filling in
|
||||
let indexed = libraries.reduce(
|
||||
(sum, lib) => sum + lib.indexed + lib.indexedAttachments, 0);
|
||||
let total = libraries.reduce(
|
||||
(sum, lib) => sum + lib.eligible + lib.eligibleAttachments, 0);
|
||||
if (indexed >= total) {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,6 +74,8 @@ Services.scriptloader.loadSubScript('chrome://zotero/content/elements/itemTreeMe
|
|||
['attachment-row', 'chrome://zotero/content/elements/attachmentRow.js'],
|
||||
['attachment-annotations-box', 'chrome://zotero/content/elements/attachmentAnnotationsBox.js'],
|
||||
['annotation-row', 'chrome://zotero/content/elements/annotationRow.js'],
|
||||
['search-results-box', 'chrome://zotero/content/elements/searchResultsBox.js'],
|
||||
['search-result-row', 'chrome://zotero/content/elements/searchResultRow.js'],
|
||||
['annotation-items-pane', 'chrome://zotero/content/elements/annotationItemsPane.js'],
|
||||
['context-notes-list', 'chrome://zotero/content/elements/contextNotesList.js'],
|
||||
['note-row', 'chrome://zotero/content/elements/noteRow.js'],
|
||||
|
|
|
|||
|
|
@ -55,6 +55,8 @@
|
|||
<tags-box id="zotero-editpane-tags" class="zotero-editpane-tags" data-pane="tags"/>
|
||||
|
||||
<related-box id="zotero-editpane-related" class="zotero-editpane-related" data-pane="related"/>
|
||||
|
||||
<search-results-box id="zotero-editpane-search-results" data-pane="search-results" hidden="true"/>
|
||||
</html:div>
|
||||
</html:div>
|
||||
</hbox>
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@
|
|||
}
|
||||
|
||||
get _builtInPanes() {
|
||||
return ["info", "abstract", "attachments", "notes", "note-info", "attachment-info", "attachment-annotations", "libraries-collections", "tags", "related"];
|
||||
return ["info", "abstract", "attachments", "notes", "note-info", "attachment-info", "attachment-annotations", "libraries-collections", "tags", "related", "search-results"];
|
||||
}
|
||||
|
||||
get container() {
|
||||
|
|
|
|||
131
chrome/content/zotero/elements/searchResultRow.js
Normal file
131
chrome/content/zotero/elements/searchResultRow.js
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2026 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
https://www.zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
|
||||
{
|
||||
// A best-match search result card: one fulltext chunk of an attachment
|
||||
// (see Zotero.Embeddings.getMatchingChunks()), presented like an
|
||||
// annotation-row -- where the chunk sits in the document as the head,
|
||||
// its text as the quote. The two share their styling (see
|
||||
// scss/elements/_annotationRow.scss).
|
||||
class SearchResultRow extends XULElementBase {
|
||||
content = MozXULElement.parseXULToFragment(`
|
||||
<html:div class="head">
|
||||
<html:div class="title">
|
||||
<html:span class="path"/>
|
||||
<html:span class="part"/>
|
||||
</html:div>
|
||||
<html:div class="location"/>
|
||||
</html:div>
|
||||
<html:div class="body">
|
||||
<html:div class="quote"/>
|
||||
<html:button class="show-more" data-l10n-id="search-result-row-show-more" hidden="true"/>
|
||||
</html:div>
|
||||
`);
|
||||
|
||||
_chunk = null;
|
||||
|
||||
get chunk() {
|
||||
return this._chunk;
|
||||
}
|
||||
|
||||
set chunk(chunk) {
|
||||
this._chunk = chunk;
|
||||
this.render();
|
||||
}
|
||||
|
||||
init() {
|
||||
this._path = this.querySelector('.path');
|
||||
this._part = this.querySelector('.part');
|
||||
this._location = this.querySelector('.location');
|
||||
this._quote = this.querySelector('.quote');
|
||||
this._showMore = this.querySelector('.show-more');
|
||||
this._showMore.addEventListener('click', (event) => {
|
||||
// The card's activation (open the attachment) shouldn't fire
|
||||
// for the toggle
|
||||
event.stopPropagation();
|
||||
this._toggleExpanded();
|
||||
});
|
||||
this.render();
|
||||
}
|
||||
|
||||
render() {
|
||||
if (!this.initialized || !this._chunk) return;
|
||||
|
||||
// The chunk's outline path says where in the document it came
|
||||
// from; a chunk from a document without an outline falls back to
|
||||
// a generic label
|
||||
if (this._chunk.outlinePath) {
|
||||
this._path.removeAttribute('data-l10n-id');
|
||||
this._path.textContent = this._chunk.outlinePath;
|
||||
}
|
||||
else {
|
||||
document.l10n.setAttributes(this._path, 'search-result-row-fulltext');
|
||||
}
|
||||
// Which piece of a split section this is, so a match reads as
|
||||
// coming from the middle or the end of its section
|
||||
let parts = this._chunk.sectionParts;
|
||||
this._part.hidden = !(parts > 1);
|
||||
if (parts > 1) {
|
||||
this._part.textContent = `${this._chunk.sectionPart}/${parts}`;
|
||||
}
|
||||
// The page the chunk's section starts on, labeled the way
|
||||
// annotation rows label theirs
|
||||
this._location.hidden = !this._chunk.pageLabel;
|
||||
if (this._chunk.pageLabel) {
|
||||
this._location.textContent
|
||||
= Zotero.getString('pdfReader.page') + ' ' + this._chunk.pageLabel;
|
||||
}
|
||||
|
||||
this._quote.textContent = this._chunk.text || '';
|
||||
|
||||
// Offer "Show More" only when the quote is actually clamped,
|
||||
// which is only measurable once the card has a layout
|
||||
this.classList.remove('expanded');
|
||||
this._showMore.hidden = true;
|
||||
requestAnimationFrame(() => {
|
||||
this._showMore.hidden
|
||||
= this._quote.scrollHeight <= this._quote.clientHeight;
|
||||
});
|
||||
|
||||
// A11y - make focusable and describe the card
|
||||
this.setAttribute('tabindex', 0);
|
||||
this.setAttribute('aria-label', [
|
||||
this._chunk.outlinePath,
|
||||
this._location.hidden ? '' : this._location.textContent,
|
||||
this._chunk.text
|
||||
].filter(Boolean).join('. '));
|
||||
}
|
||||
|
||||
_toggleExpanded() {
|
||||
let expanded = this.classList.toggle('expanded');
|
||||
document.l10n.setAttributes(this._showMore,
|
||||
expanded ? 'search-result-row-show-less' : 'search-result-row-show-more');
|
||||
}
|
||||
}
|
||||
|
||||
customElements.define('search-result-row', SearchResultRow);
|
||||
}
|
||||
192
chrome/content/zotero/elements/searchResultsBox.js
Normal file
192
chrome/content/zotero/elements/searchResultsBox.js
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2026 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
https://www.zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
{
|
||||
const { ItemPaneSectionElementBase } = ChromeUtils.importESModule(
|
||||
"chrome://zotero/content/elements/itemPaneSectionElementBase.mjs",
|
||||
{ global: "current" }
|
||||
);
|
||||
|
||||
// Most matching chunks shown for an attachment
|
||||
const MAX_RESULTS = 5;
|
||||
|
||||
// Why an attachment matched the active best-match search: cards with the
|
||||
// fulltext chunks most similar to the query, so they can be previewed
|
||||
// without opening the file. Shown only while a best-match search is
|
||||
// active, for attachments with matching indexed chunks.
|
||||
class SearchResultsBox extends ItemPaneSectionElementBase {
|
||||
content = MozXULElement.parseXULToFragment(`
|
||||
<collapsible-section data-l10n-id="section-search-results" data-pane="search-results">
|
||||
<html:div class="body">
|
||||
</html:div>
|
||||
</collapsible-section>
|
||||
`);
|
||||
|
||||
get item() {
|
||||
return this._item;
|
||||
}
|
||||
|
||||
set item(item) {
|
||||
super.item = (item instanceof Zotero.Item && item.isFileAttachment()) ? item : null;
|
||||
// A new item's emptiness isn't known until asyncRender scores it
|
||||
this._count = undefined;
|
||||
}
|
||||
|
||||
get collectionTreeRows() {
|
||||
return super.collectionTreeRows;
|
||||
}
|
||||
|
||||
// The item pane sets collectionTreeRows after item, so this is where
|
||||
// everything visibility depends on is finally known
|
||||
set collectionTreeRows(collectionTreeRows) {
|
||||
super.collectionTreeRows = collectionTreeRows;
|
||||
this._updateHidden();
|
||||
}
|
||||
|
||||
init() {
|
||||
this.initCollapsibleSection();
|
||||
this._body = this.querySelector('.body');
|
||||
// The header's count placeholder needs a value before the first
|
||||
// async render fills in the real one
|
||||
this._section.setCount(0);
|
||||
// Double-click (or Enter on a focused card) opens the attachment
|
||||
// at the chunk
|
||||
this._body.addEventListener('dblclick', this._handleActivate);
|
||||
this._body.addEventListener('keydown', (event) => {
|
||||
if (event.key == 'Enter') {
|
||||
this._handleActivate(event);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// The query the selected collection rows are ranked by, or false when
|
||||
// no best-match search is active. Rows can be duck-typed stand-ins
|
||||
// (e.g. the citation dialog's), which implement only part of the row
|
||||
// API.
|
||||
get _query() {
|
||||
// Quick-search state (setSearch()) lives on collection tree row
|
||||
// *instances*, and the ones passed down the item pane are
|
||||
// re-fetched from the collections view at item-selection time --
|
||||
// which can have rebuilt its rows since the items view got its
|
||||
// own set. Only the items view's instances are guaranteed to
|
||||
// carry the active search, so prefer those; the passed rows are
|
||||
// the fallback for hosts without an items view.
|
||||
let itemsView = this.closest('item-pane')?.itemsView;
|
||||
let rows = itemsView?.collectionTreeRows?.length
|
||||
? itemsView.collectionTreeRows
|
||||
: this.collectionTreeRows;
|
||||
for (let row of rows || []) {
|
||||
if (typeof row.getBestMatchQuery == 'function') {
|
||||
let query = row.getBestMatchQuery();
|
||||
if (query) {
|
||||
return query;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// A query change re-renders even when the item didn't change
|
||||
get _renderDependencies() {
|
||||
return [...super._renderDependencies, this._query];
|
||||
}
|
||||
|
||||
render() {}
|
||||
|
||||
async asyncRender() {
|
||||
if (!this.initialized) return;
|
||||
if (this._isAlreadyRendered("async")) return;
|
||||
|
||||
let item = this.item;
|
||||
let query = this._query;
|
||||
this._body.replaceChildren();
|
||||
if (!item || !query) {
|
||||
return;
|
||||
}
|
||||
|
||||
let chunks = [];
|
||||
try {
|
||||
chunks = await Zotero.Embeddings.getMatchingChunks(query, item.id,
|
||||
{ limit: MAX_RESULTS });
|
||||
}
|
||||
catch (e) {
|
||||
// Nothing to show while the model is still downloading or the
|
||||
// index is being rebuilt
|
||||
if (!(e instanceof Zotero.Embeddings.IndexNotReadyError)) {
|
||||
Zotero.logError(e);
|
||||
}
|
||||
}
|
||||
// Only fulltext chunks carry their own text; anything else about
|
||||
// the item is already visible in the pane
|
||||
chunks = chunks.filter(chunk => chunk.text);
|
||||
// The selection may have moved on while scoring
|
||||
if (this.item !== item) {
|
||||
return;
|
||||
}
|
||||
this._count = chunks.length;
|
||||
this._section.setCount(chunks.length);
|
||||
this._updateHidden();
|
||||
// Left in the order getMatchingChunks() returns them, strongest
|
||||
// match first: with only a handful of cards shown, the best one
|
||||
// earning the top slot matters more than reading them in
|
||||
// document order
|
||||
for (let chunk of chunks) {
|
||||
let row = document.createXULElement('search-result-row');
|
||||
row.chunk = chunk;
|
||||
this._body.append(row);
|
||||
}
|
||||
}
|
||||
|
||||
// Open the attachment where the activated card's chunk is: for a PDF,
|
||||
// scrolled to and highlighting the chunk's section; without a stored
|
||||
// position (EPUB, snapshot, flat-text fallback), just open it
|
||||
_handleActivate = (event) => {
|
||||
let row = event.target.closest('search-result-row');
|
||||
// The Show More toggle isn't an activation
|
||||
if (!row || !this.item || event.target.closest('.show-more')) {
|
||||
return;
|
||||
}
|
||||
if (typeof ZoteroPane == 'undefined') {
|
||||
return;
|
||||
}
|
||||
let position = row.chunk?.position;
|
||||
ZoteroPane.viewAttachment(this.item.id, null, false,
|
||||
position ? { location: { position } } : undefined)
|
||||
.catch(e => Zotero.logError(e));
|
||||
};
|
||||
|
||||
_updateHidden() {
|
||||
// Visible only for a file attachment during a best-match search;
|
||||
// asyncRender hides it again when nothing matched. Deciding
|
||||
// emptiness needs the async scoring, so unlike the annotations
|
||||
// section this one can't know its final state synchronously --
|
||||
// it appears, then empties out, rather than flickering in late.
|
||||
this.hidden = !this.item || !this._query || this.tabType == 'reader'
|
||||
|| this._count === 0;
|
||||
}
|
||||
}
|
||||
|
||||
customElements.define("search-results-box", SearchResultsBox);
|
||||
}
|
||||
|
|
@ -152,13 +152,17 @@ Zotero_Preferences.Advanced = {
|
|||
updateSemanticSearchUI: function (status) {
|
||||
let statusBox = document.getElementById('semantic-search-status');
|
||||
statusBox.hidden = !status.enabled;
|
||||
// Fulltext indexing only means something with a model selected
|
||||
document.getElementById('semantic-search-index-fulltext').disabled = !status.enabled;
|
||||
if (!status.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Phase / status message
|
||||
let phaseLabel = document.getElementById('semantic-search-phase');
|
||||
let hasRemaining = status.libraries.some(lib => lib.indexed < lib.eligible);
|
||||
let hasRemaining = status.libraries.some(
|
||||
lib => lib.indexed < lib.eligible
|
||||
|| lib.indexedAttachments < lib.eligibleAttachments);
|
||||
if (status.error) {
|
||||
document.l10n.setAttributes(phaseLabel, 'preferences-advanced-semantic-search-error', { error: status.error });
|
||||
}
|
||||
|
|
@ -214,10 +218,18 @@ Zotero_Preferences.Advanced = {
|
|||
'value',
|
||||
Zotero.Utilities.Internal.stringWithColon(lib.name)
|
||||
);
|
||||
grid.children[i * 2 + 1].setAttribute(
|
||||
'value',
|
||||
`${lib.indexed.toLocaleString()} / ${lib.eligible.toLocaleString()}`
|
||||
);
|
||||
// Attachment fulltext is reported on its own, since it's a much
|
||||
// larger and much slower job than the rest -- one combined count
|
||||
// would look stalled. With fulltext indexing off, none are
|
||||
// eligible and only the item count is shown.
|
||||
let counts = `${lib.indexed.toLocaleString()} / ${lib.eligible.toLocaleString()}`;
|
||||
if (lib.eligibleAttachments) {
|
||||
counts += ` ${Zotero.getString('general.and')} `
|
||||
+ `${lib.indexedAttachments.toLocaleString()} / `
|
||||
+ `${lib.eligibleAttachments.toLocaleString()} `
|
||||
+ Zotero.getString('itemTypes.attachment');
|
||||
}
|
||||
grid.children[i * 2 + 1].setAttribute('value', counts);
|
||||
});
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -307,6 +307,10 @@
|
|||
</hbox>
|
||||
<description id="semantic-search-model-description"
|
||||
data-l10n-id="preferences-advanced-semantic-search-model-description"/>
|
||||
<checkbox id="semantic-search-index-fulltext"
|
||||
data-l10n-id="preferences-advanced-semantic-search-index-fulltext"
|
||||
preference="extensions.zotero.embeddings.indexFulltext"
|
||||
native="true"/>
|
||||
<vbox id="semantic-search-status" hidden="true">
|
||||
<hbox align="center">
|
||||
<label data-l10n-id="preferences-advanced-semantic-search-status"/>
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -152,6 +152,273 @@ Zotero.SDT = new function () {
|
|||
return _openPack(new Uint8Array(result.bytes));
|
||||
};
|
||||
|
||||
/**
|
||||
* Get an attachment's text as outline-based sections. Each section runs
|
||||
* from one outline heading to the next; a document without an outline
|
||||
* falls back to per-page sections, and to a single section when there are
|
||||
* no pages either.
|
||||
*
|
||||
* Blocks the document marks as excluded flow (running heads, page
|
||||
* numbers) are left out. The rest is reported as the document describes
|
||||
* it, for callers to filter as their use calls for: a section lists the
|
||||
* blocks it's made of, each with its flow class and whether it's a
|
||||
* reference entry, and `text` is simply those blocks joined.
|
||||
*
|
||||
* Sections and blocks carry what the document knows about where they are:
|
||||
* - pageIndex: 0-based index into the document's pages
|
||||
* - pageLabel: page label ('ix', '15'), or the ordinal page number for a
|
||||
* PDF without labels. An EPUB's synthetic locations produce none.
|
||||
* - position: for PDFs, a reader-navigable { pageIndex, rects }
|
||||
* A section's location is that of its first block.
|
||||
*
|
||||
* @param {Integer} itemID
|
||||
* @param {Object} [options] - See getPack()
|
||||
* @returns {Promise<Object>} { ok: true, sections: [{ text, outlinePath,
|
||||
* startBlock, endBlock, pageIndex, pageLabel, position, blocks:
|
||||
* [{ index, text, flowClass, reference, pageIndex, pageLabel,
|
||||
* position }] }] }, or { ok: false, reason } (see getPack())
|
||||
*/
|
||||
this.getSections = async function (itemID, options = {}) {
|
||||
let result = await this.getPack(itemID, options);
|
||||
if (!result.ok) {
|
||||
return { ok: false, reason: result.reason };
|
||||
}
|
||||
try {
|
||||
let reader = await _openPack(new Uint8Array(result.bytes));
|
||||
let structure = await reader.materialize();
|
||||
return { ok: true, sections: _getStructureSections(structure) };
|
||||
}
|
||||
catch (e) {
|
||||
Zotero.logError(e);
|
||||
return { ok: false, reason: 'failed' };
|
||||
}
|
||||
};
|
||||
|
||||
// The section walk over a materialized structure. Mirrors the outline
|
||||
// handling of the document-worker's own section chunker
|
||||
// (structured-document-text/src/chunker.js), which the bundled reader
|
||||
// module doesn't export; switch to that export if it grows one.
|
||||
function _getStructureSections(structure) {
|
||||
let content = Array.isArray(structure?.content) ? structure.content : [];
|
||||
if (!content.length) {
|
||||
return [];
|
||||
}
|
||||
// Section boundaries: the outline's heading blocks, or page starts for
|
||||
// a document without an outline. Block 0 is always a boundary, so
|
||||
// front matter before the first heading forms a section of its own.
|
||||
let headings = _flattenOutline(structure.catalog?.outline, []);
|
||||
let boundaries = headings.length
|
||||
? headings
|
||||
: _getPageBoundaries(structure.catalog, content.length);
|
||||
// A boundary at block 0 emits no section of its own -- it just seeds
|
||||
// the first section's path
|
||||
boundaries = boundaries
|
||||
.filter(b => b.blockIndex < content.length)
|
||||
.sort((a, b) => a.blockIndex - b.blockIndex);
|
||||
let blockPages = _getBlockPages(structure.catalog, content.length);
|
||||
|
||||
let sections = [];
|
||||
let startBlock = 0;
|
||||
let path = [];
|
||||
let isHeading = false;
|
||||
for (let boundary of [...boundaries, { blockIndex: content.length, path }]) {
|
||||
if (boundary.blockIndex > startBlock) {
|
||||
let endBlock = boundary.blockIndex - 1;
|
||||
let blocks = [];
|
||||
let location = null;
|
||||
// A section that starts at an outline heading skips the
|
||||
// heading block's own text: the heading is already the last
|
||||
// component of the section's outline path
|
||||
for (let i = startBlock + (isHeading ? 1 : 0); i <= endBlock; i++) {
|
||||
let block = content[i];
|
||||
if (!block || block.flowClass === 'excluded') {
|
||||
continue;
|
||||
}
|
||||
let text = _getNestedBlockPlainText(block).trim();
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
let entry = { index: i, text, reference: _isReferenceBlock(block) };
|
||||
if (block.flowClass) {
|
||||
entry.flowClass = block.flowClass;
|
||||
}
|
||||
let blockLocation = _getBlockLocation(structure, blockPages, i);
|
||||
location = location || blockLocation;
|
||||
blocks.push(Object.assign(entry, blockLocation));
|
||||
}
|
||||
if (blocks.length) {
|
||||
// A section is located where it begins -- at its heading
|
||||
// when it has one, since that's where a reader would land
|
||||
// -- falling back to its first reported block
|
||||
let start = _getBlockLocation(structure, blockPages, startBlock);
|
||||
sections.push(Object.assign({
|
||||
text: blocks.map(block => block.text).join('\n'),
|
||||
outlinePath: path.join(' > '),
|
||||
startBlock,
|
||||
endBlock
|
||||
}, start.pageIndex === undefined ? location : start, { blocks }));
|
||||
}
|
||||
}
|
||||
startBlock = boundary.blockIndex;
|
||||
path = boundary.path;
|
||||
isHeading = !!boundary.isHeading;
|
||||
}
|
||||
return sections;
|
||||
}
|
||||
|
||||
// Where a block sits in the source document, as far as the document says.
|
||||
// Only the fields that are actually known are returned.
|
||||
function _getBlockLocation(structure, blockPages, index) {
|
||||
let location = {};
|
||||
// PDF blocks carry page geometry: [pageIndex, x1, y1, x2, y2] rects,
|
||||
// which the reader can scroll to and highlight
|
||||
let pageRects = structure.content[index]?.anchor?.pageRects;
|
||||
let pageIndex = null;
|
||||
if (Array.isArray(pageRects) && pageRects.length
|
||||
&& Number.isInteger(pageRects[0][0])) {
|
||||
pageIndex = pageRects[0][0];
|
||||
let rects = pageRects
|
||||
.filter(rect => rect[0] === pageIndex && rect.length >= 5)
|
||||
.map(rect => rect.slice(1));
|
||||
if (rects.length) {
|
||||
location.position = { pageIndex, rects };
|
||||
}
|
||||
}
|
||||
// Without geometry (EPUB, snapshot), the catalog's per-page content
|
||||
// ranges still say which page a block falls on
|
||||
if (pageIndex === null) {
|
||||
pageIndex = blockPages[index];
|
||||
}
|
||||
if (pageIndex === null) {
|
||||
return location;
|
||||
}
|
||||
location.pageIndex = pageIndex;
|
||||
// An EPUB's synthetic locations aren't page numbers -- a label from
|
||||
// them would read as one
|
||||
if (structure.catalog?.pageMappingType !== 'locations') {
|
||||
let label = structure.catalog?.pages?.[pageIndex]?.label;
|
||||
if (!label && structure.metadata?.processor?.type === 'pdf') {
|
||||
label = String(pageIndex + 1);
|
||||
}
|
||||
if (label) {
|
||||
location.pageLabel = label;
|
||||
}
|
||||
}
|
||||
return location;
|
||||
}
|
||||
|
||||
// blockIndex -> pageIndex, from the catalog's per-page content ranges: the
|
||||
// last page starting at or before the block. Built in one pass up front,
|
||||
// since every block needs it.
|
||||
function _getBlockPages(catalog, blockCount) {
|
||||
let pages = Array.isArray(catalog?.pages) ? catalog.pages : [];
|
||||
let starts = [];
|
||||
for (let i = 0; i < pages.length; i++) {
|
||||
let start = pages[i]?.contentRange?.[0]?.[0];
|
||||
if (Number.isInteger(start) && start >= 0) {
|
||||
starts.push({ start, pageIndex: i });
|
||||
}
|
||||
}
|
||||
starts.sort((a, b) => a.start - b.start);
|
||||
let blockPages = new Array(blockCount).fill(null);
|
||||
let current = null;
|
||||
let next = 0;
|
||||
for (let i = 0; i < blockCount; i++) {
|
||||
while (next < starts.length && starts[next].start <= i) {
|
||||
current = starts[next].pageIndex;
|
||||
next++;
|
||||
}
|
||||
blockPages[i] = current;
|
||||
}
|
||||
return blockPages;
|
||||
}
|
||||
|
||||
// Whether a block is a bibliography entry: flagged as one by the processor
|
||||
// (from entry structure and the in-text citation graph, so
|
||||
// language-independent), or made up entirely of blocks that are, as a
|
||||
// reference list is.
|
||||
function _isReferenceBlock(node) {
|
||||
if (node.reference) {
|
||||
return true;
|
||||
}
|
||||
let children = Array.isArray(node.content)
|
||||
? node.content.filter(child => child.text === undefined)
|
||||
: [];
|
||||
return children.length > 0 && children.every(_isReferenceBlock);
|
||||
}
|
||||
|
||||
// Outline entries flattened to their top-level block indexes, each with
|
||||
// its full heading path
|
||||
function _flattenOutline(items, ancestors) {
|
||||
let result = [];
|
||||
if (!Array.isArray(items)) {
|
||||
return result;
|
||||
}
|
||||
for (let item of items) {
|
||||
if (!item || typeof item !== 'object' || typeof item.title !== 'string') {
|
||||
continue;
|
||||
}
|
||||
let blockIndex = Array.isArray(item.ref) && Number.isInteger(item.ref[0])
|
||||
? item.ref[0]
|
||||
: null;
|
||||
let path = [...ancestors, item.title];
|
||||
if (blockIndex !== null && blockIndex >= 0) {
|
||||
result.push({ blockIndex, path, isHeading: true });
|
||||
}
|
||||
result.push(..._flattenOutline(item.children, path));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Page-start block indexes, for sectioning a document with no outline.
|
||||
// A page's contentRange starts with a content point whose first component
|
||||
// is the top-level block index.
|
||||
function _getPageBoundaries(catalog, blockCount) {
|
||||
let pages = Array.isArray(catalog?.pages) ? catalog.pages : [];
|
||||
let boundaries = [];
|
||||
let seen = new Set();
|
||||
for (let page of pages) {
|
||||
let start = page?.contentRange?.[0]?.[0];
|
||||
if (Number.isInteger(start) && start >= 0 && start < blockCount && !seen.has(start)) {
|
||||
seen.add(start);
|
||||
boundaries.push({ blockIndex: start, path: [] });
|
||||
}
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
// Plain text of a block, recursing into nested blocks. A local copy of the
|
||||
// module's un-exported text helper (structured-document-text/src/text.js).
|
||||
function _getNestedBlockPlainText(node) {
|
||||
if (node.text !== undefined) {
|
||||
return node.text;
|
||||
}
|
||||
if (!node.content) {
|
||||
return '';
|
||||
}
|
||||
let hasChildBlock = node.content.some(child => child.text === undefined);
|
||||
if (!hasChildBlock) {
|
||||
let result = '';
|
||||
for (let child of node.content) {
|
||||
if (child.text !== undefined) {
|
||||
result += child.text;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
let parts = [];
|
||||
for (let child of node.content) {
|
||||
if (child.text !== undefined) {
|
||||
continue;
|
||||
}
|
||||
let text = _getNestedBlockPlainText(child);
|
||||
if (text) {
|
||||
parts.push(text);
|
||||
}
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
async function _readValidCache({ sourceHash, cachePath, processorType }, options) {
|
||||
let bytes;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@ preferences-advanced-semantic-search-chinese =
|
|||
preferences-advanced-semantic-search-multilingual =
|
||||
.label = Multilingual
|
||||
preferences-advanced-semantic-search-model-description = “{ preferences-advanced-semantic-search-english.label }” and “{ preferences-advanced-semantic-search-chinese.label }” give the best results for libraries in those languages. “{ preferences-advanced-semantic-search-multilingual.label }” supports searching in and across many languages.
|
||||
preferences-advanced-semantic-search-index-fulltext =
|
||||
.label = Search full text of attachment files
|
||||
preferences-advanced-semantic-search-downloading = Downloading…
|
||||
preferences-advanced-semantic-search-downloading-progress = Downloading… { $percent }%
|
||||
preferences-advanced-semantic-search-indexing = Indexing…
|
||||
|
|
|
|||
|
|
@ -643,6 +643,7 @@ pane-related = Related
|
|||
pane-attachment-info = Attachment Info
|
||||
pane-attachment-preview = Preview
|
||||
pane-attachment-annotations = Annotations
|
||||
pane-search-results = Search Results
|
||||
|
||||
pane-header-attachment-associated =
|
||||
.label = Rename associated file
|
||||
|
|
@ -682,6 +683,14 @@ section-related =
|
|||
.label = { $count } Related
|
||||
section-attachment-info =
|
||||
.label = { pane-attachment-info }
|
||||
section-search-results =
|
||||
.label = { $count ->
|
||||
[one] { $count } Search Result
|
||||
*[other] { $count } Search Results
|
||||
}
|
||||
search-result-row-fulltext = Full Text
|
||||
search-result-row-show-more = Show More
|
||||
search-result-row-show-less = Show Less
|
||||
|
||||
section-button-remove =
|
||||
.tooltiptext = { general-remove }
|
||||
|
|
@ -720,6 +729,8 @@ sidenav-attachment-preview =
|
|||
.tooltiptext = { pane-attachment-preview }
|
||||
sidenav-attachment-annotations =
|
||||
.tooltiptext = { pane-attachment-annotations }
|
||||
sidenav-search-results =
|
||||
.tooltiptext = { pane-search-results }
|
||||
sidenav-libraries-collections =
|
||||
.tooltiptext = { pane-libraries-collections }
|
||||
sidenav-tags =
|
||||
|
|
|
|||
|
|
@ -118,6 +118,8 @@ pref("extensions.zotero.search.useLeftBound", true);
|
|||
pref("extensions.zotero.embeddings.model", "");
|
||||
// Set when the user stops indexing; nothing is indexed until indexing is started again
|
||||
pref("extensions.zotero.embeddings.indexingPaused", false);
|
||||
// Also index the full text of PDF/EPUB/snapshot attachments
|
||||
pref("extensions.zotero.embeddings.indexFulltext", false);
|
||||
|
||||
// Notes
|
||||
pref("extensions.zotero.note.fontFamily", "-apple-system, BlinkMacSystemFont, \"Segoe UI\", \"Helvetica Neue\", Helvetica, Arial, sans-serif");
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@
|
|||
@import "elements/attachmentRow";
|
||||
@import "elements/attachmentAnnotationsBox";
|
||||
@import "elements/annotationRow";
|
||||
@import "elements/searchResultsBox";
|
||||
@import "elements/noteRow";
|
||||
@import "elements/librariesCollectionsBox";
|
||||
@import "elements/duplicatesMergePane";
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ $item-pane-sections: (
|
|||
"libraries-collections": var(--accent-teal),
|
||||
"tags": var(--accent-orange),
|
||||
"related": var(--accent-wood),
|
||||
"search-results": var(--accent-gold),
|
||||
);
|
||||
|
||||
$tagColorsLookup: (
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
annotation-row {
|
||||
// search-result-row (the search-results section's chunk cards) shares this
|
||||
// structure and look, minus the parts it doesn't have (icon, action, tags):
|
||||
// the section already carries the magnifier in its head, so repeating it on
|
||||
// every card says nothing
|
||||
annotation-row, search-result-row {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
|
|
|
|||
75
scss/elements/_searchResultsBox.scss
Normal file
75
scss/elements/_searchResultsBox.scss
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// The section's icon is the quick-search magnifier, the same one the search
|
||||
// bar uses, rather than a per-size copy of its own. Both the sidenav button
|
||||
// and the section head otherwise derive their icon from the pane name (see
|
||||
// _itemPaneSidenav.scss and _collapsibleSection.scss), so each needs pointing
|
||||
// at the shared file.
|
||||
item-pane-sidenav .btn[data-pane="search-results"] {
|
||||
background-image: url("chrome://zotero/skin/20/universal/magnifier.svg");
|
||||
}
|
||||
|
||||
collapsible-section[data-pane="search-results"] > .head .title::before {
|
||||
background-image: icon-url("16/universal/magnifier.svg");
|
||||
}
|
||||
|
||||
search-results-box {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
&[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
& > collapsible-section {
|
||||
& > .body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
|
||||
@include comfortable {
|
||||
gap: 8px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The parts search-result-row adds on top of the shared annotation-row look
|
||||
// (see _annotationRow.scss): the section-part indicator and page label in the
|
||||
// head, and the clamp toggle under the quote
|
||||
search-result-row {
|
||||
.head {
|
||||
.part {
|
||||
font-weight: 400;
|
||||
color: var(--fill-secondary);
|
||||
margin-inline-start: 4px;
|
||||
}
|
||||
|
||||
.location {
|
||||
margin-inline-start: auto;
|
||||
color: var(--fill-secondary);
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
&.expanded .body .quote {
|
||||
-webkit-line-clamp: none;
|
||||
}
|
||||
|
||||
.show-more {
|
||||
align-self: flex-start;
|
||||
margin: 0 8px 4px 16px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--fill-secondary);
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
|
||||
&:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
&[hidden] {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,23 @@ describe("Zotero.Embeddings", function () {
|
|||
};
|
||||
}
|
||||
|
||||
// A Zotero.SDT.getSections() section, built from its blocks -- which are
|
||||
// what indexing reads, the section's own text and span being derived.
|
||||
// A block is its text, or an object adding flowClass/reference/location.
|
||||
function sdtSection(outlinePath, start, blocks) {
|
||||
let entries = blocks.map((block, i) => Object.assign(
|
||||
{ index: start + i, reference: false },
|
||||
typeof block == 'string' ? { text: block } : block
|
||||
));
|
||||
return {
|
||||
text: entries.map(entry => entry.text).join('\n'),
|
||||
outlinePath,
|
||||
startBlock: start,
|
||||
endBlock: start + entries.length - 1,
|
||||
blocks: entries
|
||||
};
|
||||
}
|
||||
|
||||
// Stands in for a measured model (see Zotero.Embeddings.ensureCalibration()),
|
||||
// which the test environment has no downloaded model to measure. The mean
|
||||
// is shaped like a real one -- mixed signs, and shorter than unit length,
|
||||
|
|
@ -335,6 +352,98 @@ describe("Zotero.Embeddings", function () {
|
|||
});
|
||||
});
|
||||
|
||||
describe("#getMatchingChunks()", function () {
|
||||
it("should return an item's matching chunks with their locations, best first", async function () {
|
||||
let axis = (index, scale = 1) => {
|
||||
let vector = Float32Array.from(testMean);
|
||||
vector[index] += scale;
|
||||
return vector;
|
||||
};
|
||||
let store = async (item, chunkIndex, vector, props = {}) => {
|
||||
let blob = new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO embeddings.itemEmbeddings "
|
||||
+ "(itemID, chunkIndex, embedding, sourceHash, chunkText, outlinePath, "
|
||||
+ "startBlock, endBlock, pageLabel, navPosition, sectionPart, sectionParts) "
|
||||
+ "VALUES (?, ?, ?, 'hash', ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[
|
||||
item.id,
|
||||
chunkIndex,
|
||||
blob,
|
||||
props.text ?? null,
|
||||
props.outlinePath ?? null,
|
||||
props.startBlock ?? null,
|
||||
props.endBlock ?? null,
|
||||
props.pageLabel ?? null,
|
||||
props.navPosition ?? null,
|
||||
props.sectionPart ?? null,
|
||||
props.sectionParts ?? null
|
||||
],
|
||||
{ debugParams: false }
|
||||
);
|
||||
};
|
||||
let item = await createDataObject('item');
|
||||
// A weak match, a strong match, and a chunk below the floor
|
||||
let mixed = axis(0, 0.4);
|
||||
mixed[1] += 1;
|
||||
await store(item, 0, mixed, {
|
||||
text: 'The introduction text', outlinePath: 'Introduction', startBlock: 0, endBlock: 4
|
||||
});
|
||||
await store(item, 1, axis(0), {
|
||||
text: 'The sampling text',
|
||||
outlinePath: 'Methods > Sampling',
|
||||
startBlock: 5,
|
||||
endBlock: 11,
|
||||
pageLabel: '7',
|
||||
navPosition: JSON.stringify({ pageIndex: 6, rects: [[10, 20, 300, 40]] }),
|
||||
sectionPart: 2,
|
||||
sectionParts: 3
|
||||
});
|
||||
await store(item, 2, axis(2), {
|
||||
text: 'The references text', outlinePath: 'References', startBlock: 12, endBlock: 20
|
||||
});
|
||||
|
||||
let query = axis(0, 0.9);
|
||||
query[1] += 0.1;
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'embedQuery').resolves(query)
|
||||
];
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO embeddings.itemEmbeddingsMeta (key, value) "
|
||||
+ "VALUES ('modelVersion', 'test-model/1')"
|
||||
);
|
||||
try {
|
||||
let chunks = await Zotero.Embeddings.getMatchingChunks('anything', item.id);
|
||||
// The chunk below the model's floor isn't a match
|
||||
assert.lengthOf(chunks, 2);
|
||||
// Best chunk first, each with its text and where it came from
|
||||
assert.equal(chunks[0].chunkIndex, 1);
|
||||
assert.equal(chunks[0].text, 'The sampling text');
|
||||
assert.equal(chunks[0].outlinePath, 'Methods > Sampling');
|
||||
assert.equal(chunks[0].startBlock, 5);
|
||||
assert.equal(chunks[0].endBlock, 11);
|
||||
assert.equal(chunks[0].pageLabel, '7');
|
||||
assert.deepEqual(chunks[0].position,
|
||||
{ pageIndex: 6, rects: [[10, 20, 300, 40]] });
|
||||
assert.equal(chunks[0].sectionPart, 2);
|
||||
assert.equal(chunks[0].sectionParts, 3);
|
||||
assert.equal(chunks[1].chunkIndex, 0);
|
||||
assert.isNull(chunks[1].position);
|
||||
assert.isAbove(chunks[0].score, chunks[1].score);
|
||||
// The limit caps how many come back
|
||||
let limited = await Zotero.Embeddings.getMatchingChunks('anything', item.id,
|
||||
{ limit: 1 });
|
||||
assert.lengthOf(limited, 1);
|
||||
assert.equal(limited[0].chunkIndex, 1);
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("#chunkText()", function () {
|
||||
// bge has no passage prefix, so the window less the two special tokens
|
||||
// that wrap every input is what a chunk's own text gets (see MODELS).
|
||||
|
|
@ -473,6 +582,180 @@ describe("Zotero.Embeddings", function () {
|
|||
|
||||
});
|
||||
|
||||
describe("#chunkSections()", function () {
|
||||
var fakeTokenizer = wordTokenizer();
|
||||
var stubs = [];
|
||||
|
||||
beforeEach(function () {
|
||||
stubs.push(sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'));
|
||||
stubs.push(sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(fakeTokenizer));
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
stubs = [];
|
||||
});
|
||||
|
||||
var words = (tag, n) => Array.from({ length: n }, (x, i) => `${tag}${i}`).join(' ');
|
||||
|
||||
it("shouldn't put two substantial sections in one chunk", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: words('alpha', 200), outlinePath: 'Introduction', startBlock: 0, endBlock: 4 },
|
||||
{ text: words('bravo', 200), outlinePath: 'Methods', startBlock: 5, endBlock: 9 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 2);
|
||||
// Neither chunk mixes the two sections, and each points back at
|
||||
// the blocks it covers
|
||||
assert.include(chunks[0].text, 'alpha0');
|
||||
assert.notInclude(chunks[0].text, 'bravo');
|
||||
assert.equal(chunks[0].startBlock, 0);
|
||||
assert.equal(chunks[0].endBlock, 4);
|
||||
assert.include(chunks[1].text, 'bravo0');
|
||||
assert.notInclude(chunks[1].text, 'alpha0');
|
||||
assert.equal(chunks[1].startBlock, 5);
|
||||
assert.equal(chunks[1].endBlock, 9);
|
||||
});
|
||||
|
||||
it("should prefix the embedded text with the section's outline path", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: words('alpha', 200), outlinePath: 'Results > Field studies', startBlock: 2, endBlock: 7 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 1);
|
||||
// What gets embedded carries the heading context; the display
|
||||
// text stays the plain piece
|
||||
assert.isTrue(chunks[0].embedText.startsWith('Results > Field studies\n\n'));
|
||||
assert.isTrue(chunks[0].text.startsWith('alpha0'));
|
||||
assert.equal(chunks[0].outlinePath, 'Results > Field studies');
|
||||
});
|
||||
|
||||
it("should combine sections too small to embed on their own", async function () {
|
||||
// Front matter before the first heading rides along with the
|
||||
// section that follows it, the way small paragraphs do in a note
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: 'Title page', outlinePath: '', startBlock: 0, endBlock: 0 },
|
||||
{ text: 'Copyright notice', outlinePath: '', startBlock: 1, endBlock: 1 },
|
||||
{ text: words('alpha', 200), outlinePath: 'Introduction', startBlock: 2, endBlock: 9 },
|
||||
{ text: words('bravo', 200), outlinePath: 'Methods', startBlock: 10, endBlock: 19 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 2);
|
||||
assert.include(chunks[0].text, 'Title page');
|
||||
assert.include(chunks[0].text, 'Copyright notice');
|
||||
assert.include(chunks[0].text, 'alpha0');
|
||||
assert.equal(chunks[0].startBlock, 0);
|
||||
assert.equal(chunks[0].endBlock, 9);
|
||||
// The substantial section that follows still gets a chunk of its own
|
||||
assert.include(chunks[1].text, 'bravo0');
|
||||
assert.notInclude(chunks[1].text, 'alpha0');
|
||||
});
|
||||
|
||||
it("should join a trailing small section to the previous chunk", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: words('alpha', 200), outlinePath: 'Body', startBlock: 0, endBlock: 9 },
|
||||
{ text: 'Short appendix note', outlinePath: 'Appendix', startBlock: 10, endBlock: 11 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 1);
|
||||
assert.include(chunks[0].text, 'Short appendix note');
|
||||
assert.equal(chunks[0].startBlock, 0);
|
||||
assert.equal(chunks[0].endBlock, 11);
|
||||
});
|
||||
|
||||
it("should split an oversized section into numbered pieces sharing its location", async function () {
|
||||
// 60 ten-token sentences: over the window, no paragraph breaks
|
||||
let sentences = Array.from({ length: 60 },
|
||||
(x, i) => `Sentence ${i} has some words about subject number ${i}.`);
|
||||
let position = { pageIndex: 4, rects: [[10, 20, 300, 40]] };
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{
|
||||
text: sentences.join(' '),
|
||||
outlinePath: 'Discussion',
|
||||
startBlock: 3,
|
||||
endBlock: 20,
|
||||
pageIndex: 4,
|
||||
pageLabel: '5',
|
||||
position
|
||||
}
|
||||
]);
|
||||
assert.isAbove(chunks.length, 1);
|
||||
// Every piece keeps its section's heading context and location,
|
||||
// and knows which piece of the section it is
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
let chunk = chunks[i];
|
||||
assert.isTrue(chunk.embedText.startsWith('Discussion\n\n'));
|
||||
assert.equal(chunk.outlinePath, 'Discussion');
|
||||
assert.equal(chunk.startBlock, 3);
|
||||
assert.equal(chunk.endBlock, 20);
|
||||
assert.equal(chunk.pageLabel, '5');
|
||||
assert.deepEqual(chunk.position, position);
|
||||
assert.equal(chunk.sectionPart, i + 1);
|
||||
assert.equal(chunk.sectionParts, chunks.length);
|
||||
}
|
||||
// No sentence was dropped
|
||||
let joined = chunks.map(chunk => chunk.text).join('\n');
|
||||
for (let sentence of sentences) {
|
||||
assert.include(joined, sentence);
|
||||
}
|
||||
});
|
||||
|
||||
it("should keep auxiliary sections as standalone chunks", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
// A small body section, a tiny caption, then a substantial
|
||||
// body section
|
||||
{ text: 'A short opening paragraph.', outlinePath: 'Results', startBlock: 0, endBlock: 0 },
|
||||
{
|
||||
text: 'Figure 3: Owl migration routes across the Baltic.',
|
||||
outlinePath: 'Results',
|
||||
startBlock: 1,
|
||||
endBlock: 1,
|
||||
auxiliary: true
|
||||
},
|
||||
{ text: words('alpha', 200), outlinePath: 'Results', startBlock: 2, endBlock: 9 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 2);
|
||||
// The caption is a chunk of its own, however small...
|
||||
let caption = chunks.find(chunk => chunk.auxiliary);
|
||||
assert.ok(caption);
|
||||
assert.equal(caption.text, 'Figure 3: Owl migration routes across the Baltic.');
|
||||
assert.equal(caption.sectionParts, 1);
|
||||
// ...and the small body section still merges with the body that
|
||||
// follows, straight across it
|
||||
let body = chunks.find(chunk => !chunk.auxiliary);
|
||||
assert.include(body.text, 'A short opening paragraph.');
|
||||
assert.include(body.text, 'alpha0');
|
||||
assert.notInclude(body.text, 'Figure 3');
|
||||
});
|
||||
|
||||
it("shouldn't fold a trailing small body section into an auxiliary chunk", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: words('alpha', 200), outlinePath: 'Body', startBlock: 0, endBlock: 9 },
|
||||
{
|
||||
text: 'Figure 1: A caption with enough words to keep.',
|
||||
outlinePath: 'Body',
|
||||
startBlock: 10,
|
||||
endBlock: 10,
|
||||
auxiliary: true
|
||||
},
|
||||
{ text: 'A trailing remnant paragraph.', outlinePath: 'Body', startBlock: 11, endBlock: 11 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 2);
|
||||
// The remnant joins the last body chunk, not the caption
|
||||
let caption = chunks.find(chunk => chunk.auxiliary);
|
||||
assert.equal(caption.text, 'Figure 1: A caption with enough words to keep.');
|
||||
let body = chunks.find(chunk => !chunk.auxiliary);
|
||||
assert.include(body.text, 'A trailing remnant paragraph.');
|
||||
});
|
||||
|
||||
it("should mark an unsplit section as its only piece", async function () {
|
||||
let chunks = await Zotero.Embeddings.Chunking.chunkSections([
|
||||
{ text: words('alpha', 200), outlinePath: 'Body', startBlock: 0, endBlock: 9 }
|
||||
]);
|
||||
assert.lengthOf(chunks, 1);
|
||||
assert.equal(chunks[0].sectionPart, 1);
|
||||
assert.equal(chunks[0].sectionParts, 1);
|
||||
assert.isNull(chunks[0].pageLabel);
|
||||
assert.isNull(chunks[0].position);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#getScoreFraction()", function () {
|
||||
it("should clamp scores into the measured display range", async function () {
|
||||
let stub = sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1');
|
||||
|
|
@ -769,7 +1052,8 @@ describe("Zotero.Embeddings", function () {
|
|||
try {
|
||||
await Zotero.Embeddings.initDB();
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO embeddings.itemEmbeddings VALUES (?, 0, ?, ?)",
|
||||
"REPLACE INTO embeddings.itemEmbeddings "
|
||||
+ "(itemID, chunkIndex, embedding, sourceHash) VALUES (?, 0, ?, ?)",
|
||||
[item.id, new Uint8Array([0, 0, 0, 0]), 'hash']
|
||||
);
|
||||
// The model switch clears the old vectors and announces the
|
||||
|
|
@ -800,7 +1084,8 @@ describe("Zotero.Embeddings", function () {
|
|||
try {
|
||||
let item = await createDataObject('item');
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO embeddings.itemEmbeddings VALUES (?, 0, ?, ?)",
|
||||
"INSERT INTO embeddings.itemEmbeddings "
|
||||
+ "(itemID, chunkIndex, embedding, sourceHash) VALUES (?, 0, ?, ?)",
|
||||
[item.id, new Uint8Array([0, 0, 0, 0]), 'hash']
|
||||
);
|
||||
await item.eraseTx();
|
||||
|
|
@ -1072,6 +1357,404 @@ describe("Zotero.Embeddings", function () {
|
|||
), 1);
|
||||
});
|
||||
|
||||
it("should index an attachment's sections when fulltext indexing is enabled", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of fulltext attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let texts = [];
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages').callsFake(async (passages) => {
|
||||
texts.push(...passages);
|
||||
return passages.map(() => vector);
|
||||
}),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
// The extraction itself is sdt.js's concern (see sdtTest.js);
|
||||
// what's under test is what indexing does with the sections
|
||||
sinon.stub(Zotero.SDT, 'getSections').resolves({
|
||||
ok: true,
|
||||
sections: [
|
||||
sdtSection('Introduction', 0, [{
|
||||
text: 'Owls migrate south when the winters turn cold.',
|
||||
pageIndex: 0,
|
||||
pageLabel: '2',
|
||||
position: { pageIndex: 0, rects: [[10, 20, 300, 40]] }
|
||||
}]),
|
||||
sdtSection('Methods', 4, [{
|
||||
text: 'Tracking devices recorded the routes of forty owls.',
|
||||
pageIndex: 1,
|
||||
pageLabel: '3',
|
||||
position: { pageIndex: 1, rects: [[10, 20, 300, 40]] }
|
||||
}])
|
||||
]
|
||||
})
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
// Both sections are far too small to embed on their own, so
|
||||
// they land in one chunk, prefixed with the first section's
|
||||
// outline path and covering both sections' blocks
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT chunkIndex, chunkText, outlinePath, startBlock, endBlock, "
|
||||
+ "pageLabel, navPosition, sectionPart, sectionParts "
|
||||
+ "FROM embeddings.itemEmbeddings WHERE itemID=?",
|
||||
attachment.id
|
||||
);
|
||||
assert.lengthOf(rows, 1);
|
||||
assert.equal(rows[0].outlinePath, 'Introduction');
|
||||
assert.equal(rows[0].startBlock, 0);
|
||||
assert.equal(rows[0].endBlock, 4);
|
||||
// The merged chunk takes its first section's page and
|
||||
// position, and is its section's only piece
|
||||
assert.equal(rows[0].pageLabel, '2');
|
||||
assert.deepEqual(JSON.parse(rows[0].navPosition),
|
||||
{ pageIndex: 0, rects: [[10, 20, 300, 40]] });
|
||||
assert.equal(rows[0].sectionPart, 1);
|
||||
assert.equal(rows[0].sectionParts, 1);
|
||||
// The stored preview text is the plain chunk, without the
|
||||
// outline-path context the embedded text carries
|
||||
assert.include(rows[0].chunkText, 'Owls migrate south');
|
||||
assert.include(rows[0].chunkText, 'Tracking devices');
|
||||
assert.isFalse(rows[0].chunkText.startsWith('Introduction\n\n'));
|
||||
let text = texts.find(t => t.includes('Owls migrate south'));
|
||||
assert.ok(text);
|
||||
assert.isTrue(text.startsWith('Introduction\n\n'));
|
||||
assert.include(text, 'Tracking devices');
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should drop attachment chunks when fulltext indexing is turned off", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of pruned attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.SDT, 'getSections').resolves({
|
||||
ok: true,
|
||||
sections: [
|
||||
sdtSection('', 0, ['A section with enough words to be worth indexing.'])
|
||||
]
|
||||
})
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
assert.ok(await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM embeddings.itemEmbeddings WHERE itemID=?",
|
||||
attachment.id
|
||||
));
|
||||
|
||||
// Turning the pref off makes attachments ineligible, and the
|
||||
// pref observer prunes their stored chunks. The observer runs
|
||||
// asynchronously, so poll (the test times out on failure).
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', false);
|
||||
while (await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM embeddings.itemEmbeddings WHERE itemID=?",
|
||||
attachment.id)) {
|
||||
await Zotero.Promise.delay(10);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should index auxiliary chunks with words and drop bare labels", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of captioned attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.SDT, 'getSections').resolves({
|
||||
ok: true,
|
||||
sections: [
|
||||
sdtSection('Results', 0, [
|
||||
'Body text about owl migration patterns.',
|
||||
{
|
||||
text: 'Figure 3: Owl migration routes across the Baltic.',
|
||||
flowClass: 'auxiliary'
|
||||
},
|
||||
'More body text about the wintering grounds.',
|
||||
// A standalone label with no words to rank by
|
||||
{ text: 'Figure 4', flowClass: 'auxiliary' }
|
||||
])
|
||||
]
|
||||
})
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT chunkText FROM embeddings.itemEmbeddings WHERE itemID=? "
|
||||
+ "ORDER BY chunkIndex",
|
||||
attachment.id
|
||||
);
|
||||
let texts = rows.map(row => row.chunkText);
|
||||
// The caption is lifted out into its own chunk, and the body
|
||||
// around it reads straight through
|
||||
assert.lengthOf(texts, 2);
|
||||
assert.include(texts[0], 'Body text about owl migration');
|
||||
assert.include(texts[0], 'More body text about the wintering');
|
||||
assert.notInclude(texts[0], 'Figure 3');
|
||||
assert.equal(texts[1], 'Figure 3: Owl migration routes across the Baltic.');
|
||||
// The bare label was dropped
|
||||
assert.notInclude(texts, 'Figure 4');
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should skip an attachment's reference entries", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of cited attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.SDT, 'getSections').resolves({
|
||||
ok: true,
|
||||
sections: [
|
||||
sdtSection('Discussion', 0, [
|
||||
'Owls migrate south when the winters turn cold.',
|
||||
// An entry cited inline, inside a body section
|
||||
{
|
||||
text: 'Smith, J. (2019). Owls. J. Birds 4, 1-10.',
|
||||
reference: true
|
||||
}
|
||||
]),
|
||||
// A section that's nothing but references
|
||||
sdtSection('References', 2, [
|
||||
{
|
||||
text: 'Doe, A. (2020). Migration. Nature 1, 2-3.',
|
||||
reference: true
|
||||
}
|
||||
])
|
||||
]
|
||||
})
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
let texts = await Zotero.DB.columnQueryAsync(
|
||||
"SELECT chunkText FROM embeddings.itemEmbeddings WHERE itemID=? "
|
||||
+ "ORDER BY chunkIndex",
|
||||
attachment.id
|
||||
);
|
||||
// Only the prose is indexed; a section left with nothing but
|
||||
// references contributes no chunk at all
|
||||
assert.lengthOf(texts, 1);
|
||||
assert.include(texts[0], 'Owls migrate south');
|
||||
assert.notInclude(texts[0], 'Smith, J.');
|
||||
assert.notInclude(texts[0], 'Doe, A.');
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should index smaller attachments before larger ones", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of sized attachments' });
|
||||
// The big one is created first, so insertion order can't account
|
||||
// for the result on its own
|
||||
let big = await importPDFAttachment(item);
|
||||
let small = await importPDFAttachment(item);
|
||||
// The size the enqueue order goes by comes from Zotero's own
|
||||
// fulltext index
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO fulltextItems (itemID, totalPages) VALUES (?, ?)", [big.id, 800]);
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO fulltextItems (itemID, totalPages) VALUES (?, ?)", [small.id, 2]);
|
||||
|
||||
let extracted = [];
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.SDT, 'getSections').callsFake(async (itemID) => {
|
||||
if (itemID === big.id || itemID === small.id) {
|
||||
extracted.push(itemID);
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
sections: [
|
||||
sdtSection('', 0, ['A section with enough words to be worth indexing.'])
|
||||
]
|
||||
};
|
||||
})
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
assert.deepEqual(extracted, [small.id, big.id]);
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should fall back to an attachment's plain text when structured extraction fails", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of fallback attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
Object.defineProperty(attachment, 'attachmentText', {
|
||||
get: () => Promise.resolve('A plain paragraph of text about owl migration routes.'),
|
||||
configurable: true
|
||||
});
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.SDT, 'getSections').resolves({ ok: false, reason: 'failed' })
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
// The flat text is chunked like a note: embedded, previewable,
|
||||
// but without section locations
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT embedding, chunkText, outlinePath, startBlock, endBlock "
|
||||
+ "FROM embeddings.itemEmbeddings WHERE itemID=?",
|
||||
attachment.id
|
||||
);
|
||||
assert.lengthOf(rows, 1);
|
||||
assert.isNotNull(rows[0].embedding);
|
||||
assert.include(rows[0].chunkText, 'owl migration routes');
|
||||
assert.isNull(rows[0].outlinePath);
|
||||
assert.isNull(rows[0].startBlock);
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
delete attachment.attachmentText;
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should record an attachment with no extractable text as processed", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item', { title: 'Parent of empty attachment' });
|
||||
let attachment = await importPDFAttachment(item);
|
||||
Object.defineProperty(attachment, 'attachmentText', {
|
||||
get: () => Promise.resolve(''),
|
||||
configurable: true
|
||||
});
|
||||
|
||||
let vector = new Float32Array(4).fill(0.5);
|
||||
let getSectionsStub = sinon.stub(Zotero.SDT, 'getSections')
|
||||
.resolves({ ok: false, reason: 'failed' });
|
||||
let ourCalls = () => getSectionsStub.getCalls()
|
||||
.filter(call => call.args[0] === attachment.id).length;
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Embeddings, 'embedPassages')
|
||||
.callsFake(async texts => texts.map(() => vector)),
|
||||
sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
|
||||
sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
|
||||
sinon.stub(Zotero.Embeddings, 'preloadModel').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'ensureCalibration').resolves(),
|
||||
sinon.stub(Zotero.Embeddings, 'getModelName').returns('bge-small-en-v1.5'),
|
||||
sinon.stub(Zotero.Embeddings.Chunking, 'getTokenizer').resolves(wordTokenizer()),
|
||||
sinon.stub(Zotero.Embeddings, 'embedQuery').resolves(Float32Array.from(testMean)),
|
||||
getSectionsStub
|
||||
];
|
||||
try {
|
||||
Zotero.Prefs.set('embeddings.indexFulltext', true);
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
|
||||
// The attempt is recorded as a single embedding-less row, so
|
||||
// the item counts as processed and the progress counts align
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT embedding, sourceHash FROM embeddings.itemEmbeddings WHERE itemID=?",
|
||||
attachment.id
|
||||
);
|
||||
assert.lengthOf(rows, 1);
|
||||
assert.isNull(rows[0].embedding);
|
||||
assert.ok(rows[0].sourceHash);
|
||||
assert.equal(ourCalls(), 1);
|
||||
|
||||
// A processed-but-empty item can't be scored, and doesn't
|
||||
// break scoring for anything else
|
||||
let scores = await Zotero.Embeddings.scoreItemIDs('anything', [attachment.id]);
|
||||
assert.isFalse(scores.has(attachment.id));
|
||||
|
||||
// The record makes later passes skip the attachment without
|
||||
// re-extracting, until the file changes
|
||||
await Zotero.Embeddings.Indexing.startIndexing();
|
||||
assert.equal(ourCalls(), 1);
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
delete attachment.attachmentText;
|
||||
Zotero.Prefs.clear('embeddings.indexFulltext');
|
||||
}
|
||||
});
|
||||
|
||||
it("should look up stored hashes without a query per item", async function () {
|
||||
this.timeout(60000);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@ describe("Zotero.SDT", function () {
|
|||
let testSDTPackBytes;
|
||||
let documentWorkerMetadata;
|
||||
|
||||
// A section without its block detail, for asserting the overall shape
|
||||
function summarize({ blocks, ...section }) {
|
||||
return section;
|
||||
}
|
||||
|
||||
before(async function () {
|
||||
let pako = getTestRequire()('pako');
|
||||
documentWorkerMetadata = JSON.parse(await Zotero.File.getContentsFromURLAsync(
|
||||
|
|
@ -31,6 +36,218 @@ describe("Zotero.SDT", function () {
|
|||
assert.deepEqual(progress, []);
|
||||
});
|
||||
|
||||
it("should return outline-based sections from getSections()", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let pako = getTestRequire()('pako');
|
||||
let bytes = makeTestSDTPackV1WithContent(documentWorkerMetadata, pako, {
|
||||
outline: [
|
||||
{ title: 'Introduction', ref: [1] },
|
||||
{ title: 'Methods', ref: [3] },
|
||||
],
|
||||
blocks: [
|
||||
{ content: [{ text: 'Front matter on the title page' }] },
|
||||
{ content: [{ text: 'Introduction' }] },
|
||||
{ content: [{ text: 'Owls are nocturnal birds of prey.' }] },
|
||||
{ content: [{ text: 'Methods' }] },
|
||||
{ content: [{ text: 'We tracked forty owls with GPS loggers.' }] },
|
||||
{ flowClass: 'excluded', content: [{ text: 'Page 3' }] },
|
||||
],
|
||||
});
|
||||
await writeTestSDTCache(item, bytes);
|
||||
|
||||
let result = await Zotero.SDT.getSections(item.id);
|
||||
assert.isTrue(result.ok);
|
||||
assert.lengthOf(result.sections, 3);
|
||||
// Content before the first heading is a section of its own
|
||||
assert.deepEqual(summarize(result.sections[0]), {
|
||||
text: 'Front matter on the title page',
|
||||
outlinePath: '',
|
||||
startBlock: 0,
|
||||
endBlock: 0,
|
||||
});
|
||||
// A section starting at a heading carries the heading in its path,
|
||||
// not its text, and runs to the next heading
|
||||
assert.deepEqual(summarize(result.sections[1]), {
|
||||
text: 'Owls are nocturnal birds of prey.',
|
||||
outlinePath: 'Introduction',
|
||||
startBlock: 1,
|
||||
endBlock: 2,
|
||||
});
|
||||
assert.deepEqual(result.sections[1].blocks, [
|
||||
{
|
||||
index: 2,
|
||||
text: 'Owls are nocturnal birds of prey.',
|
||||
reference: false,
|
||||
},
|
||||
]);
|
||||
// Excluded blocks (running heads, page numbers) are left out
|
||||
assert.deepEqual(summarize(result.sections[2]), {
|
||||
text: 'We tracked forty owls with GPS loggers.',
|
||||
outlinePath: 'Methods',
|
||||
startBlock: 3,
|
||||
endBlock: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it("should fall back to per-page sections from getSections() when there's no outline", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let pako = getTestRequire()('pako');
|
||||
let bytes = makeTestSDTPackV1WithContent(documentWorkerMetadata, pako, {
|
||||
pages: [
|
||||
{ contentRange: [[0], [1]] },
|
||||
{ contentRange: [[2], [3]] },
|
||||
],
|
||||
blocks: [
|
||||
{ content: [{ text: 'First page first paragraph.' }] },
|
||||
{ content: [{ text: 'First page second paragraph.' }] },
|
||||
{ content: [{ text: 'Second page first paragraph.' }] },
|
||||
{ content: [{ text: 'Second page second paragraph.' }] },
|
||||
],
|
||||
});
|
||||
await writeTestSDTCache(item, bytes);
|
||||
|
||||
let result = await Zotero.SDT.getSections(item.id);
|
||||
assert.isTrue(result.ok);
|
||||
assert.lengthOf(result.sections, 2);
|
||||
assert.equal(result.sections[0].text,
|
||||
'First page first paragraph.\nFirst page second paragraph.');
|
||||
assert.equal(result.sections[0].startBlock, 0);
|
||||
assert.equal(result.sections[0].endBlock, 1);
|
||||
assert.equal(result.sections[1].text,
|
||||
'Second page first paragraph.\nSecond page second paragraph.');
|
||||
assert.equal(result.sections[1].outlinePath, '');
|
||||
});
|
||||
|
||||
it("should flag reference entries in getSections()", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let pako = getTestRequire()('pako');
|
||||
let bytes = makeTestSDTPackV1WithContent(documentWorkerMetadata, pako, {
|
||||
outline: [
|
||||
{ title: 'Discussion', ref: [0] },
|
||||
{ title: 'References', ref: [3] },
|
||||
],
|
||||
blocks: [
|
||||
{ content: [{ text: 'Heading text' }] },
|
||||
{ content: [{ text: 'Prose about owl migration.' }] },
|
||||
// An entry cited inline, footnote-style, inside a body section
|
||||
{ reference: true, content: [{ text: 'Smith, J. (2019). Owls. J. Birds 4, 1-10.' }] },
|
||||
{ content: [{ text: 'References' }] },
|
||||
// A bibliography as a list whose items carry the flag
|
||||
{
|
||||
type: 'list',
|
||||
content: [
|
||||
{
|
||||
type: 'listitem',
|
||||
reference: true,
|
||||
content: [{ text: 'Doe, A. (2020). Migration. Nature 1, 2-3.' }],
|
||||
},
|
||||
{
|
||||
type: 'listitem',
|
||||
reference: true,
|
||||
content: [{ text: 'Roe, B. (2021). Wintering. Science 2, 4-5.' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
await writeTestSDTCache(item, bytes);
|
||||
|
||||
let result = await Zotero.SDT.getSections(item.id);
|
||||
assert.isTrue(result.ok);
|
||||
// Reference entries are reported, not dropped -- it's the caller's
|
||||
// call whether they're worth reading
|
||||
assert.lengthOf(result.sections, 2);
|
||||
assert.equal(result.sections[0].outlinePath, 'Discussion');
|
||||
assert.deepEqual(result.sections[0].blocks.map(block => block.reference),
|
||||
[false, true]);
|
||||
// A list whose items all carry the flag is a reference block itself
|
||||
assert.equal(result.sections[1].outlinePath, 'References');
|
||||
assert.lengthOf(result.sections[1].blocks, 1);
|
||||
assert.isTrue(result.sections[1].blocks[0].reference);
|
||||
assert.include(result.sections[1].blocks[0].text, 'Doe, A. (2020)');
|
||||
assert.include(result.sections[1].blocks[0].text, 'Roe, B. (2021)');
|
||||
});
|
||||
|
||||
it("should report each block's flow class in getSections()", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let pako = getTestRequire()('pako');
|
||||
let bytes = makeTestSDTPackV1WithContent(documentWorkerMetadata, pako, {
|
||||
outline: [
|
||||
{ title: 'Results', ref: [0] },
|
||||
],
|
||||
blocks: [
|
||||
{ content: [{ text: 'Results' }] },
|
||||
{ content: [{ text: 'Prose before the figure.' }] },
|
||||
{
|
||||
flowClass: 'auxiliary',
|
||||
content: [{ text: 'Figure 3: Owl migration routes across the Baltic.' }],
|
||||
},
|
||||
{ content: [{ text: 'Prose after the figure.' }] },
|
||||
],
|
||||
});
|
||||
await writeTestSDTCache(item, bytes);
|
||||
|
||||
let result = await Zotero.SDT.getSections(item.id);
|
||||
assert.isTrue(result.ok);
|
||||
// The caption stays where the document put it, marked as auxiliary
|
||||
// flow so a caller can lift it out if that suits them
|
||||
assert.lengthOf(result.sections, 1);
|
||||
assert.equal(result.sections[0].text, 'Prose before the figure.\n'
|
||||
+ 'Figure 3: Owl migration routes across the Baltic.\n'
|
||||
+ 'Prose after the figure.');
|
||||
assert.deepEqual(result.sections[0].blocks.map(block => block.flowClass),
|
||||
[undefined, 'auxiliary', undefined]);
|
||||
assert.deepEqual(result.sections[0].blocks.map(block => block.index), [1, 2, 3]);
|
||||
});
|
||||
|
||||
it("should report page and position info from getSections()", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let pako = getTestRequire()('pako');
|
||||
let bytes = makeTestSDTPackV1WithContent(documentWorkerMetadata, pako, {
|
||||
outline: [
|
||||
{ title: 'Introduction', ref: [1] },
|
||||
],
|
||||
pages: [
|
||||
{ label: 'ix', contentRange: [[0], [1]] },
|
||||
{ label: '10', contentRange: [[1], [3]] },
|
||||
],
|
||||
blocks: [
|
||||
{
|
||||
anchor: { pageRects: [[0, 10, 700, 300, 720]] },
|
||||
content: [{ text: 'Front matter on the title page' }],
|
||||
},
|
||||
{
|
||||
anchor: { pageRects: [[1, 10, 700, 300, 720]] },
|
||||
content: [{ text: 'Introduction' }],
|
||||
},
|
||||
{
|
||||
anchor: { pageRects: [[1, 10, 600, 300, 680], [1, 10, 500, 300, 580]] },
|
||||
content: [{ text: 'Owls are nocturnal birds of prey.' }],
|
||||
},
|
||||
],
|
||||
});
|
||||
await writeTestSDTCache(item, bytes);
|
||||
|
||||
let result = await Zotero.SDT.getSections(item.id);
|
||||
assert.isTrue(result.ok);
|
||||
assert.lengthOf(result.sections, 2);
|
||||
// Page and label come from the section's first anchored block, and
|
||||
// position is that block's page geometry
|
||||
assert.equal(result.sections[0].pageIndex, 0);
|
||||
assert.equal(result.sections[0].pageLabel, 'ix');
|
||||
assert.deepEqual(result.sections[0].position,
|
||||
{ pageIndex: 0, rects: [[10, 700, 300, 720]] });
|
||||
// A heading-started section is anchored at its heading, so the
|
||||
// reader lands on the section start
|
||||
assert.equal(result.sections[1].pageIndex, 1);
|
||||
assert.equal(result.sections[1].pageLabel, '10');
|
||||
assert.deepEqual(result.sections[1].position,
|
||||
{ pageIndex: 1, rects: [[10, 700, 300, 720]] });
|
||||
// Its blocks are located individually, at their own geometry
|
||||
assert.deepEqual(result.sections[1].blocks[0].position,
|
||||
{ pageIndex: 1, rects: [[10, 600, 300, 680], [10, 500, 300, 580]] });
|
||||
});
|
||||
|
||||
it("should generate the pack when missing", async function () {
|
||||
let item = await importFileAttachment('test.pdf');
|
||||
let cachePath = getSDTCachePath(item);
|
||||
|
|
@ -450,6 +667,64 @@ describe("Zotero.SDT", function () {
|
|||
return bytes;
|
||||
}
|
||||
|
||||
// A v1 pack with real content blocks and catalog, for section/outline
|
||||
// consumers (see makeEmptyTestSDTPackV1() for the layout)
|
||||
function makeTestSDTPackV1WithContent(metadata, pako, { outline = [], pages = [], blocks = [] } = {}) {
|
||||
if (metadata.SDT_PACK_VERSION !== 1) {
|
||||
throw new Error('Unsupported test SDT pack version');
|
||||
}
|
||||
const HEADER_LENGTH = 16;
|
||||
// Two entries each of chunk byte offsets and chunk block starts, after
|
||||
// the metadata and catalog lengths
|
||||
const INDEX_LENGTH = 8 + 2 * 4 + 2 * 4;
|
||||
let encoder = new TextEncoder();
|
||||
let schemaVersion = metadata.SDT_SCHEMA_VERSION.split('.').map(Number);
|
||||
let metadataBytes = pako.deflateRaw(JSON.stringify({
|
||||
processor: {
|
||||
type: 'pdf',
|
||||
version: metadata.SDT_PROCESSOR_VERSIONS.pdf,
|
||||
},
|
||||
dateCreated: '2026-01-01T00:00:00.000Z',
|
||||
source: { hash: TEST_PDF_HASH },
|
||||
}));
|
||||
let catalogBytes = pako.deflateRaw(JSON.stringify({ pages, outline }));
|
||||
// One content chunk: an offset table, then the block JSON back to back
|
||||
let blockByteArrays = blocks.map(block => encoder.encode(JSON.stringify(block)));
|
||||
let chunkBytes = new Uint8Array(
|
||||
blocks.length * 4 + blockByteArrays.reduce((sum, b) => sum + b.byteLength, 0)
|
||||
);
|
||||
let chunkView = new DataView(chunkBytes.buffer);
|
||||
let blockOffset = 0;
|
||||
let writeOffset = blocks.length * 4;
|
||||
for (let i = 0; i < blockByteArrays.length; i++) {
|
||||
chunkView.setUint32(i * 4, blockOffset, true);
|
||||
blockOffset += blockByteArrays[i].byteLength;
|
||||
chunkBytes.set(blockByteArrays[i], writeOffset);
|
||||
writeOffset += blockByteArrays[i].byteLength;
|
||||
}
|
||||
let compressedChunk = pako.deflateRaw(chunkBytes);
|
||||
|
||||
let payloadOffset = HEADER_LENGTH + INDEX_LENGTH;
|
||||
let bytes = new Uint8Array(
|
||||
payloadOffset + metadataBytes.byteLength + catalogBytes.byteLength
|
||||
+ compressedChunk.byteLength
|
||||
);
|
||||
bytes.set(SDT_PACK_MAGIC, 0);
|
||||
bytes.set([metadata.SDT_PACK_VERSION, ...schemaVersion], 8);
|
||||
let view = new DataView(bytes.buffer);
|
||||
view.setUint32(12, INDEX_LENGTH, true);
|
||||
view.setUint32(HEADER_LENGTH, metadataBytes.byteLength, true);
|
||||
view.setUint32(HEADER_LENGTH + 4, catalogBytes.byteLength, true);
|
||||
// chunkByteOffsets [0, byteLength], chunkBlockStarts [0, blockCount]
|
||||
view.setUint32(HEADER_LENGTH + 12, compressedChunk.byteLength, true);
|
||||
view.setUint32(HEADER_LENGTH + 20, blocks.length, true);
|
||||
bytes.set(metadataBytes, payloadOffset);
|
||||
bytes.set(catalogBytes, payloadOffset + metadataBytes.byteLength);
|
||||
bytes.set(compressedChunk,
|
||||
payloadOffset + metadataBytes.byteLength + catalogBytes.byteLength);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function decodeBase64Bytes(base64) {
|
||||
let binary = atob(base64);
|
||||
let bytes = new Uint8Array(binary.length);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue