diff --git a/chrome/content/zotero/components/windowed-list.js b/chrome/content/zotero/components/windowed-list.js index cd264e6851..9f820b0cfe 100644 --- a/chrome/content/zotero/components/windowed-list.js +++ b/chrome/content/zotero/components/windowed-list.js @@ -159,11 +159,7 @@ module.exports = class { Object.assign(this, options); const { itemHeight, targetElement, innerElem } = this; const itemCount = this._getItemCount(); - const [offsetIdx, offset] = this._rowOffsets.at(-1); - const listHeight = offset + (itemCount - offsetIdx) * this.itemHeight; - innerElem.style.position = 'relative'; - innerElem.style.height = `${listHeight}px`; - + // Recalculate custom row height offsets this._rowOffsets = [[0, 0]]; let previousRowOffset = 0; @@ -176,6 +172,13 @@ module.exports = class { previousRowOffset = offset; } + // From the offsets just recalculated: the list is as tall as the rows + // it now has, not as the rows it had + const [offsetIdx, offset] = this._rowOffsets.at(-1); + const listHeight = offset + (itemCount - offsetIdx) * itemHeight; + innerElem.style.position = 'relative'; + innerElem.style.height = `${listHeight}px`; + this.scrollDirection = 0; this.scrollOffset = targetElement.scrollTop; } diff --git a/chrome/content/zotero/itemTree.jsx b/chrome/content/zotero/itemTree.jsx index c27c01be8e..a6057d04f9 100644 --- a/chrome/content/zotero/itemTree.jsx +++ b/chrome/content/zotero/itemTree.jsx @@ -1288,6 +1288,57 @@ var ItemTree = class ItemTree extends LibraryTree { this._cachedScrollPosition = this._saveScrollPosition(); } + /** + * A search-match row shows two lines -- where its passage is, and the + * line of it worth reading -- so it stands a line taller than the rows + * around it. Neither line wraps, so the height is the same for every one + * of them and can be told without measuring anything. + * + * @return {Number} + */ + _getSearchMatchRowHeight() { + let textHeight = this.tree._renderedTextHeight + * (this.tree.props.disableFontSizeScaling ? 1 : Zotero.Prefs.get('fontSize')); + // Two lines, and half the room a one-line row leaves around its text: + // the second line gives the row enough weight without it + let padding = (this.tree._rowHeight - textHeight) / 2; + return Math.round(textHeight * 2 + padding); + } + + /** + * Tell the table which rows are taller than the rest. + * + * The heights are keyed by row index, so they mean something different + * after every insertion, removal and sort -- which is why this runs from + * handleRowModelUpdate(), where all of those end up, rather than from the + * places that change rows. + */ + _updateSearchMatchRowHeights() { + if (!this.tree?._jsWindow) { + return; + } + let height = null; + let heights = []; + let indexes = []; + for (let i = 0, count = this.getRowCount(); i < count; i++) { + let type = this.getRow(i)?.type; + if (type == 'search-match' || type == 'search-match-placeholder') { + height ??= this._getSearchMatchRowHeight(); + heights.push([i, height]); + indexes.push(i); + } + } + // Most updates leave the tall rows exactly where they were. Telling + // the table again would have it rebuild its offsets and forget which + // way the view was moving for nothing. + let signature = height + '|' + indexes.join(','); + if (signature === this._searchMatchRowHeights) { + return; + } + this._searchMatchRowHeights = signature; + this.tree.updateCustomRowHeights(heights); + } + /** * NOTE: This method must not trigger further update events (e.g. by calling * sort() or refresh()) to avoid recursive update loops and UI flashing. @@ -1333,6 +1384,10 @@ var ItemTree = class ItemTree extends LibraryTree { this._treebox && this._treebox.scrollTo(0); } + // Before anything is drawn or scrolled to: the rows just changed, and + // heights are what say where each one sits + this._updateSearchMatchRowHeights(); + if (rows === true) { if (this.tree) { this.tree.invalidate(); @@ -2354,6 +2409,12 @@ var ItemTree = class ItemTree extends LibraryTree { div.classList.toggle('first-highlighted', this._highlightedRows.has(rowData.id) && !this._highlightedRows.has(prevRowID)); div.classList.toggle('last-highlighted', this._highlightedRows.has(rowData.id) && !this._highlightedRows.has(nextRowID)); div.classList.toggle('annotation-row', row.type === 'annotation'); + // Both kinds of match row: one stands in for the other, and they lay + // out the same. Toggled here rather than set while rendering, since + // the tree recycles a row's div for whatever row next needs one. + div.classList.toggle( + 'search-match-row', + row.type === 'search-match' || row.type === 'search-match-placeholder'); div.classList.toggle('library-header-row', row.type === 'library-header'); div.classList.toggle('spacer-row', row.type === 'spacer'); if (row.type !== 'annotation') { @@ -2437,8 +2498,10 @@ var ItemTree = class ItemTree extends LibraryTree { } if (isFirstColumn) { + // A row with no icon of its own gets none: the indent and twisty + // the tree adds don't depend on one const icon = row.getIcon(); - icon.classList.add('cell-icon', 'item-icon'); + icon?.classList.add('cell-icon', 'item-icon'); if (cell.querySelector('.cell-text') === null) { let textSpan = document.createElement('span'); @@ -2448,7 +2511,9 @@ var ItemTree = class ItemTree extends LibraryTree { cell.append(textSpan); } - cell.prepend(icon); + if (icon) { + cell.prepend(icon); + } cell.classList.add('first-column'); } diff --git a/chrome/content/zotero/itemTreeRow.js b/chrome/content/zotero/itemTreeRow.js index 165964d64b..7f05756c5c 100644 --- a/chrome/content/zotero/itemTreeRow.js +++ b/chrome/content/zotero/itemTreeRow.js @@ -12,6 +12,10 @@ XPCOMUtils.defineLazyPreferenceGetter( const ATTACHMENT_STATE_LOAD_DELAY = 150; +// Headings named above a search match. The path from the document's root can +// be several levels deep, and the deepest are the ones that place the passage. +const LOCATION_HEADINGS = 2; + /** * Base row in an ItemTree. * @@ -690,6 +694,30 @@ class SearchMatchItemTreeRow extends ItemTreeRow { return super.getField(field); } + /** + * Where in the document this row's passage sits, for the line above the + * quote: the headings it falls under and the page it starts on. Only the + * deepest headings are named -- a full outline path is longer than the + * line, and the leaf is what says where you'd land. + * + * Empty for a passage that knows neither, which is what a document with + * no structured text to read gives. + * + * @return {String} + */ + getLocationLabel() { + let { outlinePath, pageLabel } = this.ref.entry; + let parts = []; + if (outlinePath) { + parts.push(outlinePath.split(' > ').slice(-LOCATION_HEADINGS).join(' › ')); + } + if (pageLabel) { + parts.push(Zotero.ftl.formatValueSync( + 'items-search-match-page', { page: pageLabel })); + } + return parts.join(' · '); + } + /** * A match row's bar shows the strength of the evidence it displays, * rather than its item's relevance @@ -698,12 +726,20 @@ class SearchMatchItemTreeRow extends ItemTreeRow { return this.ref.entry?.strength ?? null; } + /** + * No icon: every match row would carry the same one, which would say + * nothing while taking room from the quote + */ getIcon() { - let icon = getCSSIcon('search'); - icon.classList.add('icon-item-type'); - return icon; + return null; } + /** + * A match row spans the tree's whole width with one cell. It shows a + * passage rather than an item, so the columns describe nothing about it + * -- including the relevance bar, which would rank passages against each + * other where the eye is meant to be reading them. + */ renderRow(div, index, columns, rowData, renderCtx) { let titleColumn = Object.assign( {}, @@ -711,19 +747,37 @@ class SearchMatchItemTreeRow extends ItemTreeRow { { className: 'title' } ); div.appendChild(renderCtx.renderCell(index, rowData.title, titleColumn, true)); - // The relevance bar while a best-match search shows the Relevance column - let relevanceColumn = columns.find(column => column.dataKey == 'relevance'); - if (relevanceColumn && !relevanceColumn.hidden) { - let cell = renderCtx.renderCell(index, rowData?.relevance, relevanceColumn, false); - if (cell) { - div.appendChild(cell); - } - } } + /** + * Stack the row's lines beside the tree's indent and twisty, which are + * added to the first cell of every row and would otherwise be stacked + * along with them + * + * @param {...Element} lines + * @return {Element} + */ + _renderLines(...lines) { + let wrapper = document.createElement('span'); + wrapper.className = 'search-match-lines'; + wrapper.append(...lines); + return wrapper; + } + + /** + * Two lines: where the passage is, and the line of it worth reading. + * Neither wraps, so every match row is the same height and the tree can + * tell what that height is without measuring (see + * ItemTree#_getSearchMatchRowHeight()). + */ renderPrimaryCell(index, data, column) { let span = document.createElement('span'); span.className = `cell ${column.className} primary`; + + let locationSpan = document.createElement('span'); + locationSpan.className = 'search-match-location'; + locationSpan.textContent = this.getLocationLabel(); + let textSpan = document.createElement('span'); textSpan.className = 'cell-text'; let { text, ranges } = this.getQuotedLine(); @@ -741,7 +795,8 @@ class SearchMatchItemTreeRow extends ItemTreeRow { if (last < text.length) { textSpan.append(text.slice(last)); } - span.append(textSpan); + + span.append(this._renderLines(locationSpan, textSpan)); return span; } } @@ -761,13 +816,17 @@ class SearchMatchPlaceholderItemTreeRow extends SearchMatchItemTreeRow { return ''; } + getLocationLabel() { + return ''; + } + renderPrimaryCell(index, data, column) { let span = document.createElement('span'); span.className = `cell ${column.className} primary`; let textSpan = document.createElement('span'); textSpan.className = 'cell-text search-match-pending'; textSpan.textContent = Zotero.ftl.formatValueSync('items-search-match-pending'); - span.append(textSpan); + span.append(this._renderLines(textSpan)); return span; } } diff --git a/chrome/content/zotero/xpcom/bestMatch.js b/chrome/content/zotero/xpcom/bestMatch.js index 45a7081052..bb33d32143 100644 --- a/chrome/content/zotero/xpcom/bestMatch.js +++ b/chrome/content/zotero/xpcom/bestMatch.js @@ -45,7 +45,7 @@ Zotero.BestMatch = new function () { const SEMANTIC_WEIGHT = 0.7; const LEXICAL_WEIGHT = 0.3; // About a line: what a passage is quoted down to for a one-line preview - const SNIPPET_CHARS = 200; + const SNIPPET_CHARS = 150; // Most passages shown for one item. The strongest few say what the item // has to offer, and quoting a passage costs work -- sometimes the model's // -- so passages past this are not worth deriving. @@ -441,12 +441,14 @@ Zotero.BestMatch = new function () { return []; } let texts = passages.map(passage => passage.text); - // Chunks always highlight the query's literal matches: finding - // them in texts already in hand is cheap, unlike scanning a - // document, so it isn't gated on the item having matched - // lexically + // Locating the query's words in texts already in hand is cheap, + // unlike scanning a document, so it isn't gated on the item + // having matched lexically -- only on the lexical engine being + // one this session listens to at all let [ranges, lexical] = await Promise.all([ - Zotero.Lexical.findMatchRanges(queryText, texts), + this._lexicalEnabled() + ? Zotero.Lexical.findMatchRanges(queryText, texts) + : texts.map(() => []), this._lexicalApplies(itemID) ? Zotero.Lexical.scoreTexts(queryText, texts) : null ]); let entries = []; @@ -459,13 +461,18 @@ Zotero.BestMatch = new function () { if (passage.score === undefined && !share) { continue; } + // Over the engines that spoke for this item, so a strength is + // the same 0-1 fraction whether one weighed the passage or + // both did + let weighed = passage.score !== undefined; + let semanticWeight = weighed ? SEMANTIC_WEIGHT : 0; + let lexicalWeight = lexical ? LEXICAL_WEIGHT : 0; + let fraction = weighed ? Zotero.Embeddings.getScoreFraction(passage.score) : 0; entries.push({ ...passage, ranges: ranges[i], - strength: passage.score === undefined - ? share - : SEMANTIC_WEIGHT * Zotero.Embeddings.getScoreFraction(passage.score) - + LEXICAL_WEIGHT * share + strength: (semanticWeight * fraction + lexicalWeight * share) + / (semanticWeight + lexicalWeight) }); } entries.sort((a, b) => b.strength - a.strength); @@ -473,7 +480,7 @@ Zotero.BestMatch = new function () { // Quoting is the expensive half -- a passage the query's words // aren't in has to be read by the model -- so it happens only for // the passages that survived - await this._pickSnippets(entries); + await this._pickSnippets(entries, itemID); return entries; } @@ -557,26 +564,35 @@ Zotero.BestMatch = new function () { * * Where a passage says the query outright, that's the window covering * the most of it. Where it only means it -- the model matched what no - * word of the query says -- the passage is cut into lines and the + * word of the query says -- the passage is cut into sentences and the * model picks the one it finds nearest, which is the only thing that - * knows where the resemblance lives. + * knows where the resemblance lives. What's quoted from there is + * whole sentences (see _quoteFrom()). * * The passages needing the model are asked about together, in one * call: embedding costs far more per call than per text, so asking - * once for an item's lines is several times cheaper than asking per - * passage. Every passage gets its opening first, so a model that + * once for an item's sentences is several times cheaper than asking + * per passage. Every passage gets its opening first, so a model that * can't answer leaves a usable quote rather than none. * + * Both halves answer to the same gates the passages did: a session + * pinned to one engine quotes the way that engine would, rather than + * ranking with it and then quoting with the other. + * * @param {Object[]} entries - Set in place + * @param {Number} itemID */ - async _pickSnippets(entries) { + async _pickSnippets(entries, itemID) { let chunking = Zotero.Utilities.Internal.Chunking; + let useModel = this._semanticApplies(itemID); let pending = []; for (let entry of entries) { - entry.snippet = { - start: 0, - end: Math.min(entry.text.length, SNIPPET_CHARS) - }; + let sentences = chunking.splitSentences( + entry.text, chunking.getCharacterMetrics(entry.text)); + // The passage's opening, for a passage nothing chooses within + entry.snippet = sentences.length + ? _quoteFrom(sentences, 0) + : { start: 0, end: Math.min(entry.text.length, SNIPPET_CHARS) }; if (entry.ranges.length) { let window = await Zotero.Lexical.pickSnippetWindow( this._queryText, entry.text, { width: SNIPPET_CHARS }); @@ -585,19 +601,11 @@ Zotero.BestMatch = new function () { continue; } } - if (!this._modelApplies()) { + // A passage that is a single sentence has nothing to choose + if (!useModel || sentences.length < 2) { continue; } - let lines = chunking.chunkText(entry.text, { - ...chunking.getCharacterMetrics(entry.text), - budget: SNIPPET_CHARS, - minSize: Math.floor(SNIPPET_CHARS / 4), - overlap: 0 - }); - // A passage that is already one line has nothing to choose - if (lines.length > 1) { - pending.push({ entry, lines }); - } + pending.push({ entry, sentences }); } if (!pending.length) { return; @@ -606,7 +614,7 @@ Zotero.BestMatch = new function () { try { scores = await Zotero.Embeddings.scoreTexts( this._queryText, - pending.flatMap(({ lines }) => lines.map(line => line.text)) + pending.flatMap(({ sentences }) => sentences.map(s => s.text)) ); } catch (e) { @@ -614,16 +622,16 @@ Zotero.BestMatch = new function () { return; } let offset = 0; - for (let { entry, lines } of pending) { - let mine = scores.slice(offset, offset + lines.length); - offset += lines.length; - let best = mine.indexOf(Math.max(...mine)); - entry.snippet = { start: lines[best].start, end: lines[best].end }; + for (let { entry, sentences } of pending) { + let mine = scores.slice(offset, offset + sentences.length); + offset += sentences.length; + entry.snippet = _quoteFrom(sentences, mine.indexOf(Math.max(...mine))); } } // Whether this session's query reaches each engine for the item being - // derived. The bestMatchEngine pref is temporary, for testing. + // derived: the engine has to be one the session listens to, and to + // have found something in the item worth speaking about. _semanticApplies(itemID) { return this._previews.get(itemID)?.semantic !== false && this._modelApplies(); @@ -631,17 +639,23 @@ Zotero.BestMatch = new function () { _lexicalApplies(itemID) { return this._previews.get(itemID)?.lexical !== false - && Zotero.Prefs.get('search.bestMatchEngine') != 'semantic'; + && this._lexicalEnabled(); } - // Whether the model can be asked about this query at all, apart from - // what it made of any one item + // Whether each engine reaches this session at all, apart from what it + // made of any one item. The bestMatchEngine pref is temporary, for + // testing: it pins a session to a single engine, which then decides + // not only what matched but how a match is quoted. _modelApplies() { return Zotero.Prefs.get('search.bestMatchEngine') != 'lexical' && _useSemantic() && !!Zotero.Embeddings.normalizeQuery(this._queryText || ''); } + _lexicalEnabled() { + return Zotero.Prefs.get('search.bestMatchEngine') != 'semantic'; + } + // Derive one item's entries, all at once. A preview replaced while // deriving (a re-score, an invalidate) keeps the newer object // untouched. @@ -665,6 +679,24 @@ Zotero.BestMatch = new function () { return new this.Session(queryText); }; + // The extent to quote starting at one of a passage's sentences: that + // sentence, plus the ones after it that still fit SNIPPET_CHARS. + // + // The chosen sentence is taken whole however long it is -- half a sentence + // reads as a truncation rather than as a passage, and the row clips what + // doesn't fit anyway. A short one alone reads as a fragment, so the rest + // of the line goes to what follows it. + function _quoteFrom(sentences, index) { + let { start, end } = sentences[index]; + for (let i = index + 1; i < sentences.length; i++) { + if (sentences[i].end - start > SNIPPET_CHARS) { + break; + } + end = sentences[i].end; + } + return { start, end }; + } + // Fuse the two engines' scores with strength-weighted Reciprocal Rank // Fusion: an item's fused score sums fraction / (RRF_K + rank) over the // engines that matched it, where fraction is that engine's own 0-1 diff --git a/chrome/content/zotero/xpcom/utilities_internal.js b/chrome/content/zotero/xpcom/utilities_internal.js index f505a0a2cb..ddf03f12a5 100644 --- a/chrome/content/zotero/xpcom/utilities_internal.js +++ b/chrome/content/zotero/xpcom/utilities_internal.js @@ -3472,12 +3472,7 @@ Zotero.Utilities.Internal.Chunking = new function () { // real boundaries too. function _splitToSentences(source, start, end, budget, count) { let units = []; - let segmenter = new Intl.Segmenter(undefined, { granularity: 'sentence' }); - for (let { segment, index } of segmenter.segment(source.slice(start, end))) { - let unit = _measureRange(source, start + index, start + index + segment.length, count); - if (!unit) { - continue; - } + for (let unit of _segmentSentences(source, start, end, count)) { if (unit.size <= budget) { units.push(unit); continue; @@ -3487,6 +3482,35 @@ Zotero.Utilities.Internal.Chunking = new function () { return units; } + // The sentences of a range as they stand, whatever their size + function _segmentSentences(source, start, end, count) { + let units = []; + let segmenter = new Intl.Segmenter(undefined, { granularity: 'sentence' }); + for (let { segment, index } of segmenter.segment(source.slice(start, end))) { + let unit = _measureRange(source, start + index, start + index + segment.length, count); + if (unit) { + units.push(unit); + } + } + return units; + } + + /** + * The sentences of a text, in order, each located within it. + * + * No budget: a sentence comes back whole however long it is, which is + * what a caller quoting one wants -- half a sentence reads as a + * truncation rather than as a passage. + * + * @param {String} text + * @param {Object} metrics - See getCharacterMetrics(); only `count` is read + * @return {Object[]} - { text, size, start, end } per sentence, trimmed, + * with whitespace-only segments dropped + */ + this.splitSentences = function (text, metrics) { + return _segmentSentences(text, 0, text.length, metrics.count); + }; + // Split an oversized block into as few and as even pieces as possible. // Filling each to the budget instead would leave a short remainder at the // end -- the same thing minSize exists to prevent. Each piece is the diff --git a/chrome/locale/en-US/zotero/zotero.ftl b/chrome/locale/en-US/zotero/zotero.ftl index 354fc28c43..5457de94e7 100644 --- a/chrome/locale/en-US/zotero/zotero.ftl +++ b/chrome/locale/en-US/zotero/zotero.ftl @@ -461,6 +461,8 @@ items-column-relevance-rank = Rank { $rank } items-best-match-indexing = Indexing in progress — { $indexed } of { $total } items indexed items-best-match-indexing-paused = Indexing is paused — { $indexed } of { $total } items indexed items-search-match-pending = Loading matches… +# $page (String) - a page label, e.g. "12" or "ix" +items-search-match-page = p. { $page } report-error = .label = Report Error… diff --git a/scss/components/_item-tree.scss b/scss/components/_item-tree.scss index dac83e07e7..77566d7199 100644 --- a/scss/components/_item-tree.scss +++ b/scss/components/_item-tree.scss @@ -375,18 +375,34 @@ } .search-match-row { + // One cell across the row: a match shows a passage, which the + // columns say nothing about .cell { font-size: $font-size-small; - &.title { - flex-grow: 1; - flex-basis: 0; - max-width: fit-content; - } - // Push the relevance bar to the row's end, where the Relevance - // column sits while a best-match search is active - &.relevance { - margin-inline-start: auto; - } + flex: 1 1 auto; + } + // Location above quote, beside the indent and twisty the tree adds + // to the first cell. The row's height is set from JS for exactly + // these two lines (ItemTree#_getSearchMatchRowHeight()), so + // neither may wrap. + .search-match-lines { + display: flex; + flex-direction: column; + justify-content: center; + flex: 1 1 auto; + // Without this a flex item refuses to shrink below its text, + // and nothing is ever clipped to an ellipsis + min-width: 0; + } + .search-match-location, + .cell-text { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .search-match-location { + color: var(--fill-secondary); } .search-match-pending { color: var(--fill-secondary); diff --git a/test/tests/bestMatchTest.js b/test/tests/bestMatchTest.js index 0bbe8751df..053323b97a 100644 --- a/test/tests/bestMatchTest.js +++ b/test/tests/bestMatchTest.js @@ -193,6 +193,16 @@ describe("Zotero.BestMatch", function () { // A session that has scored the attachment, recording which engines // matched it -- what getMatchingExcerpts() consults instead of being // told per call + // Pin the temporary bestMatchEngine pref, leaving every other pref + // reading through to the profile + function pinEngine(engine) { + let get = Zotero.Prefs.get; + stubs.push(sinon.stub(Zotero.Prefs, 'get').callsFake( + (key, ...rest) => (key == 'search.bestMatchEngine' + ? engine + : get.call(Zotero.Prefs, key, ...rest)))); + } + async function sessionFor({ lexical = true, semantic = true } = {}) { stubs.push(sinon.stub(Zotero.BestMatch, 'scoreItemIDs').resolves({ scores: new Map([[attachment.id, 0.9]]), @@ -314,6 +324,90 @@ describe("Zotero.BestMatch", function () { assert.isUndefined(excerpts[0].score); }); + it("should quote the way a pinned semantic engine would", async function () { + stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); + stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction').callsFake(score => score)); + let head = 'A first paragraph that never mentions the bird at all. '.repeat(5); + let tail = 'A second paragraph where the owl is finally named outright. '.repeat(5); + stubs.push(sinon.stub(Zotero.Embeddings, 'getMatchingChunks').resolves([ + { text: head + '\n\n' + tail, score: 0.6 } + ])); + let rangesStub = sinon.stub(Zotero.Lexical, 'findMatchRanges'); + stubs.push(rangesStub); + let windowStub = sinon.stub(Zotero.Lexical, 'pickSnippetWindow'); + stubs.push(windowStub); + // The model reads the lines and prefers the last + stubs.push(sinon.stub(Zotero.Embeddings, 'scoreTexts') + .callsFake(async (query, texts) => texts.map((text, i) => i / texts.length))); + + pinEngine('semantic'); + // The item matched lexically too, so only the pin can be keeping + // the lexical engine out of the quote + let session = await sessionFor(); + let [excerpt] = await session.getMatchingExcerpts(attachment.id); + + assert.isFalse(rangesStub.called); + assert.isFalse(windowStub.called); + assert.isEmpty(excerpt.ranges); + // The model's own choice of line, not the one saying 'owl' + assert.isAbove(excerpt.snippet.start, 0); + // Nothing but the model weighed it, so its strength is the + // model's fraction rather than a share of a blend + assert.closeTo(excerpt.strength, 0.6, 1e-9); + }); + + it("should fill out a short chosen sentence with what follows it", async function () { + stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); + stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction').callsFake(score => score)); + let first = 'The owl is here.'; + let second = 'A modest follow-up sentence that adds a little context.'; + let third = 'A third sentence long enough that adding it would overrun the ' + + 'budget for a quoted line, so it has to be left out of one that ' + + 'already holds two sentences before it, whatever else is true.'; + stubs.push(sinon.stub(Zotero.Embeddings, 'getMatchingChunks').resolves([ + { text: [first, second, third].join(' '), score: 0.6 } + ])); + // The model likes the first sentence best, and it is far too + // short to stand as a quote on its own + stubs.push(sinon.stub(Zotero.Embeddings, 'scoreTexts') + .callsFake(async (query, texts) => texts.map((text, i) => 1 - i))); + + pinEngine('semantic'); + let session = await sessionFor(); + let [excerpt] = await session.getMatchingExcerpts(attachment.id); + let quoted = excerpt.text.slice(excerpt.snippet.start, excerpt.snippet.end); + + assert.equal(excerpt.snippet.start, 0); + assert.include(quoted, first); + // The next sentence fits alongside it... + assert.include(quoted, second); + // ...and the one after that doesn't + assert.notInclude(quoted, third); + }); + + it("should not ask the model to quote an item it never ranked", async function () { + stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); + stubs.push(sinon.stub(Zotero.Embeddings, 'getChunks').resolves([ + { text: 'A passage of owlish things. '.repeat(20), chunkIndex: 0 } + ])); + // The lexical engine scored the passage on a term it then can't + // point at -- the one way a passage arrives with no ranges to + // quote around + stubs.push(sinon.stub(Zotero.Lexical, 'scoreTexts').resolves([0.8])); + stubs.push(sinon.stub(Zotero.Lexical, 'findMatchRanges').resolves([[]])); + let scoreTextsStub = sinon.stub(Zotero.Embeddings, 'scoreTexts'); + stubs.push(scoreTextsStub); + + // Hybrid, but scoring recorded no semantic match for this item + let session = await sessionFor({ semantic: false }); + let [excerpt] = await session.getMatchingExcerpts(attachment.id); + + assert.isFalse(scoreTextsStub.called); + // Left with the passage's opening, and its lexical share whole + assert.equal(excerpt.snippet.start, 0); + assert.closeTo(excerpt.strength, 0.8, 1e-9); + }); + it("should read an indexed item's chunks when the model shows nothing", async function () { stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); stubs.push(sinon.stub(Zotero.Embeddings, 'getChunks').resolves([