From 6ab8ba61b673515ad323103ea0b0747803ddf1a1 Mon Sep 17 00:00:00 2001 From: Bogdan Abaev Date: Sat, 12 Sep 2026 15:27:57 -0700 Subject: [PATCH] cut result tail against the strongest match Every engine returned hundreds of items barely above its floor for a query with a few strong answers. Items now stay only within a margin of the engine's top score (semantic: display-band fraction; lexical: share), applied per engine before fusion and in single-engine modes. A lone result is never cut, and match sets follow. Margin is search.bestMatchMargin, in percent, exposed in the Advanced pane. Default 50%. The setting trades list length for precision, and how much it bites depends on the library. Where everything is on one subject, the best match sits close above its neighbours, so a lower value is what separates a specific answer from the rest of the field, at the cost of cutting broad questions short. Where subjects are many, the tail falls away on its own and the value matters little. --- .../preferences/preferences_advanced.xhtml | 7 + chrome/content/zotero/xpcom/bestMatch.js | 64 ++++++-- chrome/locale/en-US/zotero/preferences.ftl | 1 + defaults/preferences/zotero.js | 4 + test/tests/bestMatchTest.js | 137 +++++++++++++++++- 5 files changed, 194 insertions(+), 19 deletions(-) diff --git a/chrome/content/zotero/preferences/preferences_advanced.xhtml b/chrome/content/zotero/preferences/preferences_advanced.xhtml index 46104c9c1f..d76a66a8be 100644 --- a/chrome/content/zotero/preferences/preferences_advanced.xhtml +++ b/chrome/content/zotero/preferences/preferences_advanced.xhtml @@ -328,6 +328,13 @@ + + [ - itemID, - Zotero.Embeddings.getScoreFraction(score, { clamped: false }) - ])); + let fractions = new Map([...kept].map( + ([itemID, score]) => [itemID, _semanticFraction(score)] + )); let scale = Math.max(1, ...fractions.values()); return { scores: new Map([...fractions].map( ([itemID, fraction]) => [itemID, fraction / scale] )), - matches: { lexical: new Set(), semantic: semantic.previewableIDs } + matches: { + lexical: new Set(), + semantic: new Set([...semantic.previewableIDs].filter(id => kept.has(id))) + } }; } // A query the semantic engine can't embed ranks lexically alone if (engine == 'lexical' || !_useSemantic() || !Zotero.Embeddings.normalizeQuery(queryText || '')) { - let scores = await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options); + let scores = _nearTop( + await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options), share => share + ); return { scores, matches: { lexical: new Set(scores.keys()), semantic: new Set() } @@ -261,9 +266,10 @@ Zotero.BestMatch = new function () { + semantic.reason.message); // With no model to read the fulltext, the lexical engine // covers all of it, whatever the query's length - let scores = allSources + let scores = _nearTop(allSources ? lexical.value - : await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options); + : await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options), + share => share); return { scores, matches: { @@ -274,11 +280,17 @@ Zotero.BestMatch = new function () { } throw semantic.reason; } + // Each engine's tail is cut against its own strongest match, so + // an item enters fusion only with the evidence that stood up + let lexicalScores = _nearTop(lexical.value, share => share); + let semanticScores = _nearTop(semantic.value.scores, _semanticFraction); return { - scores: _fuse(lexical.value, semantic.value.scores), + scores: _fuse(lexicalScores, semanticScores), matches: { - lexical: new Set(lexical.value.keys()), - semantic: semantic.value.previewableIDs + lexical: new Set(lexicalScores.keys()), + semantic: new Set( + [...semantic.value.previewableIDs].filter(id => semanticScores.has(id)) + ) } }; } @@ -849,6 +861,31 @@ Zotero.BestMatch = new function () { return { start, end }; } + // The semantic engine's raw score as an unclamped fraction of the display + // band (see Zotero.Embeddings.getScoreFraction()): the strength fusion + // and the margin read + function _semanticFraction(score) { + return Zotero.Embeddings.getScoreFraction(score, { clamped: false }); + } + + // An engine's results that stand within the margin of its strongest: the + // items whose fraction is at least (1 - margin) of the top fraction. + // Every engine returns a tail of items barely above its floor -- for a + // query with a few strong answers, hundreds of them -- that aren't + // matches for that query in any sense a reader would accept. Measuring + // the cut from the top rather than by count lets a broad query keep + // hundreds of comparable results while a specific one keeps a handful. + // A lone result is never cut. The margin is a pref, in percent. + function _nearTop(scores, toFraction) { + let margin = Zotero.Prefs.get('search.bestMatchMargin') / 100; + if (scores.size < 2 || !(margin < 1)) { + return scores; + } + let fractions = new Map([...scores].map(([itemID, score]) => [itemID, toFraction(score)])); + let cutoff = Math.max(...fractions.values()) * (1 - margin); + return new Map([...scores].filter(([itemID]) => fractions.get(itemID) >= cutoff)); + } + // Fuse the two engines' scores with strength-weighted Reciprocal Rank // Fusion: an item's fused score sums fraction / (RRF_K + rank) over the // engines that matched it, where fraction is that engine's own 0-1 @@ -876,10 +913,7 @@ Zotero.BestMatch = new function () { function _fuse(lexicalScores, semanticScores) { let engines = [ [lexicalScores, score => Math.min(1, Math.max(0, score))], - [ - semanticScores, - score => Zotero.Embeddings.getScoreFraction(score, { clamped: false }) - ] + [semanticScores, _semanticFraction] ]; let scores = new Map(); for (let [engineScores, toFraction] of engines) { diff --git a/chrome/locale/en-US/zotero/preferences.ftl b/chrome/locale/en-US/zotero/preferences.ftl index b3469a25a6..1f33377dec 100644 --- a/chrome/locale/en-US/zotero/preferences.ftl +++ b/chrome/locale/en-US/zotero/preferences.ftl @@ -112,6 +112,7 @@ preferences-advanced-best-match-engine-lexical = .label = Lexical preferences-advanced-best-match-engine-semantic = .label = Semantic +preferences-advanced-best-match-margin = Drop results scoring more than this far below the best result (%): preferences-advanced-semantic-search-downloading = Downloading… preferences-advanced-semantic-search-downloading-progress = Downloading… { $percent }% preferences-advanced-semantic-search-extracting = Preparing documents… diff --git a/defaults/preferences/zotero.js b/defaults/preferences/zotero.js index 85114c50d3..e3688b1529 100644 --- a/defaults/preferences/zotero.js +++ b/defaults/preferences/zotero.js @@ -111,6 +111,10 @@ pref("extensions.zotero.search.quicksearch-mode", "fields"); // Temporary, for testing: which engine best-match search runs -- 'lexical', // 'semantic', or 'hybrid' (both, fused) pref("extensions.zotero.search.bestMatchEngine", "hybrid"); +// How far below an engine's strongest match an item may fall and still count +// as one of its results, as a percentage of that strongest score -- 10 keeps +// items within 10 percent of the top, 100 keeps everything above the floor +pref("extensions.zotero.search.bestMatchMargin", 50); // Fulltext indexing pref("extensions.zotero.fulltext.textMaxLength", 500000); diff --git a/test/tests/bestMatchTest.js b/test/tests/bestMatchTest.js index 6814789bb3..b7f64d0f66 100644 --- a/test/tests/bestMatchTest.js +++ b/test/tests/bestMatchTest.js @@ -43,7 +43,7 @@ describe("Zotero.BestMatch", function () { describe("#scoreItemIDs()", function () { it("should return lexical scores directly when no semantic model is enabled", async function () { - let lexicalScores = new Map([[1, 0.8], [2, 0.3]]); + let lexicalScores = new Map([[1, 0.8], [2, 0.5]]); let semanticStub = sinon.stub(Zotero.Embeddings, 'scoreItemIDs'); stubs.push(semanticStub); stubEngines({ @@ -53,7 +53,7 @@ describe("Zotero.BestMatch", function () { let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]); assert.isFalse(semanticStub.called); - assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]); + assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]); }); it("should fuse the engines' rankings reciprocally over their union", async function () { @@ -210,7 +210,7 @@ describe("Zotero.BestMatch", function () { }); it("should rank lexically when the semantic index isn't ready", async function () { - let lexicalScores = new Map([[1, 0.8], [2, 0.3]]); + let lexicalScores = new Map([[1, 0.8], [2, 0.5]]); stubEngines({ enabled: true, lexical: async () => lexicalScores, @@ -220,7 +220,7 @@ describe("Zotero.BestMatch", function () { }); let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]); - assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]); + assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]); // The engine that didn't rank shows matches in nothing assert.equal(matches.semantic.size, 0); assert.sameMembers([...matches.lexical], [1, 2]); @@ -279,6 +279,135 @@ describe("Zotero.BestMatch", function () { let e = await getPromiseError(Zotero.BestMatch.scoreItemIDs('owl', [1])); assert.equal(e.message, 'model exploded'); }); + + describe("margin", function () { + let margin = (value) => { + Zotero.Prefs.set('search.bestMatchMargin', value); + }; + + afterEach(function () { + Zotero.Prefs.clear('search.bestMatchMargin'); + }); + + it("should cut an engine's tail against its own strongest match", async function () { + margin(50); + stubEngines({ + enabled: true, + lexical: async () => new Map(), + // Item 3 sits at a tenth of the top; item 2 at half of it stays + semantic: async () => new Map([[1, 0.9], [2, 0.5], [3, 0.09]]) + }); + + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]); + assert.sameMembers([...scores.keys()], [1, 2]); + }); + + it("should keep items tied at the boundary together", async function () { + margin(50); + stubEngines({ + enabled: true, + lexical: async () => new Map([[1, 1], [2, 0.5], [3, 0.5]]), + semantic: async () => new Map() + }); + + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]); + assert.sameMembers([...scores.keys()], [1, 2, 3]); + }); + + it("should never cut a lone result", async function () { + margin(10); + stubEngines({ + enabled: true, + lexical: async () => new Map([[1, 0.06]]), + semantic: async () => new Map([[2, 0.01]]) + }); + + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]); + assert.sameMembers([...scores.keys()], [1, 2]); + }); + + it("should keep everything above the floor at a margin of 1", async function () { + margin(100); + stubEngines({ + enabled: true, + lexical: async () => new Map([[1, 0.9], [2, 0.05]]), + semantic: async () => new Map([[3, 0.9], [4, 0.001]]) + }); + + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3, 4]); + assert.sameMembers([...scores.keys()], [1, 2, 3, 4]); + }); + + it("should measure the semantic cut on the display band, not the raw score", async function () { + margin(50); + stubEngines({ + enabled: true, + lexical: async () => new Map(), + // Raw scores 0.6 and 0.4 look close; on a band starting at + // 0.35 they are fractions 1.0 and 0.2, and the second is cut + semantic: async () => new Map([[1, 0.6], [2, 0.4]]), + fraction: score => Math.max(0, (score - 0.35) / 0.25) + }); + + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]); + assert.sameMembers([...scores.keys()], [1]); + }); + + it("should take an item's semantic voice and match away when only lexical keeps it", async function () { + margin(50); + stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); + stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction') + .callsFake(score => score)); + stubs.push(sinon.stub(Zotero.Lexical, 'getScoringTermCount').resolves(1)); + stubs.push(sinon.stub(Zotero.Lexical, 'isFullyQuotedQuery').returns(false)); + stubs.push(sinon.stub(Zotero.Lexical, 'scoreItemIDs') + .resolves(new Map([[1, 0.8], [2, 0.9]]))); + stubs.push(sinon.stub(Zotero.Embeddings, 'scoreItemIDs').resolves({ + scores: new Map([[1, 0.9], [2, 0.05]]), + previewableIDs: new Set([1, 2]) + })); + + let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]); + // Item 2 stays, on its lexical evidence alone + assert.closeTo(scores.get(2), rrf([0.9, 1]), 1e-12); + assert.closeTo(scores.get(1), rrf([0.8, 2], [0.9, 1]), 1e-12); + assert.sameMembers([...matches.lexical], [1, 2]); + assert.sameMembers([...matches.semantic], [1]); + }); + + it("should cut the tail in single-engine modes too", async function () { + margin(50); + let saved = Zotero.Prefs.get('search.bestMatchEngine'); + try { + Zotero.Prefs.set('search.bestMatchEngine', 'semantic'); + stubEngines({ + enabled: true, + semantic: async () => new Map([[1, 1.8], [2, 1.2], [3, 0.3]]), + fraction: (score, options) => (options && options.clamped === false + ? score + : Math.min(1, score)) + }); + let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]); + assert.sameMembers([...scores.keys()], [1, 2]); + // Rescaled by the strongest kept fraction + assert.equal(scores.get(1), 1); + assert.closeTo(scores.get(2), 1.2 / 1.8, 1e-12); + + Zotero.Prefs.set('search.bestMatchEngine', 'lexical'); + stubs.forEach(stub => stub.restore()); + stubs = []; + stubEngines({ + enabled: true, + lexical: async () => new Map([[1, 0.9], [2, 0.3]]) + }); + ({ scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2])); + assert.sameMembers([...scores.keys()], [1]); + } + finally { + Zotero.Prefs.set('search.bestMatchEngine', saved); + } + }); + }); }); describe("Session#getMatchingExcerpts()", function () {