cut result tail against the strongest match

Every engine returned hundreds of items barely above its floor for a
query with a few strong answers. Items now stay only within a margin
of the engine's top score (semantic: display-band fraction; lexical:
share), applied per engine before fusion and in single-engine modes.
A lone result is never cut, and match sets follow.

Margin is search.bestMatchMargin, in percent, exposed in the Advanced
pane. Default 50%.

The setting trades list length for precision, and how much it bites
depends on the library. Where everything is on one subject, the best
match sits close above its neighbours, so a lower value is what
separates a specific answer from the rest of the field, at the cost
of cutting broad questions short. Where subjects are many, the tail
falls away on its own and the value matters little.

chrome/content/zotero/preferences/preferences_advanced.xhtml

@@ -328,6 +328,13 @@
 					</menupopup>
 				</menulist>
 			</hbox>
+			<hbox align="center">
+				<label data-l10n-id="preferences-advanced-best-match-margin"
+					control="best-match-margin"/>
+				<html:input id="best-match-margin" class="html-input" type="number"
+					min="1" max="100" step="1" size="3"
+					data-preference="search.bestMatchMargin"/>
+			</hbox>
 			<checkbox id="semantic-search-index-fulltext"
 				data-l10n-id="preferences-advanced-semantic-search-index-fulltext"
 				preference="extensions.zotero.embeddings.indexFulltext"

chrome/content/zotero/xpcom/bestMatch.js

@@ -211,26 +211,31 @@ Zotero.BestMatch = new function () {
 			let engine = Zotero.Prefs.get('search.bestMatchEngine');
 			if (engine == 'semantic') {
 				let semantic = await Zotero.Embeddings.scoreItemIDs(queryText, itemIDs, options);
+				let kept = _nearTop(semantic.scores, _semanticFraction);
 				// On the model's display band, so scores are 0-1 like the other
 				// modes' -- but unclamped and rescaled by the strongest score,
 				// so a top tier past the band's ceiling keeps its ordering
 				// instead of flattening into a tie
-				let fractions = new Map([...semantic.scores].map(([itemID, score]) => [
-					itemID,
-					Zotero.Embeddings.getScoreFraction(score, { clamped: false })
-				]));
+				let fractions = new Map([...kept].map(
+					([itemID, score]) => [itemID, _semanticFraction(score)]
+				));
 				let scale = Math.max(1, ...fractions.values());
 				return {
 					scores: new Map([...fractions].map(
 						([itemID, fraction]) => [itemID, fraction / scale]
 					)),
-					matches: { lexical: new Set(), semantic: semantic.previewableIDs }
+					matches: {
+						lexical: new Set(),
+						semantic: new Set([...semantic.previewableIDs].filter(id => kept.has(id)))
+					}
 				};
 			}
 			// A query the semantic engine can't embed ranks lexically alone
 			if (engine == 'lexical' || !_useSemantic()
 					|| !Zotero.Embeddings.normalizeQuery(queryText || '')) {
-				let scores = await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options);
+				let scores = _nearTop(
+					await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options), share => share
+				);
 				return {
 					scores,
 					matches: { lexical: new Set(scores.keys()), semantic: new Set() }
@@ -261,9 +266,10 @@ Zotero.BestMatch = new function () {
 						+ semantic.reason.message);
 					// With no model to read the fulltext, the lexical engine
 					// covers all of it, whatever the query's length
-					let scores = allSources
+					let scores = _nearTop(allSources
 						? lexical.value
-						: await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options);
+						: await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options),
+					share => share);
 					return {
 						scores,
 						matches: {
@@ -274,11 +280,17 @@ Zotero.BestMatch = new function () {
 				}
 				throw semantic.reason;
 			}
+			// Each engine's tail is cut against its own strongest match, so
+			// an item enters fusion only with the evidence that stood up
+			let lexicalScores = _nearTop(lexical.value, share => share);
+			let semanticScores = _nearTop(semantic.value.scores, _semanticFraction);
 			return {
-				scores: _fuse(lexical.value, semantic.value.scores),
+				scores: _fuse(lexicalScores, semanticScores),
 				matches: {
-					lexical: new Set(lexical.value.keys()),
-					semantic: semantic.value.previewableIDs
+					lexical: new Set(lexicalScores.keys()),
+					semantic: new Set(
+						[...semantic.value.previewableIDs].filter(id => semanticScores.has(id))
+					)
 				}
 			};
 		}
@@ -849,6 +861,31 @@ Zotero.BestMatch = new function () {
 		return { start, end };
 	}
 
+	// The semantic engine's raw score as an unclamped fraction of the display
+	// band (see Zotero.Embeddings.getScoreFraction()): the strength fusion
+	// and the margin read
+	function _semanticFraction(score) {
+		return Zotero.Embeddings.getScoreFraction(score, { clamped: false });
+	}
+
+	// An engine's results that stand within the margin of its strongest: the
+	// items whose fraction is at least (1 - margin) of the top fraction.
+	// Every engine returns a tail of items barely above its floor -- for a
+	// query with a few strong answers, hundreds of them -- that aren't
+	// matches for that query in any sense a reader would accept. Measuring
+	// the cut from the top rather than by count lets a broad query keep
+	// hundreds of comparable results while a specific one keeps a handful.
+	// A lone result is never cut. The margin is a pref, in percent.
+	function _nearTop(scores, toFraction) {
+		let margin = Zotero.Prefs.get('search.bestMatchMargin') / 100;
+		if (scores.size < 2 || !(margin < 1)) {
+			return scores;
+		}
+		let fractions = new Map([...scores].map(([itemID, score]) => [itemID, toFraction(score)]));
+		let cutoff = Math.max(...fractions.values()) * (1 - margin);
+		return new Map([...scores].filter(([itemID]) => fractions.get(itemID) >= cutoff));
+	}
+
 	// Fuse the two engines' scores with strength-weighted Reciprocal Rank
 	// Fusion: an item's fused score sums fraction / (RRF_K + rank) over the
 	// engines that matched it, where fraction is that engine's own 0-1
@@ -876,10 +913,7 @@ Zotero.BestMatch = new function () {
 	function _fuse(lexicalScores, semanticScores) {
 		let engines = [
 			[lexicalScores, score => Math.min(1, Math.max(0, score))],
-			[
-				semanticScores,
-				score => Zotero.Embeddings.getScoreFraction(score, { clamped: false })
-			]
+			[semanticScores, _semanticFraction]
 		];
 		let scores = new Map();
 		for (let [engineScores, toFraction] of engines) {

chrome/locale/en-US/zotero/preferences.ftl

@@ -112,6 +112,7 @@ preferences-advanced-best-match-engine-lexical =
     .label = Lexical
 preferences-advanced-best-match-engine-semantic =
     .label = Semantic
+preferences-advanced-best-match-margin = Drop results scoring more than this far below the best result (%):
 preferences-advanced-semantic-search-downloading = Downloading…
 preferences-advanced-semantic-search-downloading-progress = Downloading… { $percent }%
 preferences-advanced-semantic-search-extracting = Preparing documents…

defaults/preferences/zotero.js

@@ -111,6 +111,10 @@ pref("extensions.zotero.search.quicksearch-mode", "fields");
 // Temporary, for testing: which engine best-match search runs -- 'lexical',
 // 'semantic', or 'hybrid' (both, fused)
 pref("extensions.zotero.search.bestMatchEngine", "hybrid");
+// How far below an engine's strongest match an item may fall and still count
+// as one of its results, as a percentage of that strongest score -- 10 keeps
+// items within 10 percent of the top, 100 keeps everything above the floor
+pref("extensions.zotero.search.bestMatchMargin", 50);
 
 // Fulltext indexing
 pref("extensions.zotero.fulltext.textMaxLength", 500000);

test/tests/bestMatchTest.js

@@ -43,7 +43,7 @@ describe("Zotero.BestMatch", function () {
 
 	describe("#scoreItemIDs()", function () {
 		it("should return lexical scores directly when no semantic model is enabled", async function () {
-			let lexicalScores = new Map([[1, 0.8], [2, 0.3]]);
+			let lexicalScores = new Map([[1, 0.8], [2, 0.5]]);
 			let semanticStub = sinon.stub(Zotero.Embeddings, 'scoreItemIDs');
 			stubs.push(semanticStub);
 			stubEngines({
@@ -53,7 +53,7 @@ describe("Zotero.BestMatch", function () {
 
 			let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
 			assert.isFalse(semanticStub.called);
-			assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]);
+			assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]);
 		});
 
 		it("should fuse the engines' rankings reciprocally over their union", async function () {
@@ -210,7 +210,7 @@ describe("Zotero.BestMatch", function () {
 		});
 
 		it("should rank lexically when the semantic index isn't ready", async function () {
-			let lexicalScores = new Map([[1, 0.8], [2, 0.3]]);
+			let lexicalScores = new Map([[1, 0.8], [2, 0.5]]);
 			stubEngines({
 				enabled: true,
 				lexical: async () => lexicalScores,
@@ -220,7 +220,7 @@ describe("Zotero.BestMatch", function () {
 			});
 
 			let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
-			assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]);
+			assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]);
 			// The engine that didn't rank shows matches in nothing
 			assert.equal(matches.semantic.size, 0);
 			assert.sameMembers([...matches.lexical], [1, 2]);
@@ -279,6 +279,135 @@ describe("Zotero.BestMatch", function () {
 			let e = await getPromiseError(Zotero.BestMatch.scoreItemIDs('owl', [1]));
 			assert.equal(e.message, 'model exploded');
 		});
+
+		describe("margin", function () {
+			let margin = (value) => {
+				Zotero.Prefs.set('search.bestMatchMargin', value);
+			};
+
+			afterEach(function () {
+				Zotero.Prefs.clear('search.bestMatchMargin');
+			});
+
+			it("should cut an engine's tail against its own strongest match", async function () {
+				margin(50);
+				stubEngines({
+					enabled: true,
+					lexical: async () => new Map(),
+					// Item 3 sits at a tenth of the top; item 2 at half of it stays
+					semantic: async () => new Map([[1, 0.9], [2, 0.5], [3, 0.09]])
+				});
+
+				let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+				assert.sameMembers([...scores.keys()], [1, 2]);
+			});
+
+			it("should keep items tied at the boundary together", async function () {
+				margin(50);
+				stubEngines({
+					enabled: true,
+					lexical: async () => new Map([[1, 1], [2, 0.5], [3, 0.5]]),
+					semantic: async () => new Map()
+				});
+
+				let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+				assert.sameMembers([...scores.keys()], [1, 2, 3]);
+			});
+
+			it("should never cut a lone result", async function () {
+				margin(10);
+				stubEngines({
+					enabled: true,
+					lexical: async () => new Map([[1, 0.06]]),
+					semantic: async () => new Map([[2, 0.01]])
+				});
+
+				let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+				assert.sameMembers([...scores.keys()], [1, 2]);
+			});
+
+			it("should keep everything above the floor at a margin of 1", async function () {
+				margin(100);
+				stubEngines({
+					enabled: true,
+					lexical: async () => new Map([[1, 0.9], [2, 0.05]]),
+					semantic: async () => new Map([[3, 0.9], [4, 0.001]])
+				});
+
+				let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3, 4]);
+				assert.sameMembers([...scores.keys()], [1, 2, 3, 4]);
+			});
+
+			it("should measure the semantic cut on the display band, not the raw score", async function () {
+				margin(50);
+				stubEngines({
+					enabled: true,
+					lexical: async () => new Map(),
+					// Raw scores 0.6 and 0.4 look close; on a band starting at
+					// 0.35 they are fractions 1.0 and 0.2, and the second is cut
+					semantic: async () => new Map([[1, 0.6], [2, 0.4]]),
+					fraction: score => Math.max(0, (score - 0.35) / 0.25)
+				});
+
+				let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+				assert.sameMembers([...scores.keys()], [1]);
+			});
+
+			it("should take an item's semantic voice and match away when only lexical keeps it", async function () {
+				margin(50);
+				stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true));
+				stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction')
+					.callsFake(score => score));
+				stubs.push(sinon.stub(Zotero.Lexical, 'getScoringTermCount').resolves(1));
+				stubs.push(sinon.stub(Zotero.Lexical, 'isFullyQuotedQuery').returns(false));
+				stubs.push(sinon.stub(Zotero.Lexical, 'scoreItemIDs')
+					.resolves(new Map([[1, 0.8], [2, 0.9]])));
+				stubs.push(sinon.stub(Zotero.Embeddings, 'scoreItemIDs').resolves({
+					scores: new Map([[1, 0.9], [2, 0.05]]),
+					previewableIDs: new Set([1, 2])
+				}));
+
+				let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+				// Item 2 stays, on its lexical evidence alone
+				assert.closeTo(scores.get(2), rrf([0.9, 1]), 1e-12);
+				assert.closeTo(scores.get(1), rrf([0.8, 2], [0.9, 1]), 1e-12);
+				assert.sameMembers([...matches.lexical], [1, 2]);
+				assert.sameMembers([...matches.semantic], [1]);
+			});
+
+			it("should cut the tail in single-engine modes too", async function () {
+				margin(50);
+				let saved = Zotero.Prefs.get('search.bestMatchEngine');
+				try {
+					Zotero.Prefs.set('search.bestMatchEngine', 'semantic');
+					stubEngines({
+						enabled: true,
+						semantic: async () => new Map([[1, 1.8], [2, 1.2], [3, 0.3]]),
+						fraction: (score, options) => (options && options.clamped === false
+							? score
+							: Math.min(1, score))
+					});
+					let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+					assert.sameMembers([...scores.keys()], [1, 2]);
+					// Rescaled by the strongest kept fraction
+					assert.equal(scores.get(1), 1);
+					assert.closeTo(scores.get(2), 1.2 / 1.8, 1e-12);
+
+					Zotero.Prefs.set('search.bestMatchEngine', 'lexical');
+					stubs.forEach(stub => stub.restore());
+					stubs = [];
+					stubEngines({
+						enabled: true,
+						lexical: async () => new Map([[1, 0.9], [2, 0.3]])
+					});
+					({ scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]));
+					assert.sameMembers([...scores.keys()], [1]);
+				}
+				finally {
+					Zotero.Prefs.set('search.bestMatchEngine', saved);
+				}
+			});
+		});
 	});
 
 	describe("Session#getMatchingExcerpts()", function () {