diff --git a/chrome/content/zotero/preferences/preferences_advanced.xhtml b/chrome/content/zotero/preferences/preferences_advanced.xhtml
index 46104c9c1f..d76a66a8be 100644
--- a/chrome/content/zotero/preferences/preferences_advanced.xhtml
+++ b/chrome/content/zotero/preferences/preferences_advanced.xhtml
@@ -328,6 +328,13 @@
+
+
+
+
[
- itemID,
- Zotero.Embeddings.getScoreFraction(score, { clamped: false })
- ]));
+ let fractions = new Map([...kept].map(
+ ([itemID, score]) => [itemID, _semanticFraction(score)]
+ ));
let scale = Math.max(1, ...fractions.values());
return {
scores: new Map([...fractions].map(
([itemID, fraction]) => [itemID, fraction / scale]
)),
- matches: { lexical: new Set(), semantic: semantic.previewableIDs }
+ matches: {
+ lexical: new Set(),
+ semantic: new Set([...semantic.previewableIDs].filter(id => kept.has(id)))
+ }
};
}
// A query the semantic engine can't embed ranks lexically alone
if (engine == 'lexical' || !_useSemantic()
|| !Zotero.Embeddings.normalizeQuery(queryText || '')) {
- let scores = await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options);
+ let scores = _nearTop(
+ await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options), share => share
+ );
return {
scores,
matches: { lexical: new Set(scores.keys()), semantic: new Set() }
@@ -261,9 +266,10 @@ Zotero.BestMatch = new function () {
+ semantic.reason.message);
// With no model to read the fulltext, the lexical engine
// covers all of it, whatever the query's length
- let scores = allSources
+ let scores = _nearTop(allSources
? lexical.value
- : await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options);
+ : await Zotero.Lexical.scoreItemIDs(queryText, itemIDs, options),
+ share => share);
return {
scores,
matches: {
@@ -274,11 +280,17 @@ Zotero.BestMatch = new function () {
}
throw semantic.reason;
}
+ // Each engine's tail is cut against its own strongest match, so
+ // an item enters fusion only with the evidence that stood up
+ let lexicalScores = _nearTop(lexical.value, share => share);
+ let semanticScores = _nearTop(semantic.value.scores, _semanticFraction);
return {
- scores: _fuse(lexical.value, semantic.value.scores),
+ scores: _fuse(lexicalScores, semanticScores),
matches: {
- lexical: new Set(lexical.value.keys()),
- semantic: semantic.value.previewableIDs
+ lexical: new Set(lexicalScores.keys()),
+ semantic: new Set(
+ [...semantic.value.previewableIDs].filter(id => semanticScores.has(id))
+ )
}
};
}
@@ -849,6 +861,31 @@ Zotero.BestMatch = new function () {
return { start, end };
}
+ // The semantic engine's raw score as an unclamped fraction of the display
+ // band (see Zotero.Embeddings.getScoreFraction()): the strength fusion
+ // and the margin read
+ function _semanticFraction(score) {
+ return Zotero.Embeddings.getScoreFraction(score, { clamped: false });
+ }
+
+ // An engine's results that stand within the margin of its strongest: the
+ // items whose fraction is at least (1 - margin) of the top fraction.
+ // Every engine returns a tail of items barely above its floor -- for a
+ // query with a few strong answers, hundreds of them -- that aren't
+ // matches for that query in any sense a reader would accept. Measuring
+ // the cut from the top rather than by count lets a broad query keep
+ // hundreds of comparable results while a specific one keeps a handful.
+ // A lone result is never cut. The margin is a pref, in percent.
+ function _nearTop(scores, toFraction) {
+ let margin = Zotero.Prefs.get('search.bestMatchMargin') / 100;
+ if (scores.size < 2 || !(margin < 1)) {
+ return scores;
+ }
+ let fractions = new Map([...scores].map(([itemID, score]) => [itemID, toFraction(score)]));
+ let cutoff = Math.max(...fractions.values()) * (1 - margin);
+ return new Map([...scores].filter(([itemID]) => fractions.get(itemID) >= cutoff));
+ }
+
// Fuse the two engines' scores with strength-weighted Reciprocal Rank
// Fusion: an item's fused score sums fraction / (RRF_K + rank) over the
// engines that matched it, where fraction is that engine's own 0-1
@@ -876,10 +913,7 @@ Zotero.BestMatch = new function () {
function _fuse(lexicalScores, semanticScores) {
let engines = [
[lexicalScores, score => Math.min(1, Math.max(0, score))],
- [
- semanticScores,
- score => Zotero.Embeddings.getScoreFraction(score, { clamped: false })
- ]
+ [semanticScores, _semanticFraction]
];
let scores = new Map();
for (let [engineScores, toFraction] of engines) {
diff --git a/chrome/locale/en-US/zotero/preferences.ftl b/chrome/locale/en-US/zotero/preferences.ftl
index b3469a25a6..1f33377dec 100644
--- a/chrome/locale/en-US/zotero/preferences.ftl
+++ b/chrome/locale/en-US/zotero/preferences.ftl
@@ -112,6 +112,7 @@ preferences-advanced-best-match-engine-lexical =
.label = Lexical
preferences-advanced-best-match-engine-semantic =
.label = Semantic
+preferences-advanced-best-match-margin = Drop results scoring more than this far below the best result (%):
preferences-advanced-semantic-search-downloading = Downloading…
preferences-advanced-semantic-search-downloading-progress = Downloading… { $percent }%
preferences-advanced-semantic-search-extracting = Preparing documents…
diff --git a/defaults/preferences/zotero.js b/defaults/preferences/zotero.js
index 85114c50d3..e3688b1529 100644
--- a/defaults/preferences/zotero.js
+++ b/defaults/preferences/zotero.js
@@ -111,6 +111,10 @@ pref("extensions.zotero.search.quicksearch-mode", "fields");
// Temporary, for testing: which engine best-match search runs -- 'lexical',
// 'semantic', or 'hybrid' (both, fused)
pref("extensions.zotero.search.bestMatchEngine", "hybrid");
+// How far below an engine's strongest match an item may fall and still count
+// as one of its results, as a percentage of that strongest score -- 10 keeps
+// items within 10 percent of the top, 100 keeps everything above the floor
+pref("extensions.zotero.search.bestMatchMargin", 50);
// Fulltext indexing
pref("extensions.zotero.fulltext.textMaxLength", 500000);
diff --git a/test/tests/bestMatchTest.js b/test/tests/bestMatchTest.js
index 6814789bb3..b7f64d0f66 100644
--- a/test/tests/bestMatchTest.js
+++ b/test/tests/bestMatchTest.js
@@ -43,7 +43,7 @@ describe("Zotero.BestMatch", function () {
describe("#scoreItemIDs()", function () {
it("should return lexical scores directly when no semantic model is enabled", async function () {
- let lexicalScores = new Map([[1, 0.8], [2, 0.3]]);
+ let lexicalScores = new Map([[1, 0.8], [2, 0.5]]);
let semanticStub = sinon.stub(Zotero.Embeddings, 'scoreItemIDs');
stubs.push(semanticStub);
stubEngines({
@@ -53,7 +53,7 @@ describe("Zotero.BestMatch", function () {
let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
assert.isFalse(semanticStub.called);
- assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]);
+ assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]);
});
it("should fuse the engines' rankings reciprocally over their union", async function () {
@@ -210,7 +210,7 @@ describe("Zotero.BestMatch", function () {
});
it("should rank lexically when the semantic index isn't ready", async function () {
- let lexicalScores = new Map([[1, 0.8], [2, 0.3]]);
+ let lexicalScores = new Map([[1, 0.8], [2, 0.5]]);
stubEngines({
enabled: true,
lexical: async () => lexicalScores,
@@ -220,7 +220,7 @@ describe("Zotero.BestMatch", function () {
});
let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
- assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.3]]);
+ assert.deepEqual([...scores.entries()], [[1, 0.8], [2, 0.5]]);
// The engine that didn't rank shows matches in nothing
assert.equal(matches.semantic.size, 0);
assert.sameMembers([...matches.lexical], [1, 2]);
@@ -279,6 +279,135 @@ describe("Zotero.BestMatch", function () {
let e = await getPromiseError(Zotero.BestMatch.scoreItemIDs('owl', [1]));
assert.equal(e.message, 'model exploded');
});
+
+ describe("margin", function () {
+ let margin = (value) => {
+ Zotero.Prefs.set('search.bestMatchMargin', value);
+ };
+
+ afterEach(function () {
+ Zotero.Prefs.clear('search.bestMatchMargin');
+ });
+
+ it("should cut an engine's tail against its own strongest match", async function () {
+ margin(50);
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map(),
+ // Item 3 sits at a tenth of the top; item 2 at half of it stays
+ semantic: async () => new Map([[1, 0.9], [2, 0.5], [3, 0.09]])
+ });
+
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+ assert.sameMembers([...scores.keys()], [1, 2]);
+ });
+
+ it("should keep items tied at the boundary together", async function () {
+ margin(50);
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map([[1, 1], [2, 0.5], [3, 0.5]]),
+ semantic: async () => new Map()
+ });
+
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+ assert.sameMembers([...scores.keys()], [1, 2, 3]);
+ });
+
+ it("should never cut a lone result", async function () {
+ margin(10);
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map([[1, 0.06]]),
+ semantic: async () => new Map([[2, 0.01]])
+ });
+
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+ assert.sameMembers([...scores.keys()], [1, 2]);
+ });
+
+ it("should keep everything above the floor at a margin of 1", async function () {
+ margin(100);
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map([[1, 0.9], [2, 0.05]]),
+ semantic: async () => new Map([[3, 0.9], [4, 0.001]])
+ });
+
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3, 4]);
+ assert.sameMembers([...scores.keys()], [1, 2, 3, 4]);
+ });
+
+ it("should measure the semantic cut on the display band, not the raw score", async function () {
+ margin(50);
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map(),
+ // Raw scores 0.6 and 0.4 look close; on a band starting at
+ // 0.35 they are fractions 1.0 and 0.2, and the second is cut
+ semantic: async () => new Map([[1, 0.6], [2, 0.4]]),
+ fraction: score => Math.max(0, (score - 0.35) / 0.25)
+ });
+
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+ assert.sameMembers([...scores.keys()], [1]);
+ });
+
+ it("should take an item's semantic voice and match away when only lexical keeps it", async function () {
+ margin(50);
+ stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true));
+ stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction')
+ .callsFake(score => score));
+ stubs.push(sinon.stub(Zotero.Lexical, 'getScoringTermCount').resolves(1));
+ stubs.push(sinon.stub(Zotero.Lexical, 'isFullyQuotedQuery').returns(false));
+ stubs.push(sinon.stub(Zotero.Lexical, 'scoreItemIDs')
+ .resolves(new Map([[1, 0.8], [2, 0.9]])));
+ stubs.push(sinon.stub(Zotero.Embeddings, 'scoreItemIDs').resolves({
+ scores: new Map([[1, 0.9], [2, 0.05]]),
+ previewableIDs: new Set([1, 2])
+ }));
+
+ let { scores, matches } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]);
+ // Item 2 stays, on its lexical evidence alone
+ assert.closeTo(scores.get(2), rrf([0.9, 1]), 1e-12);
+ assert.closeTo(scores.get(1), rrf([0.8, 2], [0.9, 1]), 1e-12);
+ assert.sameMembers([...matches.lexical], [1, 2]);
+ assert.sameMembers([...matches.semantic], [1]);
+ });
+
+ it("should cut the tail in single-engine modes too", async function () {
+ margin(50);
+ let saved = Zotero.Prefs.get('search.bestMatchEngine');
+ try {
+ Zotero.Prefs.set('search.bestMatchEngine', 'semantic');
+ stubEngines({
+ enabled: true,
+ semantic: async () => new Map([[1, 1.8], [2, 1.2], [3, 0.3]]),
+ fraction: (score, options) => (options && options.clamped === false
+ ? score
+ : Math.min(1, score))
+ });
+ let { scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2, 3]);
+ assert.sameMembers([...scores.keys()], [1, 2]);
+ // Rescaled by the strongest kept fraction
+ assert.equal(scores.get(1), 1);
+ assert.closeTo(scores.get(2), 1.2 / 1.8, 1e-12);
+
+ Zotero.Prefs.set('search.bestMatchEngine', 'lexical');
+ stubs.forEach(stub => stub.restore());
+ stubs = [];
+ stubEngines({
+ enabled: true,
+ lexical: async () => new Map([[1, 0.9], [2, 0.3]])
+ });
+ ({ scores } = await Zotero.BestMatch.scoreItemIDs('owl', [1, 2]));
+ assert.sameMembers([...scores.keys()], [1]);
+ }
+ finally {
+ Zotero.Prefs.set('search.bestMatchEngine', saved);
+ }
+ });
+ });
});
describe("Session#getMatchingExcerpts()", function () {