scriptoriumadmin/zotero
1
0
Fork 0
mirror of https://github.com/zotero/zotero.git synced 2026-11-14 09:21:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

simplify ordering of resulting chunks

Browse source
This commit is contained in:
Bogdan Abaev 2026-10-30 17:12:19 -07:00
parent 2a068c6b7d
commit 088ce17829
2 changed files with 40 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
chrome/content/zotero/xpcom/bestMatch.js
View file

@ -43,8 +43,9 @@ Zotero.BestMatch = new function () {
// model's reading leads: it is the calibrated signal, and it chose which
// passages are worth showing. Saying the query's own words lifts a
// passage above an equally similar one that only paraphrases them.
const SEMANTIC_WEIGHT = 0.7;
const LEXICAL_WEIGHT = 0.3;
// Passages of an item weaker than this fraction below its strongest
// aren't shown
const PASSAGE_MARGIN = 0.5;
// About a line: what a passage is quoted down to for a one-line preview
const SNIPPET_CHARS = 150;
// A sentence longer than this is quoted as a line of its words rather
@ -726,9 +727,11 @@ Zotero.BestMatch = new function () {
/**
* Every passage explaining why an item matched this session's query,
* strongest first: the item's passages the model matched and the ones
* that say the query's words. Only the engines that matched the item
* are asked. The strongest MAX_QUOTED_PASSAGES also carry a `snippet`,
* the one line that best shows the query.
* that say the query's words, each as strong as its better
* explanation, and none far weaker than the strongest. Only the
* engines that matched the item are asked. The strongest
* MAX_QUOTED_PASSAGES also carry a `snippet`, the one line that best
* shows the query.
*
* @param {Number} itemID
* @return {Promise<Object[]>} - Entries with `text`, `ranges` and
@ -757,27 +760,22 @@ Zotero.BestMatch = new function () {
for (let i = 0; i < passages.length; i++) {
let passage = passages[i];
let share = lexical ? lexical[i] : 0;
// A passage the model didn't match has only its words to
// recommend it, so one that says nothing of the query isn't a
// match at all
let weighed = passage.score != null;
if (!weighed && !share) {
// A passage is as strong as its better explanation: the
// model's score on its band, unclamped so the strongest keep
// their order, or the share of the query the passage says
let fraction = typeof passage.score == 'number'
? Zotero.Embeddings.getScoreFraction(passage.score, { clamped: false })
: 0;
let strength = Math.max(fraction, share);
if (strength <= 0) {
continue;
}
// Over the engines that spoke for this item, so a strength is
// the same 0-1 fraction whether one weighed the passage or
// both did
let semanticWeight = weighed ? SEMANTIC_WEIGHT : 0;
let lexicalWeight = lexical ? LEXICAL_WEIGHT : 0;
let fraction = weighed ? Zotero.Embeddings.getScoreFraction(passage.score) : 0;
entries.push({
...passage,
ranges: ranges[i],
strength: (semanticWeight * fraction + lexicalWeight * share)
/ (semanticWeight + lexicalWeight)
});
entries.push({ ...passage, ranges: ranges[i], strength });
}
entries.sort((a, b) => b.strength - a.strength);
// Passages far below the item's best don't explain its match
let cutoff = (entries[0]?.strength ?? 0) * (1 - PASSAGE_MARGIN);
entries = entries.filter(entry => entry.strength >= cutoff);
// Only the strongest few passages are shown as rows in the tree,
// so only they get a line chosen to quote
await this._pickSnippets(entries.slice(0, MAX_QUOTED_PASSAGES));

25
test/tests/bestMatchTest.js
View file

@ -622,10 +622,25 @@ describe("Zotero.BestMatch", function () {
let session = await sessionFor();
let excerpts = await session.getMatchingExcerpts(attachment.id);
assert.lengthOf(excerpts, 2);
// 0.7 * 0.5 + 0.3 * 1 beats 0.7 * 0.9 + 0.3 * 0
// Each passage is as strong as its better explanation: saying
// the whole query beats resembling it closely
assert.include(excerpts[0].text, 'the owl itself');
assert.closeTo(excerpts[0].strength, 0.7 * 0.5 + 0.3, 1e-9);
assert.closeTo(excerpts[1].strength, 0.7 * 0.9, 1e-9);
assert.closeTo(excerpts[0].strength, 1, 1e-9);
assert.closeTo(excerpts[1].strength, 0.9, 1e-9);
});
it("should drop passages far below the item's strongest", async function () {
stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true));
stubs.push(sinon.stub(Zotero.Embeddings, 'getScoreFraction').callsFake(score => score));
stubs.push(sinon.stub(Zotero.Embeddings, 'getIndexedChunks').resolves([
{ text: 'the strongest passage about the owl', score: 0.9 },
{ text: 'a passage nearly as strong', score: 0.5 },
{ text: 'a passage far weaker', score: 0.4 }
]));
let session = await sessionFor({ lexical: false });
let excerpts = await session.getMatchingExcerpts(attachment.id);
assert.deepEqual(excerpts.map(excerpt => excerpt.strength), [0.9, 0.5]);
});
it("should skip the lexical engine for an item that didn't match it", async function () {
@ -685,9 +700,9 @@ describe("Zotero.BestMatch", function () {
assert.isTrue(chunksStub.alwaysCalledWith(attachment.id, { queryText: 'owl' }));
assert.isFalse(cutStub.called);
let byText = Object.fromEntries(excerpts.map(excerpt => [excerpt.text, excerpt]));
// Matched by the model alone: weighed by the model alone
// Matched by the model alone: as strong as the model's score
let liked = byText['a passage the model likes'];
assert.closeTo(liked.strength, 0.7 * 0.6, 1e-9);
assert.closeTo(liked.strength, 0.6, 1e-9);
assert.equal(liked.pageLabel, '3');
// Below the model's floor but saying the query: kept on the
// lexical engine's word alone, where the index places it