mirror of
https://github.com/zotero/zotero.git
synced 2026-09-11 22:51:15 +00:00
Ranked lexical scoring over the full-text indexes
Add the query-to-score flow to Zotero.Lexical, built on the content and
item-text indexes:
- Term statistics span both corpora: document frequency sums MATCH
counts over fulltextContent and fulltextItemText, corpus size sums
the state tables, so a term's rarity is a property of the library --
and libraries with few attachments still get real weights
- Units too common to matter are cut relative to the query's best unit
(INFORMATIVE_WEIGHT_FRACTION), so "of" is dropped next to "communism"
while an all-common query keeps its best word
- Matchers are index probes: matchContent() against the content index,
matchFields()/matchNotes()/matchAnnotations() against the item-text
columns. Notes fetch text only for probe matches plus stale/unindexed
notes (getStaleOrUnindexedNoteIDs); quoted phrases verify literally
against stored text everywhere (whitespace/hyphen runs interchange,
other punctuation must match)
- scoreItemIDs() assembles the score (see below), with a floor and
cancellation
Ranking algorithm, per query:
1. Parse into units (words, quoted phrases, CJK runs); trailing
mid-word token matches as a prefix
2. Weigh each unit by smoothed BM25 IDF from the combined corpora;
keep the informative ones
3. Match: presence (1) in titles, abstracts, annotations; saturated,
length-normalized term frequency for notes (computed from text)
and documents (recovered as rank ratios per unit -- for a one-unit
query, ranks compare documents exactly, and the strongest match
anchors 1)
4. Score = sum over units of weight x best boosted evidence across
sources (title x2, abstract x1.3; max, so one word never counts
twice), normalized against the query's ceiling: 1 = full-strength
match on everything asked; below SCORE_FLOOR is no match
This commit is contained in:
Bogdan Abaev
parent
628fd23069
commit
6dec06f18c
4 changed files with 1553 additions and 0 deletions
|
|
@ -2819,6 +2819,94 @@ Zotero.Fulltext = Zotero.FullText = new function () {
|
|||
};
|
||||
|
||||
|
||||
/**
|
||||
* IDs of the given notes whose note-index entries can't be relied on to reflect their
|
||||
* current text: notes edited since their last index update (see flagNoteStale()) and notes
|
||||
* not in the index at the current format version (e.g., mid-backfill). A caller matching
|
||||
* against the index should read these notes' current text instead (see
|
||||
* getNoteSearchTexts()).
|
||||
*
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Integer[]>}
|
||||
*/
|
||||
this.getStaleOrUnindexedNoteIDs = async function (itemIDs) {
|
||||
let result = [];
|
||||
let chunkSize = 500;
|
||||
for (let i = 0; i < itemIDs.length; i += chunkSize) {
|
||||
let chunk = itemIDs.slice(i, i + chunkSize);
|
||||
result.push(...await Zotero.DB.columnQueryAsync(
|
||||
"SELECT N.itemID FROM itemNotes N "
|
||||
+ "LEFT JOIN ftindex.fulltextNoteIndexState S USING (itemID) "
|
||||
+ "WHERE N.itemID IN (" + chunk.map(() => '?').join(',') + ") "
|
||||
+ "AND (S.itemID IS NULL OR S.version<?)",
|
||||
[...chunk, _contentIndexVersion]
|
||||
));
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Normalized searchable plain text of the given notes, keyed by itemID --
|
||||
* the same text note searching matches against: the note index's stored
|
||||
* plain text where the index is current, the in-memory text of notes
|
||||
* edited since their last index update (see flagNoteStale()), and a fresh
|
||||
* extraction for notes the index doesn't hold yet (e.g., mid-backfill).
|
||||
* IDs that aren't notes are simply absent from the result.
|
||||
*
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Map>} - itemID -> text
|
||||
*/
|
||||
this.getNoteSearchTexts = async function (itemIDs) {
|
||||
let texts = new Map();
|
||||
let chunkSize = 500;
|
||||
for (let i = 0; i < itemIDs.length; i += chunkSize) {
|
||||
let chunk = itemIDs.slice(i, i + chunkSize);
|
||||
let placeholders = chunk.map(() => '?').join(',');
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT itemID, text FROM ftindex.noteText WHERE itemID IN (" + placeholders + ")",
|
||||
chunk
|
||||
);
|
||||
for (let row of rows) {
|
||||
texts.set(row.itemID, row.text);
|
||||
}
|
||||
// A note edited since its last index update still holds its
|
||||
// pre-edit text in noteText -- its current text is what counts
|
||||
let staleIDs = new Set(await Zotero.DB.columnQueryAsync(
|
||||
"SELECT itemID FROM ftindex.fulltextNoteIndexState "
|
||||
+ "WHERE version=0 AND itemID IN (" + placeholders + ")",
|
||||
chunk
|
||||
));
|
||||
// Stale notes not seen since their edit (e.g., flagged in a
|
||||
// previous session) and notes with no index entry at all are both
|
||||
// extracted from the stored note
|
||||
let fetchIDs = chunk.filter((id) => {
|
||||
return staleIDs.has(id) ? !_staleNoteText.has(id) : !texts.has(id);
|
||||
});
|
||||
if (fetchIDs.length) {
|
||||
let noteRows = await Zotero.DB.queryAsync(
|
||||
"SELECT itemID, note FROM itemNotes WHERE itemID IN ("
|
||||
+ fetchIDs.map(() => '?').join(',') + ")",
|
||||
fetchIDs
|
||||
);
|
||||
for (let row of noteRows) {
|
||||
let text = _normalizeNoteText(row.note);
|
||||
texts.set(row.itemID, text);
|
||||
if (staleIDs.has(row.itemID)) {
|
||||
_staleNoteText.set(row.itemID, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let id of staleIDs) {
|
||||
if (_staleNoteText.has(id)) {
|
||||
texts.set(id, _staleNoteText.get(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
return texts;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Return the ids of attachment items in the given library whose full-text content matches
|
||||
* `searchText` (see getWordMatchClause for the matching semantics). This is the non-regexp
|
||||
|
|
|
|||
785
chrome/content/zotero/xpcom/lexical.js
Normal file
785
chrome/content/zotero/xpcom/lexical.js
Normal file
|
|
@ -0,0 +1,785 @@
|
|||
/*
|
||||
***** BEGIN LICENSE BLOCK *****
|
||||
|
||||
Copyright © 2026 Corporation for Digital Scholarship
|
||||
Vienna, Virginia, USA
|
||||
https://www.zotero.org
|
||||
|
||||
This file is part of Zotero.
|
||||
|
||||
Zotero is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Zotero is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
***** END LICENSE BLOCK *****
|
||||
*/
|
||||
|
||||
/**
|
||||
* Zotero.Lexical -- ranked lexical search over the library's own text.
|
||||
*
|
||||
* Scores how well a text answers a query rather than whether it contains
|
||||
* every word of it: any term can match, and a score accumulates the
|
||||
* evidence, so a document about owl migration in Norway still scores for
|
||||
* "owl migration in the united states" -- below the documents that cover
|
||||
* all of it.
|
||||
*
|
||||
* parseQuery() breaks a query into scoring units (words, quoted phrases,
|
||||
* CJK runs), and analyzeQuery() weighs each unit by how rare it is in the
|
||||
* user's own corpus, so that in "fall of communism", "communism" is what
|
||||
* mostly decides a score, "fall" counts a little, and "of" barely at all --
|
||||
* no stoplist, nothing curated by hand.
|
||||
*
|
||||
* The statistics behind the weights are document frequencies counted across
|
||||
* both of ftindex's word-level corpora (see Zotero.FullText): attachment
|
||||
* content and item text (titles, abstracts, notes, annotations). One count
|
||||
* per term over everything, so a term's rarity is a property of the library
|
||||
* -- a word filling every document stays cheap when it turns up in an
|
||||
* annotation, and a library with few attachments still measures rarity from
|
||||
* its items' own text.
|
||||
*
|
||||
* The match side reports which of a set of items contain which units:
|
||||
* matchContent() asks the attachment content index, and matchFields(),
|
||||
* matchNotes(), and matchAnnotations() ask the item-text index's columns. A
|
||||
* match is presence -- strength 1 -- where text is short enough that
|
||||
* containing a unit says everything (titles, abstracts, annotations);
|
||||
* notes and documents, whose lengths vary too much for that, are graded by
|
||||
* saturated, length-normalized term frequency. Quoted phrases are matched
|
||||
* literally everywhere: the indexes only prove a phrase's words adjacent,
|
||||
* so phrase matches are verified against the stored text.
|
||||
*
|
||||
* scoreItemIDs() assembles the score: each unit contributes its weight
|
||||
* times the best evidence for it across an item's sources, summed and
|
||||
* normalized against the query's ceiling -- 1 is a full-strength match on
|
||||
* everything the query asked -- with a floor below which an item isn't a
|
||||
* match at all.
|
||||
*/
|
||||
Zotero.Lexical = new function () {
|
||||
// CJK scripts (Han/Hiragana/Katakana/Hangul), the same set the full-text
|
||||
// index routes to its 2-gram tables (see fulltext.js): the word tokenizer
|
||||
// sees an unspaced CJK run as a single token, so runs are matched by
|
||||
// their bigrams instead
|
||||
const CJK_CLASS = '\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}';
|
||||
// The tokens of a normalized query part, in text order: a CJK run, or a
|
||||
// word token as the index's unicode61 tokenizer produces them -- a run of
|
||||
// letters and digits, with everything else a separator. The lookahead
|
||||
// keeps CJK characters (which are also \p{L}) out of word tokens, so
|
||||
// 'covid疫情' splits into a word and a run rather than reading as one word.
|
||||
const TOKEN_RE = new RegExp(
|
||||
`(?<cjk>[${CJK_CLASS}]+)|(?<word>(?:(?![${CJK_CLASS}])[\\p{L}\\p{N}])+)`,
|
||||
'gu'
|
||||
);
|
||||
// BM25's term-frequency shape: K1 sets how quickly repetition saturates,
|
||||
// B how much a long text discounts each occurrence (see
|
||||
// _saturatedStrength())
|
||||
const K1 = 1.2;
|
||||
const B = 0.75;
|
||||
// A unit is informative -- worth retrieving by -- when it carries at
|
||||
// least this share of the query's best unit's weight. Relative rather
|
||||
// than absolute, so "of" next to "communism" is dropped while a query of
|
||||
// nothing but common words keeps its best word and still returns results.
|
||||
const INFORMATIVE_WEIGHT_FRACTION = 0.2;
|
||||
// What a full-strength match in each source is worth relative to a
|
||||
// full-strength content match: a title names the work, an abstract
|
||||
// summarizes it, everything else speaks with equal voice
|
||||
const SOURCE_BOOSTS = {
|
||||
title: 2,
|
||||
abstract: 1.3,
|
||||
content: 1,
|
||||
note: 1,
|
||||
annotation: 1
|
||||
};
|
||||
// Normalized scores below this aren't matches and aren't returned: with
|
||||
// scores measured against the query's ceiling, this is the share of what
|
||||
// the query asked for that an item has to show. Provisional until tuned
|
||||
// against a real library.
|
||||
const SCORE_FLOOR = 0.05;
|
||||
|
||||
/**
|
||||
* Thrown when scoring is abandoned via the shouldCancel callback -- e.g.
|
||||
* because a newer query superseded the one being scored
|
||||
*/
|
||||
this.ScoringCancelledError = class extends Error {
|
||||
constructor(message = 'Scoring cancelled') {
|
||||
super(message);
|
||||
this.name = 'LexicalScoringCancelledError';
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a query into scoring units.
|
||||
*
|
||||
* A unit is the thing that matches (or doesn't) in one text and earns
|
||||
* its weight toward a score:
|
||||
* - a word: { type: 'word', text, prefix }. The query's trailing word is
|
||||
* flagged `prefix` while it's still being typed (no space or quote
|
||||
* after it yet), so it matches its completions.
|
||||
* - a quoted phrase: { type: 'phrase', text, tokens } -- multiple words
|
||||
* matched adjacently, as typed.
|
||||
* - a CJK run: { type: 'cjk', text, bigrams } -- matched contiguously via
|
||||
* the 2-gram index, so quoting adds nothing it doesn't already have.
|
||||
* `bigrams` is null for a single character, which has none.
|
||||
* A part mixing scripts splits into word and run units, and a quoted
|
||||
* part mixing scripts splits the same way, since neither index side
|
||||
* covers it whole.
|
||||
*
|
||||
* A repeated term counts once: repeating a word in the query isn't more
|
||||
* evidence about the texts it matches. When the same word appears both
|
||||
* mid-query and as the trailing prefix, the exact form wins.
|
||||
*
|
||||
* @param {String} queryText
|
||||
* @return {Object[]}
|
||||
*/
|
||||
this.parseQuery = function (queryText) {
|
||||
if (!queryText) {
|
||||
return [];
|
||||
}
|
||||
// Mid-word means the query ends in a word character; a space, a
|
||||
// closing quote, or punctuation after the word means it's finished
|
||||
let endsMidWord = /[\p{L}\p{N}]$/u.test(queryText);
|
||||
let parts = Zotero.SearchConditions.parseSearchString(queryText);
|
||||
let units = [];
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
let part = parts[i];
|
||||
let fromLastPart = i == parts.length - 1;
|
||||
let normalized = Zotero.Utilities.Internal.normalizeForSearch(part.text);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
let matches = [...normalized.matchAll(TOKEN_RE)];
|
||||
if (!matches.length) {
|
||||
continue;
|
||||
}
|
||||
let words = matches.filter(m => m.groups.word).map(m => m.groups.word);
|
||||
let hasCJK = matches.some(m => m.groups.cjk);
|
||||
if (part.inQuotes && words.length > 1 && !hasCJK) {
|
||||
units.push({
|
||||
type: 'phrase',
|
||||
text: words.join(' '),
|
||||
tokens: words,
|
||||
fromLastPart,
|
||||
quoted: true
|
||||
});
|
||||
continue;
|
||||
}
|
||||
for (let match of matches) {
|
||||
if (match.groups.cjk) {
|
||||
units.push({
|
||||
type: 'cjk',
|
||||
text: match.groups.cjk,
|
||||
bigrams: _getBigrams(match.groups.cjk),
|
||||
fromLastPart,
|
||||
quoted: part.inQuotes
|
||||
});
|
||||
}
|
||||
else {
|
||||
units.push({
|
||||
type: 'word',
|
||||
text: match.groups.word,
|
||||
prefix: false,
|
||||
fromLastPart,
|
||||
quoted: part.inQuotes
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// The trailing word of an unquoted query is the one still being
|
||||
// typed. A quoted trailing part is exact by declaration.
|
||||
if (units.length && endsMidWord) {
|
||||
let last = units[units.length - 1];
|
||||
if (last.type == 'word' && last.fromLastPart && !last.quoted) {
|
||||
last.prefix = true;
|
||||
}
|
||||
}
|
||||
let deduped = new Map();
|
||||
for (let unit of units) {
|
||||
delete unit.fromLastPart;
|
||||
delete unit.quoted;
|
||||
let key = unit.type + '\n' + unit.text;
|
||||
let existing = deduped.get(key);
|
||||
if (!existing || (existing.prefix && !unit.prefix)) {
|
||||
deduped.set(key, unit);
|
||||
}
|
||||
}
|
||||
return [...deduped.values()];
|
||||
};
|
||||
|
||||
/**
|
||||
* The scoring units of a query (see parseQuery()), each weighted by how
|
||||
* rare it is in the corpus. A score built from these accumulates the
|
||||
* weights of the units a text matches, so matching "communism" moves a
|
||||
* score far more than matching "fall".
|
||||
*
|
||||
* @param {String} queryText
|
||||
* @return {Promise<Object[]>} - parseQuery()'s units, each with:
|
||||
* df - documents in the corpus matching the unit
|
||||
* weight - what a match on this unit contributes to a score
|
||||
* informative - whether the unit carries enough of the query's
|
||||
* weight to be worth retrieving by (see
|
||||
* INFORMATIVE_WEIGHT_FRACTION); the best-weighted unit always is
|
||||
*/
|
||||
this.analyzeQuery = async function (queryText) {
|
||||
let units = this.parseQuery(queryText);
|
||||
if (!units.length) {
|
||||
return units;
|
||||
}
|
||||
let corpusSize = await this.getCorpusSize();
|
||||
for (let unit of units) {
|
||||
unit.df = await this.getDocumentFrequency(unit);
|
||||
unit.weight = _idf(unit.df, corpusSize);
|
||||
}
|
||||
let maxWeight = Math.max(...units.map(unit => unit.weight));
|
||||
for (let unit of units) {
|
||||
unit.informative = unit.weight >= maxWeight * INFORMATIVE_WEIGHT_FRACTION;
|
||||
}
|
||||
return units;
|
||||
};
|
||||
|
||||
/**
|
||||
* Number of documents the term statistics are measured against: every
|
||||
* attachment, item, note, and annotation recorded in the full-text
|
||||
* index's state tables -- including ones indexed with no text, which are
|
||||
* real documents that happen to contain nothing.
|
||||
*
|
||||
* @return {Promise<Number>}
|
||||
*/
|
||||
this.getCorpusSize = async function () {
|
||||
return (await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM ftindex.fulltextIndexState"
|
||||
)) + (await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM ftindex.fulltextItemTextState"
|
||||
)) + (await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM ftindex.fulltextNoteIndexState"
|
||||
));
|
||||
};
|
||||
|
||||
/**
|
||||
* How many documents match a unit: the `df` behind its weight, counted
|
||||
* across both word-level corpora -- attachment content and item text --
|
||||
* whose ID spaces are disjoint, so the sum counts nothing twice. Counted
|
||||
* the way the unit will be matched: a prefix word against every
|
||||
* completion, a phrase by adjacent occurrence, a CJK run by its
|
||||
* contiguous bigrams.
|
||||
*
|
||||
* A single-character CJK unit has no bigram of its own, so its count is
|
||||
* approximated by prefix-matching the bigrams that start with it. That
|
||||
* undercounts a character that only ends runs, which overstates its
|
||||
* weight -- rarer reads as more important, the safe direction to be wrong.
|
||||
*
|
||||
* @param {Object} unit - A unit from parseQuery()
|
||||
* @return {Promise<Number>}
|
||||
*/
|
||||
this.getDocumentFrequency = async function (unit) {
|
||||
let clause = _getMatchClause(unit);
|
||||
let df = 0;
|
||||
for (let table of [clause.contentTable, clause.itemTextTable]) {
|
||||
df += await Zotero.DB.valueQueryAsync(
|
||||
"SELECT COUNT(*) FROM ftindex." + table
|
||||
+ " WHERE " + table + " MATCH ?",
|
||||
[clause.match]
|
||||
);
|
||||
}
|
||||
return df;
|
||||
};
|
||||
|
||||
/**
|
||||
* Which of the given items' indexed attachment content contains which
|
||||
* units, and how strongly relative to each other. For a one-unit
|
||||
* expression the index's rank is a constant times BM25's saturated,
|
||||
* length-normalized term frequency, so ranks compare documents exactly;
|
||||
* the constant itself is unknowable, so the strongest match anchors 1 and
|
||||
* the rest scale under it. A document that keeps returning to a term
|
||||
* outranks one that mentions it once in passing -- but strengths are
|
||||
* relative to the candidates at hand, so a lone weak match still reads
|
||||
* as 1.
|
||||
*
|
||||
* A phrase's index match only proves its words adjacent -- the index
|
||||
* ignores what separates them -- so phrase matches are verified against
|
||||
* the stored document text and only literal occurrences count.
|
||||
*
|
||||
* Only items with a row in the content index (indexed attachments) can
|
||||
* match; everything else is simply absent from the result.
|
||||
*
|
||||
* @param {Object[]} units - Units from parseQuery()
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Map>} - itemID -> Map(unit -> strength)
|
||||
*/
|
||||
this.matchContent = async function (units, itemIDs) {
|
||||
let strengths = new Map();
|
||||
if (!units.length || !itemIDs.length) {
|
||||
return strengths;
|
||||
}
|
||||
for (let unit of units) {
|
||||
let clause = _getMatchClause(unit);
|
||||
let matched = await _probe(clause.contentTable, clause.match, itemIDs);
|
||||
if (!matched.length) {
|
||||
continue;
|
||||
}
|
||||
if (unit.type == 'phrase') {
|
||||
let verified = new Set(
|
||||
(await Zotero.FullText.findTextInItems(
|
||||
matched.map(row => row.rowid), unit.text
|
||||
)).map(x => x.id)
|
||||
);
|
||||
matched = matched.filter(row => verified.has(row.rowid));
|
||||
if (!matched.length) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// rank is negative, better more negative, so the best is the
|
||||
// minimum and every ratio against it lands in (0, 1]
|
||||
let best = Math.min(...matched.map(row => row.rank));
|
||||
for (let row of matched) {
|
||||
_addStrength(strengths, row.rowid, unit,
|
||||
best ? row.rank / best : 1);
|
||||
}
|
||||
}
|
||||
return strengths;
|
||||
};
|
||||
|
||||
/**
|
||||
* Which of the given items' titles and abstracts contain which units,
|
||||
* reported per column. A match is presence (strength 1): a title or
|
||||
* abstract is short enough that containing a unit says everything.
|
||||
* Titles cover the type-specific title fields (caseName, subject,
|
||||
* nameOfAct) along with `title` itself. Phrase matches are verified
|
||||
* literally against the stored field values.
|
||||
*
|
||||
* Items without a matching title or abstract are simply absent from the
|
||||
* respective result.
|
||||
*
|
||||
* @param {Object[]} units - Units from parseQuery()
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Object>} - { title: Map(itemID -> Map(unit ->
|
||||
* strength)), abstract: Map(itemID -> Map(unit -> strength)) }
|
||||
*/
|
||||
this.matchFields = async function (units, itemIDs) {
|
||||
let result = { title: new Map(), abstract: new Map() };
|
||||
if (!units.length || !itemIDs.length) {
|
||||
return result;
|
||||
}
|
||||
for (let unit of units) {
|
||||
let clause = _getMatchClause(unit);
|
||||
for (let column of ['title', 'abstract']) {
|
||||
let matched = (await _probe(
|
||||
clause.itemTextTable, column + ':' + clause.match, itemIDs
|
||||
)).map(row => row.rowid);
|
||||
if (!matched.length) {
|
||||
continue;
|
||||
}
|
||||
if (unit.type == 'phrase') {
|
||||
matched = await _verifyFieldPhrase(unit, column, matched);
|
||||
}
|
||||
for (let itemID of matched) {
|
||||
_addStrength(result[column], itemID, unit, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
/**
|
||||
* Which of the given items' note text contains which units, and how
|
||||
* strongly. Notes range from a line to a chapter, so strength is BM25's
|
||||
* saturated, length-normalized term frequency (see _saturatedStrength()):
|
||||
* a note that keeps returning to a term outranks one that mentions it
|
||||
* once in passing.
|
||||
*
|
||||
* The index answers which notes are worth reading: text is fetched only
|
||||
* for notes whose note column matches a unit, plus notes whose index
|
||||
* entries can't be trusted -- edited since their last index update, or
|
||||
* not indexed yet -- whose current text is always read. Phrases count
|
||||
* only literally (see _countPhrase()).
|
||||
*
|
||||
* Items without matching note text are simply absent from the result.
|
||||
*
|
||||
* @param {Object[]} units - Units from parseQuery()
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Map>} - itemID -> Map(unit -> strength)
|
||||
*/
|
||||
this.matchNotes = async function (units, itemIDs) {
|
||||
let strengths = new Map();
|
||||
if (!units.length || !itemIDs.length) {
|
||||
return strengths;
|
||||
}
|
||||
let fetchIDs = new Set(await Zotero.FullText.getStaleOrUnindexedNoteIDs(itemIDs));
|
||||
for (let unit of units) {
|
||||
let clause = _getMatchClause(unit);
|
||||
let matched = await _probe(
|
||||
clause.itemTextTable, 'note:' + clause.match, itemIDs);
|
||||
for (let row of matched) {
|
||||
fetchIDs.add(row.rowid);
|
||||
}
|
||||
}
|
||||
if (!fetchIDs.size) {
|
||||
return strengths;
|
||||
}
|
||||
let texts = await Zotero.FullText.getNoteSearchTexts([...fetchIDs]);
|
||||
if (!texts.size) {
|
||||
return strengths;
|
||||
}
|
||||
// Length normalization needs the typical note length. The note index
|
||||
// knows it; without one yet, the notes at hand stand in for the
|
||||
// population.
|
||||
let avgLength = await Zotero.DB.valueQueryAsync(
|
||||
"SELECT AVG(LENGTH(text)) FROM ftindex.noteText"
|
||||
);
|
||||
if (!avgLength) {
|
||||
let lengths = [...texts.values()].map(text => text.length);
|
||||
avgLength = (lengths.reduce((sum, length) => sum + length, 0)
|
||||
/ (lengths.length || 1)) || 1;
|
||||
}
|
||||
for (let [itemID, text] of texts) {
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
let scan = _scanText(text);
|
||||
for (let unit of units) {
|
||||
let tf = _countUnit(unit, scan);
|
||||
if (tf) {
|
||||
_addStrength(strengths, itemID, unit,
|
||||
_saturatedStrength(tf, text.length, avgLength));
|
||||
}
|
||||
}
|
||||
}
|
||||
return strengths;
|
||||
};
|
||||
|
||||
/**
|
||||
* Which of the given items' annotation text -- the passage an annotation
|
||||
* marks together with its comment -- contains which units. A match is
|
||||
* presence (strength 1): an annotation is short enough that containing a
|
||||
* unit says everything. Phrase matches are verified literally against
|
||||
* the stored annotation text.
|
||||
*
|
||||
* Items without matching annotation text are simply absent from the
|
||||
* result.
|
||||
*
|
||||
* @param {Object[]} units - Units from parseQuery()
|
||||
* @param {Integer[]} itemIDs
|
||||
* @return {Promise<Map>} - itemID -> Map(unit -> strength)
|
||||
*/
|
||||
this.matchAnnotations = async function (units, itemIDs) {
|
||||
let strengths = new Map();
|
||||
if (!units.length || !itemIDs.length) {
|
||||
return strengths;
|
||||
}
|
||||
for (let unit of units) {
|
||||
let clause = _getMatchClause(unit);
|
||||
let matched = (await _probe(
|
||||
clause.itemTextTable, 'annotation:' + clause.match, itemIDs
|
||||
)).map(row => row.rowid);
|
||||
if (!matched.length) {
|
||||
continue;
|
||||
}
|
||||
if (unit.type == 'phrase') {
|
||||
matched = await _verifyAnnotationPhrase(unit, matched);
|
||||
}
|
||||
for (let itemID of matched) {
|
||||
_addStrength(strengths, itemID, unit, 1);
|
||||
}
|
||||
}
|
||||
return strengths;
|
||||
};
|
||||
|
||||
/**
|
||||
* Score a given set of items by how well their text answers a query.
|
||||
*
|
||||
* Any informative unit can match (see analyzeQuery()); each contributes
|
||||
* its weight times the best evidence for it across the item's sources --
|
||||
* title, abstract, attachment content, notes, annotations, boosted per
|
||||
* source (see SOURCE_BOOSTS) -- so the same word in two places counts
|
||||
* once, at its strongest. The sum is normalized against the query's
|
||||
* ceiling (every informative unit at full strength in the best-boosted
|
||||
* source): 1 is a full-strength match on everything the query asked,
|
||||
* partial coverage lands proportionally lower, dominated by the rare
|
||||
* units. Items below SCORE_FLOOR aren't matches and aren't returned.
|
||||
*
|
||||
* Units too common to be informative play no part: they neither gate nor
|
||||
* move a score.
|
||||
*
|
||||
* @param {String} queryText
|
||||
* @param {Number[]} itemIDs - Candidate item IDs to score
|
||||
* @param {Object} [options]
|
||||
* @param {Function} [options.shouldCancel] - Checked between matching
|
||||
* stages; return true to abandon scoring with a ScoringCancelledError
|
||||
* @return {Promise<Map>} - itemID -> score (0-1, higher is better)
|
||||
*/
|
||||
this.scoreItemIDs = async function (queryText, itemIDs, { shouldCancel } = {}) {
|
||||
let scores = new Map();
|
||||
if (!itemIDs.length) {
|
||||
return scores;
|
||||
}
|
||||
let checkCancel = () => {
|
||||
if (shouldCancel && shouldCancel()) {
|
||||
throw new this.ScoringCancelledError();
|
||||
}
|
||||
};
|
||||
let units = (await this.analyzeQuery(queryText))
|
||||
.filter(unit => unit.informative);
|
||||
if (!units.length) {
|
||||
return scores;
|
||||
}
|
||||
checkCancel();
|
||||
let content = await this.matchContent(units, itemIDs);
|
||||
checkCancel();
|
||||
let fields = await this.matchFields(units, itemIDs);
|
||||
checkCancel();
|
||||
let notes = await this.matchNotes(units, itemIDs);
|
||||
checkCancel();
|
||||
let annotations = await this.matchAnnotations(units, itemIDs);
|
||||
checkCancel();
|
||||
|
||||
// Best boosted evidence per item per unit, across all sources
|
||||
let evidence = new Map();
|
||||
let sources = [
|
||||
[fields.title, SOURCE_BOOSTS.title],
|
||||
[fields.abstract, SOURCE_BOOSTS.abstract],
|
||||
[content, SOURCE_BOOSTS.content],
|
||||
[notes, SOURCE_BOOSTS.note],
|
||||
[annotations, SOURCE_BOOSTS.annotation]
|
||||
];
|
||||
for (let [strengths, boost] of sources) {
|
||||
for (let [itemID, unitStrengths] of strengths) {
|
||||
for (let [unit, strength] of unitStrengths) {
|
||||
_addStrength(evidence, itemID, unit, boost * strength);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let maxBoost = Math.max(...Object.values(SOURCE_BOOSTS));
|
||||
let ceiling = units.reduce((sum, unit) => sum + unit.weight, 0) * maxBoost;
|
||||
if (!ceiling) {
|
||||
return scores;
|
||||
}
|
||||
for (let [itemID, unitStrengths] of evidence) {
|
||||
let raw = 0;
|
||||
for (let [unit, strength] of unitStrengths) {
|
||||
raw += unit.weight * strength;
|
||||
}
|
||||
let score = raw / ceiling;
|
||||
if (score >= SCORE_FLOOR) {
|
||||
scores.set(itemID, score);
|
||||
}
|
||||
}
|
||||
return scores;
|
||||
};
|
||||
|
||||
// The MATCH expression that finds a unit, with the content and item-text
|
||||
// tables (word or CJK pair) it runs against. Unit text is all letters
|
||||
// and digits (parseQuery tokenized it), so quoting it into an FTS phrase
|
||||
// needs no escaping.
|
||||
function _getMatchClause(unit) {
|
||||
if (unit.type == 'cjk') {
|
||||
return {
|
||||
match: unit.bigrams
|
||||
? '"' + unit.bigrams + '"'
|
||||
: '"' + unit.text + '"*',
|
||||
contentTable: 'fulltextContentCJK',
|
||||
itemTextTable: 'fulltextItemTextCJK'
|
||||
};
|
||||
}
|
||||
return {
|
||||
match: unit.type == 'phrase'
|
||||
? '"' + unit.text + '"'
|
||||
: '"' + unit.text + '"' + (unit.prefix ? '*' : ''),
|
||||
contentTable: 'fulltextContent',
|
||||
itemTextTable: 'fulltextItemText'
|
||||
};
|
||||
}
|
||||
|
||||
// A CJK run's overlapping 2-grams, joined with spaces -- built the same
|
||||
// way the index builds them (see getCJKBigrams() in fulltext.js), which
|
||||
// is what makes them match. Null for a single character, which has none.
|
||||
function _getBigrams(run) {
|
||||
if (run.length < 2) {
|
||||
return null;
|
||||
}
|
||||
let bigrams = [];
|
||||
for (let i = 0; i < run.length - 1; i++) {
|
||||
bigrams.push(run.substr(i, 2));
|
||||
}
|
||||
return bigrams.join(' ');
|
||||
}
|
||||
|
||||
// Smoothed BM25 inverse document frequency:
|
||||
//
|
||||
// ln(1 + (N - df + 0.5) / (df + 0.5))
|
||||
//
|
||||
// the standard measure of how much information a term carries, and the
|
||||
// whole term-importance mechanism: no stoplist, just counting.
|
||||
// - df = 0 -- a term in no indexed document (a typo, or a word from text
|
||||
// we haven't indexed) -- gets the query's maximum: unseen reads as
|
||||
// rare reads as important
|
||||
// - df = N -- a term in everything ("the") -- approaches zero
|
||||
// - N = 0 -- nothing indexed to measure against -- gives every unit the
|
||||
// same ln(2), so ranking degrades to term coverage
|
||||
function _idf(df, corpusSize) {
|
||||
// The single-CJK-character approximation and an index mid-write can
|
||||
// disagree slightly with the row count
|
||||
df = Math.max(0, Math.min(df, corpusSize));
|
||||
return Math.log(1 + (corpusSize - df + 0.5) / (df + 0.5));
|
||||
}
|
||||
|
||||
// The requested candidates matching an FTS expression, probed in chunks
|
||||
// (the bound-parameter limit). Each row carries the index's rank for the
|
||||
// expression, for callers that grade matches against each other; callers
|
||||
// that only need membership read the rowids.
|
||||
async function _probe(table, match, itemIDs) {
|
||||
let matched = [];
|
||||
let chunkSize = 500;
|
||||
for (let i = 0; i < itemIDs.length; i += chunkSize) {
|
||||
let chunk = itemIDs.slice(i, i + chunkSize);
|
||||
matched.push(...await Zotero.DB.queryAsync(
|
||||
"SELECT rowid, rank FROM ftindex." + table
|
||||
+ " WHERE " + table + " MATCH ? "
|
||||
+ "AND rowid IN (" + chunk.map(() => '?').join(',') + ")",
|
||||
[match, ...chunk]
|
||||
));
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
// Of the given items, those whose stored field text (title-family fields
|
||||
// or the abstract) literally contains a phrase unit
|
||||
async function _verifyFieldPhrase(unit, column, itemIDs) {
|
||||
let fieldIDs = column == 'title'
|
||||
? [
|
||||
Zotero.ItemFields.getID('title'),
|
||||
...Zotero.ItemFields.getTypeFieldsFromBase('title')
|
||||
]
|
||||
: [Zotero.ItemFields.getID('abstractNote')];
|
||||
let verified = [];
|
||||
let chunkSize = 500;
|
||||
for (let i = 0; i < itemIDs.length; i += chunkSize) {
|
||||
let chunk = itemIDs.slice(i, i + chunkSize);
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT itemID, value FROM itemData "
|
||||
+ "JOIN itemDataValues USING (valueID) "
|
||||
+ "WHERE fieldID IN (" + fieldIDs.join(',') + ") "
|
||||
+ "AND itemID IN (" + chunk.map(() => '?').join(',') + ")",
|
||||
chunk
|
||||
);
|
||||
for (let row of rows) {
|
||||
let normalized = Zotero.Utilities.Internal.normalizeForSearch(row.value);
|
||||
if (normalized && _countPhrase(normalized, unit.text)) {
|
||||
verified.push(row.itemID);
|
||||
}
|
||||
}
|
||||
}
|
||||
return verified;
|
||||
}
|
||||
|
||||
// Of the given annotations, those whose passage and comment literally
|
||||
// contain a phrase unit
|
||||
async function _verifyAnnotationPhrase(unit, itemIDs) {
|
||||
let verified = [];
|
||||
let chunkSize = 500;
|
||||
for (let i = 0; i < itemIDs.length; i += chunkSize) {
|
||||
let chunk = itemIDs.slice(i, i + chunkSize);
|
||||
let rows = await Zotero.DB.queryAsync(
|
||||
"SELECT itemID, text, comment FROM itemAnnotations "
|
||||
+ "WHERE itemID IN (" + chunk.map(() => '?').join(',') + ")",
|
||||
chunk
|
||||
);
|
||||
for (let row of rows) {
|
||||
let normalized = Zotero.Utilities.Internal.normalizeForSearch(
|
||||
[row.text, row.comment].filter(Boolean).join(' ')
|
||||
);
|
||||
if (normalized && _countPhrase(normalized, unit.text)) {
|
||||
verified.push(row.itemID);
|
||||
}
|
||||
}
|
||||
}
|
||||
return verified;
|
||||
}
|
||||
|
||||
// A text prepared for unit counting: its token stream (word tokens and
|
||||
// CJK runs, in text order) and the normalized text itself, which is what
|
||||
// CJK units and phrases match against
|
||||
function _scanText(text) {
|
||||
let tokens = [];
|
||||
for (let match of text.matchAll(TOKEN_RE)) {
|
||||
tokens.push(match.groups.cjk || match.groups.word);
|
||||
}
|
||||
return { text, tokens };
|
||||
}
|
||||
|
||||
// Occurrences of a unit in a scanned text, counted the way the indexes
|
||||
// match the unit: a word as a whole token (a prefix unit by its
|
||||
// completions), a CJK run contiguously, a phrase literally (see
|
||||
// _countPhrase())
|
||||
function _countUnit(unit, scan) {
|
||||
if (unit.type == 'cjk') {
|
||||
let count = 0;
|
||||
let index = scan.text.indexOf(unit.text);
|
||||
while (index != -1) {
|
||||
count++;
|
||||
index = scan.text.indexOf(unit.text, index + unit.text.length);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
if (unit.type == 'phrase') {
|
||||
return _countPhrase(scan.text, unit.text);
|
||||
}
|
||||
if (unit.prefix) {
|
||||
return scan.tokens.filter(token => token.startsWith(unit.text)).length;
|
||||
}
|
||||
return scan.tokens.filter(token => token === unit.text).length;
|
||||
}
|
||||
|
||||
// Occurrences of a phrase in normalized text: literal, except that
|
||||
// whitespace and hyphen runs separate the phrase's words interchangeably
|
||||
// -- extraction layout and compound styling vary them -- matching the
|
||||
// collapse the content verification applies (see findTextInString() in
|
||||
// fulltext.js). Word boundaries hold at both ends, so a phrase never
|
||||
// starts or ends inside a longer word.
|
||||
function _countPhrase(text, phrase) {
|
||||
let collapsed = text.replace(/[\s-]+/g, ' ');
|
||||
let count = 0;
|
||||
let index = collapsed.indexOf(phrase);
|
||||
while (index != -1) {
|
||||
let before = index > 0 ? collapsed[index - 1] : '';
|
||||
let after = collapsed[index + phrase.length] || '';
|
||||
if (!/[\p{L}\p{N}]/u.test(before) && !/[\p{L}\p{N}]/u.test(after)) {
|
||||
count++;
|
||||
}
|
||||
index = collapsed.indexOf(phrase, index + 1);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// BM25's saturated, length-normalized term frequency, mapped onto (0, 1):
|
||||
//
|
||||
// tf / (tf + K1 * (1 - B + B * length / avgLength))
|
||||
//
|
||||
// One occurrence in an average-length text lands around 0.45, repetition
|
||||
// approaches 1, and each occurrence counts for less in a longer text.
|
||||
// Lengths are in characters on both sides of the ratio, which is all the
|
||||
// ratio needs.
|
||||
function _saturatedStrength(tf, length, avgLength) {
|
||||
return tf / (tf + K1 * (1 - B + B * (length / avgLength)));
|
||||
}
|
||||
|
||||
// Record a unit's strength for an item, keeping the strongest when the
|
||||
// same unit matches an item more than once (e.g., in two title fields)
|
||||
function _addStrength(strengths, itemID, unit, strength) {
|
||||
let unitStrengths = strengths.get(itemID);
|
||||
if (!unitStrengths) {
|
||||
unitStrengths = new Map();
|
||||
strengths.set(itemID, unitStrengths);
|
||||
}
|
||||
let previous = unitStrengths.get(unit);
|
||||
if (previous === undefined || strength > previous) {
|
||||
unitStrengths.set(unit, strength);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -109,6 +109,7 @@ const xpcomFilesLocal = [
|
|||
'httpIntegrationClient',
|
||||
'id',
|
||||
'integration',
|
||||
'lexical',
|
||||
'locale',
|
||||
'locateManager',
|
||||
'mime',
|
||||
|
|
|
|||
679
test/tests/lexicalTest.js
Normal file
679
test/tests/lexicalTest.js
Normal file
|
|
@ -0,0 +1,679 @@
|
|||
"use strict";
|
||||
|
||||
describe("Zotero.Lexical", function () {
|
||||
describe("#parseQuery()", function () {
|
||||
it("should split a query into normalized word units", function () {
|
||||
// Trailing space: the last word is finished, so nothing is a prefix
|
||||
let units = Zotero.Lexical.parseQuery("OWL Migrátion ");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'word', text: 'owl', prefix: false },
|
||||
{ type: 'word', text: 'migration', prefix: false }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should flag the trailing mid-word token as a prefix", function () {
|
||||
let units = Zotero.Lexical.parseQuery("owl migr");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'word', text: 'owl', prefix: false },
|
||||
{ type: 'word', text: 'migr', prefix: true }
|
||||
]);
|
||||
// Punctuation after the word means it's finished too
|
||||
units = Zotero.Lexical.parseQuery("owl migr.");
|
||||
assert.isFalse(units[1].prefix);
|
||||
});
|
||||
|
||||
it("should keep a quoted multi-word part as one exact phrase", function () {
|
||||
let units = Zotero.Lexical.parseQuery('"united states" owl');
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'phrase', text: 'united states', tokens: ['united', 'states'] },
|
||||
{ type: 'word', text: 'owl', prefix: true }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should treat a quoted single word as an exact word", function () {
|
||||
let units = Zotero.Lexical.parseQuery('"owl"');
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'word', text: 'owl', prefix: false }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should split a mixed-script part into word and CJK units", function () {
|
||||
let units = Zotero.Lexical.parseQuery("covid疫情");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'word', text: 'covid', prefix: false },
|
||||
{ type: 'cjk', text: '疫情', bigrams: '疫情' }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should carry a CJK run as its overlapping bigrams", function () {
|
||||
let units = Zotero.Lexical.parseQuery("疫情控制 ");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'cjk', text: '疫情控制', bigrams: '疫情 情控 控制' }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should carry a single CJK character with no bigrams", function () {
|
||||
let units = Zotero.Lexical.parseQuery("疫 ");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'cjk', text: '疫', bigrams: null }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should count a repeated term once, preferring its exact form", function () {
|
||||
// The trailing 'owl' would be a prefix, but the query already
|
||||
// contains it as a finished word
|
||||
let units = Zotero.Lexical.parseQuery("owl migration owl");
|
||||
assert.deepEqual(units, [
|
||||
{ type: 'word', text: 'owl', prefix: false },
|
||||
{ type: 'word', text: 'migration', prefix: false }
|
||||
]);
|
||||
});
|
||||
|
||||
it("should return no units for text with nothing to match", function () {
|
||||
assert.deepEqual(Zotero.Lexical.parseQuery(""), []);
|
||||
assert.deepEqual(Zotero.Lexical.parseQuery(" "), []);
|
||||
assert.deepEqual(Zotero.Lexical.parseQuery("!!! ..."), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#analyzeQuery()", function () {
|
||||
var stubs = [];
|
||||
|
||||
afterEach(function () {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
stubs = [];
|
||||
});
|
||||
|
||||
function stubStatistics(corpusSize, dfByText) {
|
||||
stubs.push(sinon.stub(Zotero.Lexical, 'getCorpusSize')
|
||||
.resolves(corpusSize));
|
||||
stubs.push(sinon.stub(Zotero.Lexical, 'getDocumentFrequency')
|
||||
.callsFake(async unit => dfByText[unit.text] ?? 0));
|
||||
}
|
||||
|
||||
it("should weight terms by rarity, with ubiquitous ones near zero", async function () {
|
||||
stubStatistics(1000, { fall: 600, of: 1000, communism: 5 });
|
||||
let units = await Zotero.Lexical.analyzeQuery("fall of communism ");
|
||||
let byText = new Map(units.map(unit => [unit.text, unit]));
|
||||
// "communism" is what decides this query's ranking
|
||||
assert.isAbove(byText.get('communism').weight, byText.get('fall').weight);
|
||||
assert.isAbove(byText.get('fall').weight, byText.get('of').weight);
|
||||
// A term in every document carries almost nothing -- the stoplist,
|
||||
// without a stoplist
|
||||
assert.isBelow(byText.get('of').weight, 0.001);
|
||||
// The documented formula: ln(1 + (N - df + 0.5) / (df + 0.5))
|
||||
assert.approximately(
|
||||
byText.get('communism').weight,
|
||||
Math.log(1 + (1000 - 5 + 0.5) / (5 + 0.5)),
|
||||
1e-12
|
||||
);
|
||||
});
|
||||
|
||||
it("should give a term in no document the query's maximum weight", async function () {
|
||||
stubStatistics(1000, { communism: 5, zzunseen: 0 });
|
||||
let units = await Zotero.Lexical.analyzeQuery("communism zzunseen ");
|
||||
let byText = new Map(units.map(unit => [unit.text, unit]));
|
||||
assert.isAbove(byText.get('zzunseen').weight, byText.get('communism').weight);
|
||||
});
|
||||
|
||||
it("should degrade to uniform weights with no corpus", async function () {
|
||||
stubStatistics(0, {});
|
||||
let units = await Zotero.Lexical.analyzeQuery("owl migration routes ");
|
||||
assert.lengthOf(units, 3);
|
||||
for (let unit of units) {
|
||||
assert.approximately(unit.weight, Math.log(2), 1e-12);
|
||||
}
|
||||
});
|
||||
|
||||
it("should look up statistics once per unique unit", async function () {
|
||||
stubStatistics(1000, { owl: 3, migration: 40 });
|
||||
await Zotero.Lexical.analyzeQuery("owl owl migration owl ");
|
||||
assert.equal(Zotero.Lexical.getDocumentFrequency.callCount, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("term statistics", function () {
|
||||
// Fabricated corpus rows inserted straight into the real content
|
||||
// index, with rowids no item can collide with
|
||||
const BASE_ROWID = 900000000;
|
||||
var inserted = [];
|
||||
|
||||
async function addDoc(id, text, cjkBigrams) {
|
||||
let rowid = BASE_ROWID + id;
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)",
|
||||
[rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || '']
|
||||
);
|
||||
if (cjkBigrams) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContentCJK (rowid, text) VALUES (?, ?)",
|
||||
[rowid, cjkBigrams]
|
||||
);
|
||||
}
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)",
|
||||
[rowid]
|
||||
);
|
||||
inserted.push(rowid);
|
||||
}
|
||||
|
||||
after(async function () {
|
||||
for (let rowid of inserted) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid);
|
||||
}
|
||||
});
|
||||
|
||||
it("should count the documents matching a word, not its occurrences", async function () {
|
||||
await addDoc(1, "lexowl migration and lexowl wintering across the lexbaltic");
|
||||
await addDoc(2, "lexowl feeding grounds");
|
||||
await addDoc(3, "entirely unrelated text");
|
||||
let df = unit => Zotero.Lexical.getDocumentFrequency(unit);
|
||||
assert.equal(await df({ type: 'word', text: 'lexowl', prefix: false }), 2);
|
||||
assert.equal(await df({ type: 'word', text: 'lexbaltic', prefix: false }), 1);
|
||||
assert.equal(await df({ type: 'word', text: 'lexmissing', prefix: false }), 0);
|
||||
// Words match whole tokens, not substrings
|
||||
assert.equal(await df({ type: 'word', text: 'lexow', prefix: false }), 0);
|
||||
});
|
||||
|
||||
it("should count a prefix unit against every completion", async function () {
|
||||
assert.equal(await Zotero.Lexical.getDocumentFrequency(
|
||||
{ type: 'word', text: 'lexow', prefix: true }
|
||||
), 2);
|
||||
});
|
||||
|
||||
it("should count a phrase by adjacent occurrence in order", async function () {
|
||||
let df = unit => Zotero.Lexical.getDocumentFrequency(unit);
|
||||
assert.equal(await df(
|
||||
{ type: 'phrase', text: 'lexowl migration', tokens: ['lexowl', 'migration'] }
|
||||
), 1);
|
||||
assert.equal(await df(
|
||||
{ type: 'phrase', text: 'migration lexowl', tokens: ['migration', 'lexowl'] }
|
||||
), 0);
|
||||
});
|
||||
|
||||
it("should count CJK units against the 2-gram index", async function () {
|
||||
await addDoc(4, "lexchinese document", '疫情 情控 控制');
|
||||
let df = unit => Zotero.Lexical.getDocumentFrequency(unit);
|
||||
assert.equal(await df(
|
||||
{ type: 'cjk', text: '疫情控制', bigrams: '疫情 情控 控制' }
|
||||
), 1);
|
||||
assert.equal(await df({ type: 'cjk', text: '疫情', bigrams: '疫情' }), 1);
|
||||
// Not adjacent in the document
|
||||
assert.equal(await df({ type: 'cjk', text: '控疫', bigrams: '控疫' }), 0);
|
||||
// A single character approximates by the bigrams it starts...
|
||||
assert.equal(await df({ type: 'cjk', text: '疫', bigrams: null }), 1);
|
||||
// ...so one that only ever ends a run undercounts -- the
|
||||
// documented blind spot of the approximation
|
||||
assert.equal(await df({ type: 'cjk', text: '制', bigrams: null }), 0);
|
||||
});
|
||||
|
||||
it("should measure corpus size from the index state", async function () {
|
||||
let before = await Zotero.Lexical.getCorpusSize();
|
||||
await addDoc(5, "lexcorpus size probe");
|
||||
assert.equal(await Zotero.Lexical.getCorpusSize(), before + 1);
|
||||
});
|
||||
});
|
||||
|
||||
// A unit from an analyzed list by its text, for asserting on one term
|
||||
function unitByText(units, text) {
|
||||
return units.find(unit => unit.text == text);
|
||||
}
|
||||
|
||||
describe("combined statistics", function () {
|
||||
const BASE_ROWID = 920000000;
|
||||
var inserted = [];
|
||||
|
||||
async function addContentDoc(id, text) {
|
||||
let rowid = BASE_ROWID + id;
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)",
|
||||
[rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || '']
|
||||
);
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)",
|
||||
[rowid]
|
||||
);
|
||||
inserted.push(rowid);
|
||||
return rowid;
|
||||
}
|
||||
|
||||
after(async function () {
|
||||
for (let rowid of inserted) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid);
|
||||
}
|
||||
});
|
||||
|
||||
it("should count document frequency across content and item text", async function () {
|
||||
await addContentDoc(1, "lexcross appears in a document");
|
||||
await createDataObject('item', { title: 'Lexcross appears in a title' });
|
||||
assert.equal(await Zotero.Lexical.getDocumentFrequency(
|
||||
{ type: 'word', text: 'lexcross', prefix: false }
|
||||
), 2);
|
||||
});
|
||||
|
||||
it("should count items toward the corpus size", async function () {
|
||||
let before = await Zotero.Lexical.getCorpusSize();
|
||||
await createDataObject('item', { title: 'Lexcorpus member item' });
|
||||
assert.equal(await Zotero.Lexical.getCorpusSize(), before + 1);
|
||||
});
|
||||
|
||||
it("should keep a term cheap for an annotation when the library is full of it", async function () {
|
||||
this.timeout(60000);
|
||||
// The term saturates the content corpus...
|
||||
let corpusSize = await Zotero.Lexical.getCorpusSize();
|
||||
for (let i = 0; i < corpusSize + 10; i++) {
|
||||
await addContentDoc(100 + i, `lexeverywhere document ${i}`);
|
||||
}
|
||||
// ...and appears once in an annotation
|
||||
let item = await createDataObject('item');
|
||||
let attachment = await importPDFAttachment(item);
|
||||
await createAnnotation('highlight', attachment,
|
||||
{ comment: 'lexeverywhere in a comment' });
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexeverywhere lexveryrare ");
|
||||
let everywhere = unitByText(units, 'lexeverywhere');
|
||||
let rare = unitByText(units, 'lexveryrare');
|
||||
// Rarity is a property of the library, not of where the match
|
||||
// lands: the saturated term is worth little anywhere, including
|
||||
// in the annotation, and gets cut from retrieval
|
||||
assert.isBelow(everywhere.weight, rare.weight * 0.2);
|
||||
assert.isFalse(everywhere.informative);
|
||||
assert.isTrue(rare.informative);
|
||||
});
|
||||
|
||||
it("should keep the best unit informative in an all-common query", async function () {
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Lexical, 'getCorpusSize').resolves(1000),
|
||||
sinon.stub(Zotero.Lexical, 'getDocumentFrequency').resolves(950)
|
||||
];
|
||||
try {
|
||||
let units = await Zotero.Lexical.analyzeQuery("common words only ");
|
||||
assert.isTrue(units.every(unit => unit.informative));
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("#matchContent()", function () {
|
||||
const BASE_ROWID = 930000000;
|
||||
var inserted = [];
|
||||
|
||||
async function addContentDoc(id, text, cjkBigrams) {
|
||||
let rowid = BASE_ROWID + id;
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)",
|
||||
[rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || '']
|
||||
);
|
||||
if (cjkBigrams) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContentCJK (rowid, text) VALUES (?, ?)",
|
||||
[rowid, cjkBigrams]
|
||||
);
|
||||
}
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)",
|
||||
[rowid]
|
||||
);
|
||||
inserted.push(rowid);
|
||||
return rowid;
|
||||
}
|
||||
|
||||
after(async function () {
|
||||
for (let rowid of inserted) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid);
|
||||
}
|
||||
});
|
||||
|
||||
it("should report presence for units a document contains", async function () {
|
||||
let doc = await addContentDoc(1, "lexprobe migration study text");
|
||||
let other = await addContentDoc(2, "entirely unrelated content");
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexprobe missingword ");
|
||||
let strengths = await Zotero.Lexical.matchContent(units, [doc, other]);
|
||||
assert.equal(strengths.get(doc).get(unitByText(units, 'lexprobe')), 1);
|
||||
assert.isFalse(strengths.get(doc).has(unitByText(units, 'missingword')));
|
||||
assert.isFalse(strengths.has(other));
|
||||
});
|
||||
|
||||
it("should only report the requested candidates", async function () {
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexprobe ");
|
||||
let strengths = await Zotero.Lexical.matchContent(units, [BASE_ROWID + 2]);
|
||||
assert.isFalse(strengths.has(BASE_ROWID + 1));
|
||||
});
|
||||
|
||||
it("should match a prefix unit against its completions", async function () {
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexprob");
|
||||
assert.isTrue(units[0].prefix);
|
||||
let strengths = await Zotero.Lexical.matchContent(units, [BASE_ROWID + 1]);
|
||||
assert.equal(strengths.get(BASE_ROWID + 1).get(units[0]), 1);
|
||||
});
|
||||
|
||||
it("should match CJK units against the 2-gram index", async function () {
|
||||
let doc = await addContentDoc(3, "lexcjk carrier", '疫情 情控 控制');
|
||||
let units = await Zotero.Lexical.analyzeQuery("疫情控制 ");
|
||||
let strengths = await Zotero.Lexical.matchContent(units, [doc]);
|
||||
assert.equal(strengths.get(doc).get(units[0]), 1);
|
||||
});
|
||||
|
||||
it("should verify a phrase against the document's stored text", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item');
|
||||
let attachment = await importPDFAttachment(item);
|
||||
await Zotero.Fulltext.indexItems([attachment.id]);
|
||||
|
||||
// "easy-to-use" in the document: hyphens and spaces separate the
|
||||
// same words, so the phrase verifies
|
||||
let units = await Zotero.Lexical.analyzeQuery('"easy to use" probe');
|
||||
let phrase = units.find(unit => unit.type == 'phrase');
|
||||
let strengths = await Zotero.Lexical.matchContent([phrase], [attachment.id]);
|
||||
assert.equal(strengths.get(attachment.id).get(phrase), 1);
|
||||
|
||||
// "collect, organize" in the document: adjacent for the index,
|
||||
// but the comma has to match literally, so verification rejects it
|
||||
units = await Zotero.Lexical.analyzeQuery('"collect organize" probe');
|
||||
phrase = units.find(unit => unit.type == 'phrase');
|
||||
strengths = await Zotero.Lexical.matchContent([phrase], [attachment.id]);
|
||||
assert.isFalse(strengths.has(attachment.id));
|
||||
});
|
||||
});
|
||||
|
||||
describe("#matchFields()", function () {
|
||||
it("should report title and abstract matches separately, at full strength", async function () {
|
||||
let item = await createDataObject('item', { title: 'Owl migration atlas' });
|
||||
item.setField('abstractNote', 'Statistical methods for tracking studies');
|
||||
await item.saveTx();
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("migration statistical ");
|
||||
let result = await Zotero.Lexical.matchFields(units, [item.id]);
|
||||
assert.equal(result.title.get(item.id).get(unitByText(units, 'migration')), 1);
|
||||
assert.isFalse(result.title.get(item.id).has(unitByText(units, 'statistical')));
|
||||
assert.equal(result.abstract.get(item.id).get(unitByText(units, 'statistical')), 1);
|
||||
assert.isFalse(result.abstract.get(item.id).has(unitByText(units, 'migration')));
|
||||
});
|
||||
|
||||
it("should match whole words, with the trailing prefix matching completions", async function () {
|
||||
let item = await createDataObject('item', { title: 'Rainfall patterns' });
|
||||
let units = await Zotero.Lexical.analyzeQuery("fall ");
|
||||
let result = await Zotero.Lexical.matchFields(units, [item.id]);
|
||||
assert.isFalse(result.title.has(item.id));
|
||||
units = await Zotero.Lexical.analyzeQuery("rain");
|
||||
result = await Zotero.Lexical.matchFields(units, [item.id]);
|
||||
assert.equal(result.title.get(item.id).get(units[0]), 1);
|
||||
});
|
||||
|
||||
it("should match titles diacritic-insensitively", async function () {
|
||||
let item = await createDataObject('item', { title: 'Müller précis studies' });
|
||||
let units = await Zotero.Lexical.analyzeQuery("muller precis ");
|
||||
let result = await Zotero.Lexical.matchFields(units, [item.id]);
|
||||
assert.equal(result.title.get(item.id).get(unitByText(units, 'muller')), 1);
|
||||
assert.equal(result.title.get(item.id).get(unitByText(units, 'precis')), 1);
|
||||
});
|
||||
|
||||
it("should verify a phrase against the stored title", async function () {
|
||||
// Adjacent for the index either way; only the hyphen reads as a
|
||||
// space literally
|
||||
let hyphenated = await createDataObject('item',
|
||||
{ title: 'Lexuno-lexdos analysis' });
|
||||
let punctuated = await createDataObject('item',
|
||||
{ title: 'Lexuno. Lexdos analysis' });
|
||||
let units = await Zotero.Lexical.analyzeQuery('"lexuno lexdos" probe');
|
||||
let phrase = units.find(unit => unit.type == 'phrase');
|
||||
let result = await Zotero.Lexical.matchFields(
|
||||
[phrase], [hyphenated.id, punctuated.id]);
|
||||
assert.equal(result.title.get(hyphenated.id).get(phrase), 1);
|
||||
assert.isFalse(result.title.has(punctuated.id));
|
||||
});
|
||||
});
|
||||
|
||||
describe("#matchNotes()", function () {
|
||||
it("should score a note that returns to a term above one passing mention", async function () {
|
||||
let focused = new Zotero.Item('note');
|
||||
focused.setNote('<p>Lexowls hunt at night. Lexowls migrate. Lexowls return.</p>');
|
||||
await focused.saveTx();
|
||||
let passing = new Zotero.Item('note');
|
||||
let filler = Array.from({ length: 200 }, (x, i) => `note${i}`).join(' ');
|
||||
passing.setNote(`<p>One mention of lexowls. ${filler}</p>`);
|
||||
await passing.saveTx();
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexowls ");
|
||||
let strengths = await Zotero.Lexical.matchNotes(units, [focused.id, passing.id]);
|
||||
let unit = units[0];
|
||||
assert.isAbove(strengths.get(focused.id).get(unit),
|
||||
strengths.get(passing.id).get(unit));
|
||||
assert.isAbove(strengths.get(passing.id).get(unit), 0);
|
||||
assert.isBelow(strengths.get(focused.id).get(unit), 1);
|
||||
});
|
||||
|
||||
it("should match whole words only", async function () {
|
||||
let note = new Zotero.Item('note');
|
||||
note.setNote('<p>Heavy rainfall in the region</p>');
|
||||
await note.saveTx();
|
||||
let units = await Zotero.Lexical.analyzeQuery("fall ");
|
||||
let strengths = await Zotero.Lexical.matchNotes(units, [note.id]);
|
||||
assert.isFalse(strengths.has(note.id));
|
||||
});
|
||||
|
||||
it("should match an indexed note through the index", async function () {
|
||||
let note = new Zotero.Item('note');
|
||||
note.setNote('<p>Lexindexed observations here today</p>');
|
||||
await note.saveTx();
|
||||
await Zotero.FullText.processNoteIndexQueue();
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexindexed ");
|
||||
let strengths = await Zotero.Lexical.matchNotes(units, [note.id]);
|
||||
assert.isAbove(strengths.get(note.id).get(units[0]), 0);
|
||||
});
|
||||
|
||||
it("should match a just-edited note by its current text", async function () {
|
||||
let note = new Zotero.Item('note');
|
||||
note.setNote('<p>lexoldword only here</p>');
|
||||
await note.saveTx();
|
||||
await Zotero.FullText.processNoteIndexQueue();
|
||||
note.setNote('<p>lexnewword replaces it</p>');
|
||||
await note.saveTx();
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexoldword lexnewword ");
|
||||
let strengths = await Zotero.Lexical.matchNotes(units, [note.id]);
|
||||
assert.isTrue(strengths.get(note.id).has(unitByText(units, 'lexnewword')));
|
||||
assert.isFalse(strengths.get(note.id).has(unitByText(units, 'lexoldword')));
|
||||
});
|
||||
|
||||
it("should match a note the index doesn't have yet", async function () {
|
||||
let note = new Zotero.Item('note');
|
||||
note.setNote('<p>lexunindexed content waiting for backfill</p>');
|
||||
await note.saveTx();
|
||||
// Simulate a note that predates the index
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextNoteIndexState WHERE itemID=?", note.id);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextItemText WHERE rowid=?", note.id);
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexunindexed ");
|
||||
let strengths = await Zotero.Lexical.matchNotes(units, [note.id]);
|
||||
assert.isAbove(strengths.get(note.id).get(units[0]), 0);
|
||||
});
|
||||
|
||||
it("should count a phrase literally", async function () {
|
||||
let hyphenated = new Zotero.Item('note');
|
||||
hyphenated.setNote('<p>The lexunited-states policy record</p>');
|
||||
await hyphenated.saveTx();
|
||||
let punctuated = new Zotero.Item('note');
|
||||
punctuated.setNote('<p>The lexunited. States policy record</p>');
|
||||
await punctuated.saveTx();
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery('"lexunited states" probe');
|
||||
let phrase = units.find(unit => unit.type == 'phrase');
|
||||
let strengths = await Zotero.Lexical.matchNotes(
|
||||
[phrase], [hyphenated.id, punctuated.id]);
|
||||
assert.isAbove(strengths.get(hyphenated.id).get(phrase), 0);
|
||||
assert.isFalse(strengths.has(punctuated.id));
|
||||
});
|
||||
});
|
||||
|
||||
describe("#scoreItemIDs()", function () {
|
||||
const BASE_ROWID = 940000000;
|
||||
var inserted = [];
|
||||
|
||||
async function addContentDoc(id, text) {
|
||||
let rowid = BASE_ROWID + id;
|
||||
await Zotero.DB.queryAsync(
|
||||
"INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)",
|
||||
[rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || '']
|
||||
);
|
||||
await Zotero.DB.queryAsync(
|
||||
"REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)",
|
||||
[rowid]
|
||||
);
|
||||
inserted.push(rowid);
|
||||
return rowid;
|
||||
}
|
||||
|
||||
after(async function () {
|
||||
for (let rowid of inserted) {
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid);
|
||||
await Zotero.DB.queryAsync(
|
||||
"DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid);
|
||||
}
|
||||
});
|
||||
|
||||
it("should rank coverage over partial matches, wherever they land", async function () {
|
||||
// The walkthrough corpus: full coverage in a title, partial
|
||||
// coverage in a title, partial coverage in a document, and noise
|
||||
// containing none of the query's words. (Whether "in"/"the" count
|
||||
// as informative depends on the corpus -- in this test library
|
||||
// they can be rare enough to -- so the full title carries every
|
||||
// query word.)
|
||||
let full = await createDataObject('item',
|
||||
{ title: 'Lexsowl lexsmigration in the lexsunited lexsstates' });
|
||||
let census = await createDataObject('item',
|
||||
{ title: 'Lexsunited lexsstates census records' });
|
||||
let norway = await addContentDoc(1,
|
||||
'lexsowl lexsmigration routes across norway seasons');
|
||||
let noise = await addContentDoc(2, 'lexsother archive entry');
|
||||
|
||||
let scores = await Zotero.Lexical.scoreItemIDs(
|
||||
'lexsowl lexsmigration in the lexsunited lexsstates ',
|
||||
[full.id, census.id, norway, noise]
|
||||
);
|
||||
// Full coverage at full strength in the best source is the
|
||||
// ceiling itself
|
||||
assert.approximately(scores.get(full.id), 1, 0.001);
|
||||
// Everything with an informative match is in, ranked under it...
|
||||
assert.isAbove(scores.get(full.id), scores.get(census.id));
|
||||
assert.isAbove(scores.get(census.id), 0);
|
||||
assert.isAbove(scores.get(norway), 0);
|
||||
// ...and matching nothing the query asked for is no match at all
|
||||
assert.isFalse(scores.has(noise));
|
||||
});
|
||||
|
||||
it("should count the same unit once, at its best source", async function () {
|
||||
let item = await createDataObject('item', { title: 'Lexsboth appears here' });
|
||||
item.setField('abstractNote', 'Lexsboth appears in the abstract too');
|
||||
await item.saveTx();
|
||||
let scores = await Zotero.Lexical.scoreItemIDs('lexsboth ', [item.id]);
|
||||
// Title and abstract both match; max, not sum -- a score of
|
||||
// exactly 1 proves no double counting
|
||||
assert.approximately(scores.get(item.id), 1, 0.001);
|
||||
});
|
||||
|
||||
it("should grade document matches relative to each other", async function () {
|
||||
let filler = Array.from({ length: 300 }, (x, i) => `lexsfill${i}`).join(' ');
|
||||
let buried = await addContentDoc(3, `lexsdeep ${filler}`);
|
||||
let focused = await addContentDoc(4, 'lexsdeep lexsdeep lexsdeep summary');
|
||||
let scores = await Zotero.Lexical.scoreItemIDs(
|
||||
'lexsdeep ', [buried, focused]);
|
||||
// The strongest document anchors the unit's full strength: one
|
||||
// unit at content boost against a title-boosted ceiling
|
||||
assert.approximately(scores.get(focused), 0.5, 0.001);
|
||||
if (scores.has(buried)) {
|
||||
assert.isBelow(scores.get(buried), scores.get(focused));
|
||||
}
|
||||
});
|
||||
|
||||
it("should drop scores below the floor", async function () {
|
||||
let stubs = [
|
||||
sinon.stub(Zotero.Lexical, 'analyzeQuery').resolves([
|
||||
{ type: 'word', text: 'big', prefix: false, df: 1, weight: 5, informative: true },
|
||||
{ type: 'word', text: 'small', prefix: false, df: 1, weight: 1, informative: true }
|
||||
]),
|
||||
sinon.stub(Zotero.Lexical, 'matchContent').callsFake(
|
||||
async (units, itemIDs) => new Map([
|
||||
// One item barely touches the small unit; another
|
||||
// matches the big one outright
|
||||
[itemIDs[0], new Map([[units[1], 0.1]])],
|
||||
[itemIDs[1], new Map([[units[0], 1]])]
|
||||
])
|
||||
),
|
||||
sinon.stub(Zotero.Lexical, 'matchFields').resolves(
|
||||
{ title: new Map(), abstract: new Map() }),
|
||||
sinon.stub(Zotero.Lexical, 'matchNotes').resolves(new Map()),
|
||||
sinon.stub(Zotero.Lexical, 'matchAnnotations').resolves(new Map())
|
||||
];
|
||||
try {
|
||||
let scores = await Zotero.Lexical.scoreItemIDs('anything', [1, 2]);
|
||||
// (1 * 0.1) / 12 is under the floor; (5 * 1) / 12 is well over
|
||||
assert.isFalse(scores.has(1));
|
||||
assert.isAbove(scores.get(2), 0.4);
|
||||
}
|
||||
finally {
|
||||
stubs.forEach(stub => stub.restore());
|
||||
}
|
||||
});
|
||||
|
||||
it("should return nothing for a query with no units", async function () {
|
||||
assert.equal((await Zotero.Lexical.scoreItemIDs('', [1])).size, 0);
|
||||
assert.equal((await Zotero.Lexical.scoreItemIDs('!!! ...', [1])).size, 0);
|
||||
});
|
||||
|
||||
it("should abandon scoring when cancelled", async function () {
|
||||
let item = await createDataObject('item', { title: 'Lexscancel target' });
|
||||
let e = await getPromiseError(Zotero.Lexical.scoreItemIDs(
|
||||
'lexscancel ', [item.id], { shouldCancel: () => true }));
|
||||
assert.instanceOf(e, Zotero.Lexical.ScoringCancelledError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("#matchAnnotations()", function () {
|
||||
it("should match an annotation's passage and comment at full strength", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item');
|
||||
let attachment = await importPDFAttachment(item);
|
||||
let annotation = await createAnnotation('highlight', attachment,
|
||||
{ comment: 'lexanno methodology worry' });
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("lexanno missingword ");
|
||||
let strengths = await Zotero.Lexical.matchAnnotations(
|
||||
units, [annotation.id, attachment.id]);
|
||||
assert.equal(strengths.get(annotation.id).get(unitByText(units, 'lexanno')), 1);
|
||||
assert.isFalse(strengths.get(annotation.id).has(unitByText(units, 'missingword')));
|
||||
assert.isFalse(strengths.has(attachment.id));
|
||||
});
|
||||
|
||||
it("should match annotations diacritic-insensitively", async function () {
|
||||
this.timeout(60000);
|
||||
let item = await createDataObject('item');
|
||||
let attachment = await importPDFAttachment(item);
|
||||
let annotation = await createAnnotation('highlight', attachment,
|
||||
{ comment: 'Müller réviewed this lexdiacritic passage' });
|
||||
|
||||
let units = await Zotero.Lexical.analyzeQuery("muller reviewed lexdiacritic ");
|
||||
let strengths = await Zotero.Lexical.matchAnnotations(units, [annotation.id]);
|
||||
for (let unit of units) {
|
||||
assert.equal(strengths.get(annotation.id).get(unit), 1);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue