diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index f85f301d53..74c959ebdc 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -2819,6 +2819,94 @@ Zotero.Fulltext = Zotero.FullText = new function () { }; + /** + * IDs of the given notes whose note-index entries can't be relied on to reflect their + * current text: notes edited since their last index update (see flagNoteStale()) and notes + * not in the index at the current format version (e.g., mid-backfill). A caller matching + * against the index should read these notes' current text instead (see + * getNoteSearchTexts()). + * + * @param {Integer[]} itemIDs + * @return {Promise} + */ + this.getStaleOrUnindexedNoteIDs = async function (itemIDs) { + let result = []; + let chunkSize = 500; + for (let i = 0; i < itemIDs.length; i += chunkSize) { + let chunk = itemIDs.slice(i, i + chunkSize); + result.push(...await Zotero.DB.columnQueryAsync( + "SELECT N.itemID FROM itemNotes N " + + "LEFT JOIN ftindex.fulltextNoteIndexState S USING (itemID) " + + "WHERE N.itemID IN (" + chunk.map(() => '?').join(',') + ") " + + "AND (S.itemID IS NULL OR S.version} - itemID -> text + */ + this.getNoteSearchTexts = async function (itemIDs) { + let texts = new Map(); + let chunkSize = 500; + for (let i = 0; i < itemIDs.length; i += chunkSize) { + let chunk = itemIDs.slice(i, i + chunkSize); + let placeholders = chunk.map(() => '?').join(','); + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, text FROM ftindex.noteText WHERE itemID IN (" + placeholders + ")", + chunk + ); + for (let row of rows) { + texts.set(row.itemID, row.text); + } + // A note edited since its last index update still holds its + // pre-edit text in noteText -- its current text is what counts + let staleIDs = new Set(await Zotero.DB.columnQueryAsync( + "SELECT itemID FROM ftindex.fulltextNoteIndexState " + + "WHERE version=0 AND itemID IN (" + placeholders + ")", + chunk + )); + // Stale notes not seen since their edit (e.g., flagged in a + // previous session) and notes with no index entry at all are both + // extracted from the stored note + let fetchIDs = chunk.filter((id) => { + return staleIDs.has(id) ? !_staleNoteText.has(id) : !texts.has(id); + }); + if (fetchIDs.length) { + let noteRows = await Zotero.DB.queryAsync( + "SELECT itemID, note FROM itemNotes WHERE itemID IN (" + + fetchIDs.map(() => '?').join(',') + ")", + fetchIDs + ); + for (let row of noteRows) { + let text = _normalizeNoteText(row.note); + texts.set(row.itemID, text); + if (staleIDs.has(row.itemID)) { + _staleNoteText.set(row.itemID, text); + } + } + } + for (let id of staleIDs) { + if (_staleNoteText.has(id)) { + texts.set(id, _staleNoteText.get(id)); + } + } + } + return texts; + }; + + /** * Return the ids of attachment items in the given library whose full-text content matches * `searchText` (see getWordMatchClause for the matching semantics). This is the non-regexp diff --git a/chrome/content/zotero/xpcom/lexical.js b/chrome/content/zotero/xpcom/lexical.js new file mode 100644 index 0000000000..213e9d5224 --- /dev/null +++ b/chrome/content/zotero/xpcom/lexical.js @@ -0,0 +1,785 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2026 Corporation for Digital Scholarship + Vienna, Virginia, USA + https://www.zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * Zotero.Lexical -- ranked lexical search over the library's own text. + * + * Scores how well a text answers a query rather than whether it contains + * every word of it: any term can match, and a score accumulates the + * evidence, so a document about owl migration in Norway still scores for + * "owl migration in the united states" -- below the documents that cover + * all of it. + * + * parseQuery() breaks a query into scoring units (words, quoted phrases, + * CJK runs), and analyzeQuery() weighs each unit by how rare it is in the + * user's own corpus, so that in "fall of communism", "communism" is what + * mostly decides a score, "fall" counts a little, and "of" barely at all -- + * no stoplist, nothing curated by hand. + * + * The statistics behind the weights are document frequencies counted across + * both of ftindex's word-level corpora (see Zotero.FullText): attachment + * content and item text (titles, abstracts, notes, annotations). One count + * per term over everything, so a term's rarity is a property of the library + * -- a word filling every document stays cheap when it turns up in an + * annotation, and a library with few attachments still measures rarity from + * its items' own text. + * + * The match side reports which of a set of items contain which units: + * matchContent() asks the attachment content index, and matchFields(), + * matchNotes(), and matchAnnotations() ask the item-text index's columns. A + * match is presence -- strength 1 -- where text is short enough that + * containing a unit says everything (titles, abstracts, annotations); + * notes and documents, whose lengths vary too much for that, are graded by + * saturated, length-normalized term frequency. Quoted phrases are matched + * literally everywhere: the indexes only prove a phrase's words adjacent, + * so phrase matches are verified against the stored text. + * + * scoreItemIDs() assembles the score: each unit contributes its weight + * times the best evidence for it across an item's sources, summed and + * normalized against the query's ceiling -- 1 is a full-strength match on + * everything the query asked -- with a floor below which an item isn't a + * match at all. + */ +Zotero.Lexical = new function () { + // CJK scripts (Han/Hiragana/Katakana/Hangul), the same set the full-text + // index routes to its 2-gram tables (see fulltext.js): the word tokenizer + // sees an unspaced CJK run as a single token, so runs are matched by + // their bigrams instead + const CJK_CLASS = '\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}'; + // The tokens of a normalized query part, in text order: a CJK run, or a + // word token as the index's unicode61 tokenizer produces them -- a run of + // letters and digits, with everything else a separator. The lookahead + // keeps CJK characters (which are also \p{L}) out of word tokens, so + // 'covid疫情' splits into a word and a run rather than reading as one word. + const TOKEN_RE = new RegExp( + `(?[${CJK_CLASS}]+)|(?(?:(?![${CJK_CLASS}])[\\p{L}\\p{N}])+)`, + 'gu' + ); + // BM25's term-frequency shape: K1 sets how quickly repetition saturates, + // B how much a long text discounts each occurrence (see + // _saturatedStrength()) + const K1 = 1.2; + const B = 0.75; + // A unit is informative -- worth retrieving by -- when it carries at + // least this share of the query's best unit's weight. Relative rather + // than absolute, so "of" next to "communism" is dropped while a query of + // nothing but common words keeps its best word and still returns results. + const INFORMATIVE_WEIGHT_FRACTION = 0.2; + // What a full-strength match in each source is worth relative to a + // full-strength content match: a title names the work, an abstract + // summarizes it, everything else speaks with equal voice + const SOURCE_BOOSTS = { + title: 2, + abstract: 1.3, + content: 1, + note: 1, + annotation: 1 + }; + // Normalized scores below this aren't matches and aren't returned: with + // scores measured against the query's ceiling, this is the share of what + // the query asked for that an item has to show. Provisional until tuned + // against a real library. + const SCORE_FLOOR = 0.05; + + /** + * Thrown when scoring is abandoned via the shouldCancel callback -- e.g. + * because a newer query superseded the one being scored + */ + this.ScoringCancelledError = class extends Error { + constructor(message = 'Scoring cancelled') { + super(message); + this.name = 'LexicalScoringCancelledError'; + } + }; + + /** + * Parse a query into scoring units. + * + * A unit is the thing that matches (or doesn't) in one text and earns + * its weight toward a score: + * - a word: { type: 'word', text, prefix }. The query's trailing word is + * flagged `prefix` while it's still being typed (no space or quote + * after it yet), so it matches its completions. + * - a quoted phrase: { type: 'phrase', text, tokens } -- multiple words + * matched adjacently, as typed. + * - a CJK run: { type: 'cjk', text, bigrams } -- matched contiguously via + * the 2-gram index, so quoting adds nothing it doesn't already have. + * `bigrams` is null for a single character, which has none. + * A part mixing scripts splits into word and run units, and a quoted + * part mixing scripts splits the same way, since neither index side + * covers it whole. + * + * A repeated term counts once: repeating a word in the query isn't more + * evidence about the texts it matches. When the same word appears both + * mid-query and as the trailing prefix, the exact form wins. + * + * @param {String} queryText + * @return {Object[]} + */ + this.parseQuery = function (queryText) { + if (!queryText) { + return []; + } + // Mid-word means the query ends in a word character; a space, a + // closing quote, or punctuation after the word means it's finished + let endsMidWord = /[\p{L}\p{N}]$/u.test(queryText); + let parts = Zotero.SearchConditions.parseSearchString(queryText); + let units = []; + for (let i = 0; i < parts.length; i++) { + let part = parts[i]; + let fromLastPart = i == parts.length - 1; + let normalized = Zotero.Utilities.Internal.normalizeForSearch(part.text); + if (!normalized) { + continue; + } + let matches = [...normalized.matchAll(TOKEN_RE)]; + if (!matches.length) { + continue; + } + let words = matches.filter(m => m.groups.word).map(m => m.groups.word); + let hasCJK = matches.some(m => m.groups.cjk); + if (part.inQuotes && words.length > 1 && !hasCJK) { + units.push({ + type: 'phrase', + text: words.join(' '), + tokens: words, + fromLastPart, + quoted: true + }); + continue; + } + for (let match of matches) { + if (match.groups.cjk) { + units.push({ + type: 'cjk', + text: match.groups.cjk, + bigrams: _getBigrams(match.groups.cjk), + fromLastPart, + quoted: part.inQuotes + }); + } + else { + units.push({ + type: 'word', + text: match.groups.word, + prefix: false, + fromLastPart, + quoted: part.inQuotes + }); + } + } + } + // The trailing word of an unquoted query is the one still being + // typed. A quoted trailing part is exact by declaration. + if (units.length && endsMidWord) { + let last = units[units.length - 1]; + if (last.type == 'word' && last.fromLastPart && !last.quoted) { + last.prefix = true; + } + } + let deduped = new Map(); + for (let unit of units) { + delete unit.fromLastPart; + delete unit.quoted; + let key = unit.type + '\n' + unit.text; + let existing = deduped.get(key); + if (!existing || (existing.prefix && !unit.prefix)) { + deduped.set(key, unit); + } + } + return [...deduped.values()]; + }; + + /** + * The scoring units of a query (see parseQuery()), each weighted by how + * rare it is in the corpus. A score built from these accumulates the + * weights of the units a text matches, so matching "communism" moves a + * score far more than matching "fall". + * + * @param {String} queryText + * @return {Promise} - parseQuery()'s units, each with: + * df - documents in the corpus matching the unit + * weight - what a match on this unit contributes to a score + * informative - whether the unit carries enough of the query's + * weight to be worth retrieving by (see + * INFORMATIVE_WEIGHT_FRACTION); the best-weighted unit always is + */ + this.analyzeQuery = async function (queryText) { + let units = this.parseQuery(queryText); + if (!units.length) { + return units; + } + let corpusSize = await this.getCorpusSize(); + for (let unit of units) { + unit.df = await this.getDocumentFrequency(unit); + unit.weight = _idf(unit.df, corpusSize); + } + let maxWeight = Math.max(...units.map(unit => unit.weight)); + for (let unit of units) { + unit.informative = unit.weight >= maxWeight * INFORMATIVE_WEIGHT_FRACTION; + } + return units; + }; + + /** + * Number of documents the term statistics are measured against: every + * attachment, item, note, and annotation recorded in the full-text + * index's state tables -- including ones indexed with no text, which are + * real documents that happen to contain nothing. + * + * @return {Promise} + */ + this.getCorpusSize = async function () { + return (await Zotero.DB.valueQueryAsync( + "SELECT COUNT(*) FROM ftindex.fulltextIndexState" + )) + (await Zotero.DB.valueQueryAsync( + "SELECT COUNT(*) FROM ftindex.fulltextItemTextState" + )) + (await Zotero.DB.valueQueryAsync( + "SELECT COUNT(*) FROM ftindex.fulltextNoteIndexState" + )); + }; + + /** + * How many documents match a unit: the `df` behind its weight, counted + * across both word-level corpora -- attachment content and item text -- + * whose ID spaces are disjoint, so the sum counts nothing twice. Counted + * the way the unit will be matched: a prefix word against every + * completion, a phrase by adjacent occurrence, a CJK run by its + * contiguous bigrams. + * + * A single-character CJK unit has no bigram of its own, so its count is + * approximated by prefix-matching the bigrams that start with it. That + * undercounts a character that only ends runs, which overstates its + * weight -- rarer reads as more important, the safe direction to be wrong. + * + * @param {Object} unit - A unit from parseQuery() + * @return {Promise} + */ + this.getDocumentFrequency = async function (unit) { + let clause = _getMatchClause(unit); + let df = 0; + for (let table of [clause.contentTable, clause.itemTextTable]) { + df += await Zotero.DB.valueQueryAsync( + "SELECT COUNT(*) FROM ftindex." + table + + " WHERE " + table + " MATCH ?", + [clause.match] + ); + } + return df; + }; + + /** + * Which of the given items' indexed attachment content contains which + * units, and how strongly relative to each other. For a one-unit + * expression the index's rank is a constant times BM25's saturated, + * length-normalized term frequency, so ranks compare documents exactly; + * the constant itself is unknowable, so the strongest match anchors 1 and + * the rest scale under it. A document that keeps returning to a term + * outranks one that mentions it once in passing -- but strengths are + * relative to the candidates at hand, so a lone weak match still reads + * as 1. + * + * A phrase's index match only proves its words adjacent -- the index + * ignores what separates them -- so phrase matches are verified against + * the stored document text and only literal occurrences count. + * + * Only items with a row in the content index (indexed attachments) can + * match; everything else is simply absent from the result. + * + * @param {Object[]} units - Units from parseQuery() + * @param {Integer[]} itemIDs + * @return {Promise} - itemID -> Map(unit -> strength) + */ + this.matchContent = async function (units, itemIDs) { + let strengths = new Map(); + if (!units.length || !itemIDs.length) { + return strengths; + } + for (let unit of units) { + let clause = _getMatchClause(unit); + let matched = await _probe(clause.contentTable, clause.match, itemIDs); + if (!matched.length) { + continue; + } + if (unit.type == 'phrase') { + let verified = new Set( + (await Zotero.FullText.findTextInItems( + matched.map(row => row.rowid), unit.text + )).map(x => x.id) + ); + matched = matched.filter(row => verified.has(row.rowid)); + if (!matched.length) { + continue; + } + } + // rank is negative, better more negative, so the best is the + // minimum and every ratio against it lands in (0, 1] + let best = Math.min(...matched.map(row => row.rank)); + for (let row of matched) { + _addStrength(strengths, row.rowid, unit, + best ? row.rank / best : 1); + } + } + return strengths; + }; + + /** + * Which of the given items' titles and abstracts contain which units, + * reported per column. A match is presence (strength 1): a title or + * abstract is short enough that containing a unit says everything. + * Titles cover the type-specific title fields (caseName, subject, + * nameOfAct) along with `title` itself. Phrase matches are verified + * literally against the stored field values. + * + * Items without a matching title or abstract are simply absent from the + * respective result. + * + * @param {Object[]} units - Units from parseQuery() + * @param {Integer[]} itemIDs + * @return {Promise} - { title: Map(itemID -> Map(unit -> + * strength)), abstract: Map(itemID -> Map(unit -> strength)) } + */ + this.matchFields = async function (units, itemIDs) { + let result = { title: new Map(), abstract: new Map() }; + if (!units.length || !itemIDs.length) { + return result; + } + for (let unit of units) { + let clause = _getMatchClause(unit); + for (let column of ['title', 'abstract']) { + let matched = (await _probe( + clause.itemTextTable, column + ':' + clause.match, itemIDs + )).map(row => row.rowid); + if (!matched.length) { + continue; + } + if (unit.type == 'phrase') { + matched = await _verifyFieldPhrase(unit, column, matched); + } + for (let itemID of matched) { + _addStrength(result[column], itemID, unit, 1); + } + } + } + return result; + }; + + /** + * Which of the given items' note text contains which units, and how + * strongly. Notes range from a line to a chapter, so strength is BM25's + * saturated, length-normalized term frequency (see _saturatedStrength()): + * a note that keeps returning to a term outranks one that mentions it + * once in passing. + * + * The index answers which notes are worth reading: text is fetched only + * for notes whose note column matches a unit, plus notes whose index + * entries can't be trusted -- edited since their last index update, or + * not indexed yet -- whose current text is always read. Phrases count + * only literally (see _countPhrase()). + * + * Items without matching note text are simply absent from the result. + * + * @param {Object[]} units - Units from parseQuery() + * @param {Integer[]} itemIDs + * @return {Promise} - itemID -> Map(unit -> strength) + */ + this.matchNotes = async function (units, itemIDs) { + let strengths = new Map(); + if (!units.length || !itemIDs.length) { + return strengths; + } + let fetchIDs = new Set(await Zotero.FullText.getStaleOrUnindexedNoteIDs(itemIDs)); + for (let unit of units) { + let clause = _getMatchClause(unit); + let matched = await _probe( + clause.itemTextTable, 'note:' + clause.match, itemIDs); + for (let row of matched) { + fetchIDs.add(row.rowid); + } + } + if (!fetchIDs.size) { + return strengths; + } + let texts = await Zotero.FullText.getNoteSearchTexts([...fetchIDs]); + if (!texts.size) { + return strengths; + } + // Length normalization needs the typical note length. The note index + // knows it; without one yet, the notes at hand stand in for the + // population. + let avgLength = await Zotero.DB.valueQueryAsync( + "SELECT AVG(LENGTH(text)) FROM ftindex.noteText" + ); + if (!avgLength) { + let lengths = [...texts.values()].map(text => text.length); + avgLength = (lengths.reduce((sum, length) => sum + length, 0) + / (lengths.length || 1)) || 1; + } + for (let [itemID, text] of texts) { + if (!text) { + continue; + } + let scan = _scanText(text); + for (let unit of units) { + let tf = _countUnit(unit, scan); + if (tf) { + _addStrength(strengths, itemID, unit, + _saturatedStrength(tf, text.length, avgLength)); + } + } + } + return strengths; + }; + + /** + * Which of the given items' annotation text -- the passage an annotation + * marks together with its comment -- contains which units. A match is + * presence (strength 1): an annotation is short enough that containing a + * unit says everything. Phrase matches are verified literally against + * the stored annotation text. + * + * Items without matching annotation text are simply absent from the + * result. + * + * @param {Object[]} units - Units from parseQuery() + * @param {Integer[]} itemIDs + * @return {Promise} - itemID -> Map(unit -> strength) + */ + this.matchAnnotations = async function (units, itemIDs) { + let strengths = new Map(); + if (!units.length || !itemIDs.length) { + return strengths; + } + for (let unit of units) { + let clause = _getMatchClause(unit); + let matched = (await _probe( + clause.itemTextTable, 'annotation:' + clause.match, itemIDs + )).map(row => row.rowid); + if (!matched.length) { + continue; + } + if (unit.type == 'phrase') { + matched = await _verifyAnnotationPhrase(unit, matched); + } + for (let itemID of matched) { + _addStrength(strengths, itemID, unit, 1); + } + } + return strengths; + }; + + /** + * Score a given set of items by how well their text answers a query. + * + * Any informative unit can match (see analyzeQuery()); each contributes + * its weight times the best evidence for it across the item's sources -- + * title, abstract, attachment content, notes, annotations, boosted per + * source (see SOURCE_BOOSTS) -- so the same word in two places counts + * once, at its strongest. The sum is normalized against the query's + * ceiling (every informative unit at full strength in the best-boosted + * source): 1 is a full-strength match on everything the query asked, + * partial coverage lands proportionally lower, dominated by the rare + * units. Items below SCORE_FLOOR aren't matches and aren't returned. + * + * Units too common to be informative play no part: they neither gate nor + * move a score. + * + * @param {String} queryText + * @param {Number[]} itemIDs - Candidate item IDs to score + * @param {Object} [options] + * @param {Function} [options.shouldCancel] - Checked between matching + * stages; return true to abandon scoring with a ScoringCancelledError + * @return {Promise} - itemID -> score (0-1, higher is better) + */ + this.scoreItemIDs = async function (queryText, itemIDs, { shouldCancel } = {}) { + let scores = new Map(); + if (!itemIDs.length) { + return scores; + } + let checkCancel = () => { + if (shouldCancel && shouldCancel()) { + throw new this.ScoringCancelledError(); + } + }; + let units = (await this.analyzeQuery(queryText)) + .filter(unit => unit.informative); + if (!units.length) { + return scores; + } + checkCancel(); + let content = await this.matchContent(units, itemIDs); + checkCancel(); + let fields = await this.matchFields(units, itemIDs); + checkCancel(); + let notes = await this.matchNotes(units, itemIDs); + checkCancel(); + let annotations = await this.matchAnnotations(units, itemIDs); + checkCancel(); + + // Best boosted evidence per item per unit, across all sources + let evidence = new Map(); + let sources = [ + [fields.title, SOURCE_BOOSTS.title], + [fields.abstract, SOURCE_BOOSTS.abstract], + [content, SOURCE_BOOSTS.content], + [notes, SOURCE_BOOSTS.note], + [annotations, SOURCE_BOOSTS.annotation] + ]; + for (let [strengths, boost] of sources) { + for (let [itemID, unitStrengths] of strengths) { + for (let [unit, strength] of unitStrengths) { + _addStrength(evidence, itemID, unit, boost * strength); + } + } + } + + let maxBoost = Math.max(...Object.values(SOURCE_BOOSTS)); + let ceiling = units.reduce((sum, unit) => sum + unit.weight, 0) * maxBoost; + if (!ceiling) { + return scores; + } + for (let [itemID, unitStrengths] of evidence) { + let raw = 0; + for (let [unit, strength] of unitStrengths) { + raw += unit.weight * strength; + } + let score = raw / ceiling; + if (score >= SCORE_FLOOR) { + scores.set(itemID, score); + } + } + return scores; + }; + + // The MATCH expression that finds a unit, with the content and item-text + // tables (word or CJK pair) it runs against. Unit text is all letters + // and digits (parseQuery tokenized it), so quoting it into an FTS phrase + // needs no escaping. + function _getMatchClause(unit) { + if (unit.type == 'cjk') { + return { + match: unit.bigrams + ? '"' + unit.bigrams + '"' + : '"' + unit.text + '"*', + contentTable: 'fulltextContentCJK', + itemTextTable: 'fulltextItemTextCJK' + }; + } + return { + match: unit.type == 'phrase' + ? '"' + unit.text + '"' + : '"' + unit.text + '"' + (unit.prefix ? '*' : ''), + contentTable: 'fulltextContent', + itemTextTable: 'fulltextItemText' + }; + } + + // A CJK run's overlapping 2-grams, joined with spaces -- built the same + // way the index builds them (see getCJKBigrams() in fulltext.js), which + // is what makes them match. Null for a single character, which has none. + function _getBigrams(run) { + if (run.length < 2) { + return null; + } + let bigrams = []; + for (let i = 0; i < run.length - 1; i++) { + bigrams.push(run.substr(i, 2)); + } + return bigrams.join(' '); + } + + // Smoothed BM25 inverse document frequency: + // + // ln(1 + (N - df + 0.5) / (df + 0.5)) + // + // the standard measure of how much information a term carries, and the + // whole term-importance mechanism: no stoplist, just counting. + // - df = 0 -- a term in no indexed document (a typo, or a word from text + // we haven't indexed) -- gets the query's maximum: unseen reads as + // rare reads as important + // - df = N -- a term in everything ("the") -- approaches zero + // - N = 0 -- nothing indexed to measure against -- gives every unit the + // same ln(2), so ranking degrades to term coverage + function _idf(df, corpusSize) { + // The single-CJK-character approximation and an index mid-write can + // disagree slightly with the row count + df = Math.max(0, Math.min(df, corpusSize)); + return Math.log(1 + (corpusSize - df + 0.5) / (df + 0.5)); + } + + // The requested candidates matching an FTS expression, probed in chunks + // (the bound-parameter limit). Each row carries the index's rank for the + // expression, for callers that grade matches against each other; callers + // that only need membership read the rowids. + async function _probe(table, match, itemIDs) { + let matched = []; + let chunkSize = 500; + for (let i = 0; i < itemIDs.length; i += chunkSize) { + let chunk = itemIDs.slice(i, i + chunkSize); + matched.push(...await Zotero.DB.queryAsync( + "SELECT rowid, rank FROM ftindex." + table + + " WHERE " + table + " MATCH ? " + + "AND rowid IN (" + chunk.map(() => '?').join(',') + ")", + [match, ...chunk] + )); + } + return matched; + } + + // Of the given items, those whose stored field text (title-family fields + // or the abstract) literally contains a phrase unit + async function _verifyFieldPhrase(unit, column, itemIDs) { + let fieldIDs = column == 'title' + ? [ + Zotero.ItemFields.getID('title'), + ...Zotero.ItemFields.getTypeFieldsFromBase('title') + ] + : [Zotero.ItemFields.getID('abstractNote')]; + let verified = []; + let chunkSize = 500; + for (let i = 0; i < itemIDs.length; i += chunkSize) { + let chunk = itemIDs.slice(i, i + chunkSize); + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, value FROM itemData " + + "JOIN itemDataValues USING (valueID) " + + "WHERE fieldID IN (" + fieldIDs.join(',') + ") " + + "AND itemID IN (" + chunk.map(() => '?').join(',') + ")", + chunk + ); + for (let row of rows) { + let normalized = Zotero.Utilities.Internal.normalizeForSearch(row.value); + if (normalized && _countPhrase(normalized, unit.text)) { + verified.push(row.itemID); + } + } + } + return verified; + } + + // Of the given annotations, those whose passage and comment literally + // contain a phrase unit + async function _verifyAnnotationPhrase(unit, itemIDs) { + let verified = []; + let chunkSize = 500; + for (let i = 0; i < itemIDs.length; i += chunkSize) { + let chunk = itemIDs.slice(i, i + chunkSize); + let rows = await Zotero.DB.queryAsync( + "SELECT itemID, text, comment FROM itemAnnotations " + + "WHERE itemID IN (" + chunk.map(() => '?').join(',') + ")", + chunk + ); + for (let row of rows) { + let normalized = Zotero.Utilities.Internal.normalizeForSearch( + [row.text, row.comment].filter(Boolean).join(' ') + ); + if (normalized && _countPhrase(normalized, unit.text)) { + verified.push(row.itemID); + } + } + } + return verified; + } + + // A text prepared for unit counting: its token stream (word tokens and + // CJK runs, in text order) and the normalized text itself, which is what + // CJK units and phrases match against + function _scanText(text) { + let tokens = []; + for (let match of text.matchAll(TOKEN_RE)) { + tokens.push(match.groups.cjk || match.groups.word); + } + return { text, tokens }; + } + + // Occurrences of a unit in a scanned text, counted the way the indexes + // match the unit: a word as a whole token (a prefix unit by its + // completions), a CJK run contiguously, a phrase literally (see + // _countPhrase()) + function _countUnit(unit, scan) { + if (unit.type == 'cjk') { + let count = 0; + let index = scan.text.indexOf(unit.text); + while (index != -1) { + count++; + index = scan.text.indexOf(unit.text, index + unit.text.length); + } + return count; + } + if (unit.type == 'phrase') { + return _countPhrase(scan.text, unit.text); + } + if (unit.prefix) { + return scan.tokens.filter(token => token.startsWith(unit.text)).length; + } + return scan.tokens.filter(token => token === unit.text).length; + } + + // Occurrences of a phrase in normalized text: literal, except that + // whitespace and hyphen runs separate the phrase's words interchangeably + // -- extraction layout and compound styling vary them -- matching the + // collapse the content verification applies (see findTextInString() in + // fulltext.js). Word boundaries hold at both ends, so a phrase never + // starts or ends inside a longer word. + function _countPhrase(text, phrase) { + let collapsed = text.replace(/[\s-]+/g, ' '); + let count = 0; + let index = collapsed.indexOf(phrase); + while (index != -1) { + let before = index > 0 ? collapsed[index - 1] : ''; + let after = collapsed[index + phrase.length] || ''; + if (!/[\p{L}\p{N}]/u.test(before) && !/[\p{L}\p{N}]/u.test(after)) { + count++; + } + index = collapsed.indexOf(phrase, index + 1); + } + return count; + } + + // BM25's saturated, length-normalized term frequency, mapped onto (0, 1): + // + // tf / (tf + K1 * (1 - B + B * length / avgLength)) + // + // One occurrence in an average-length text lands around 0.45, repetition + // approaches 1, and each occurrence counts for less in a longer text. + // Lengths are in characters on both sides of the ratio, which is all the + // ratio needs. + function _saturatedStrength(tf, length, avgLength) { + return tf / (tf + K1 * (1 - B + B * (length / avgLength))); + } + + // Record a unit's strength for an item, keeping the strongest when the + // same unit matches an item more than once (e.g., in two title fields) + function _addStrength(strengths, itemID, unit, strength) { + let unitStrengths = strengths.get(itemID); + if (!unitStrengths) { + unitStrengths = new Map(); + strengths.set(itemID, unitStrengths); + } + let previous = unitStrengths.get(unit); + if (previous === undefined || strength > previous) { + unitStrengths.set(unit, strength); + } + } +}; diff --git a/chrome/content/zotero/zotero.mjs b/chrome/content/zotero/zotero.mjs index 07cf391829..a2b501173b 100644 --- a/chrome/content/zotero/zotero.mjs +++ b/chrome/content/zotero/zotero.mjs @@ -109,6 +109,7 @@ const xpcomFilesLocal = [ 'httpIntegrationClient', 'id', 'integration', + 'lexical', 'locale', 'locateManager', 'mime', diff --git a/test/tests/lexicalTest.js b/test/tests/lexicalTest.js new file mode 100644 index 0000000000..c2151d2df2 --- /dev/null +++ b/test/tests/lexicalTest.js @@ -0,0 +1,679 @@ +"use strict"; + +describe("Zotero.Lexical", function () { + describe("#parseQuery()", function () { + it("should split a query into normalized word units", function () { + // Trailing space: the last word is finished, so nothing is a prefix + let units = Zotero.Lexical.parseQuery("OWL Migrátion "); + assert.deepEqual(units, [ + { type: 'word', text: 'owl', prefix: false }, + { type: 'word', text: 'migration', prefix: false } + ]); + }); + + it("should flag the trailing mid-word token as a prefix", function () { + let units = Zotero.Lexical.parseQuery("owl migr"); + assert.deepEqual(units, [ + { type: 'word', text: 'owl', prefix: false }, + { type: 'word', text: 'migr', prefix: true } + ]); + // Punctuation after the word means it's finished too + units = Zotero.Lexical.parseQuery("owl migr."); + assert.isFalse(units[1].prefix); + }); + + it("should keep a quoted multi-word part as one exact phrase", function () { + let units = Zotero.Lexical.parseQuery('"united states" owl'); + assert.deepEqual(units, [ + { type: 'phrase', text: 'united states', tokens: ['united', 'states'] }, + { type: 'word', text: 'owl', prefix: true } + ]); + }); + + it("should treat a quoted single word as an exact word", function () { + let units = Zotero.Lexical.parseQuery('"owl"'); + assert.deepEqual(units, [ + { type: 'word', text: 'owl', prefix: false } + ]); + }); + + it("should split a mixed-script part into word and CJK units", function () { + let units = Zotero.Lexical.parseQuery("covid疫情"); + assert.deepEqual(units, [ + { type: 'word', text: 'covid', prefix: false }, + { type: 'cjk', text: '疫情', bigrams: '疫情' } + ]); + }); + + it("should carry a CJK run as its overlapping bigrams", function () { + let units = Zotero.Lexical.parseQuery("疫情控制 "); + assert.deepEqual(units, [ + { type: 'cjk', text: '疫情控制', bigrams: '疫情 情控 控制' } + ]); + }); + + it("should carry a single CJK character with no bigrams", function () { + let units = Zotero.Lexical.parseQuery("疫 "); + assert.deepEqual(units, [ + { type: 'cjk', text: '疫', bigrams: null } + ]); + }); + + it("should count a repeated term once, preferring its exact form", function () { + // The trailing 'owl' would be a prefix, but the query already + // contains it as a finished word + let units = Zotero.Lexical.parseQuery("owl migration owl"); + assert.deepEqual(units, [ + { type: 'word', text: 'owl', prefix: false }, + { type: 'word', text: 'migration', prefix: false } + ]); + }); + + it("should return no units for text with nothing to match", function () { + assert.deepEqual(Zotero.Lexical.parseQuery(""), []); + assert.deepEqual(Zotero.Lexical.parseQuery(" "), []); + assert.deepEqual(Zotero.Lexical.parseQuery("!!! ..."), []); + }); + }); + + describe("#analyzeQuery()", function () { + var stubs = []; + + afterEach(function () { + stubs.forEach(stub => stub.restore()); + stubs = []; + }); + + function stubStatistics(corpusSize, dfByText) { + stubs.push(sinon.stub(Zotero.Lexical, 'getCorpusSize') + .resolves(corpusSize)); + stubs.push(sinon.stub(Zotero.Lexical, 'getDocumentFrequency') + .callsFake(async unit => dfByText[unit.text] ?? 0)); + } + + it("should weight terms by rarity, with ubiquitous ones near zero", async function () { + stubStatistics(1000, { fall: 600, of: 1000, communism: 5 }); + let units = await Zotero.Lexical.analyzeQuery("fall of communism "); + let byText = new Map(units.map(unit => [unit.text, unit])); + // "communism" is what decides this query's ranking + assert.isAbove(byText.get('communism').weight, byText.get('fall').weight); + assert.isAbove(byText.get('fall').weight, byText.get('of').weight); + // A term in every document carries almost nothing -- the stoplist, + // without a stoplist + assert.isBelow(byText.get('of').weight, 0.001); + // The documented formula: ln(1 + (N - df + 0.5) / (df + 0.5)) + assert.approximately( + byText.get('communism').weight, + Math.log(1 + (1000 - 5 + 0.5) / (5 + 0.5)), + 1e-12 + ); + }); + + it("should give a term in no document the query's maximum weight", async function () { + stubStatistics(1000, { communism: 5, zzunseen: 0 }); + let units = await Zotero.Lexical.analyzeQuery("communism zzunseen "); + let byText = new Map(units.map(unit => [unit.text, unit])); + assert.isAbove(byText.get('zzunseen').weight, byText.get('communism').weight); + }); + + it("should degrade to uniform weights with no corpus", async function () { + stubStatistics(0, {}); + let units = await Zotero.Lexical.analyzeQuery("owl migration routes "); + assert.lengthOf(units, 3); + for (let unit of units) { + assert.approximately(unit.weight, Math.log(2), 1e-12); + } + }); + + it("should look up statistics once per unique unit", async function () { + stubStatistics(1000, { owl: 3, migration: 40 }); + await Zotero.Lexical.analyzeQuery("owl owl migration owl "); + assert.equal(Zotero.Lexical.getDocumentFrequency.callCount, 2); + }); + }); + + describe("term statistics", function () { + // Fabricated corpus rows inserted straight into the real content + // index, with rowids no item can collide with + const BASE_ROWID = 900000000; + var inserted = []; + + async function addDoc(id, text, cjkBigrams) { + let rowid = BASE_ROWID + id; + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)", + [rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || ''] + ); + if (cjkBigrams) { + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContentCJK (rowid, text) VALUES (?, ?)", + [rowid, cjkBigrams] + ); + } + await Zotero.DB.queryAsync( + "REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)", + [rowid] + ); + inserted.push(rowid); + } + + after(async function () { + for (let rowid of inserted) { + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid); + } + }); + + it("should count the documents matching a word, not its occurrences", async function () { + await addDoc(1, "lexowl migration and lexowl wintering across the lexbaltic"); + await addDoc(2, "lexowl feeding grounds"); + await addDoc(3, "entirely unrelated text"); + let df = unit => Zotero.Lexical.getDocumentFrequency(unit); + assert.equal(await df({ type: 'word', text: 'lexowl', prefix: false }), 2); + assert.equal(await df({ type: 'word', text: 'lexbaltic', prefix: false }), 1); + assert.equal(await df({ type: 'word', text: 'lexmissing', prefix: false }), 0); + // Words match whole tokens, not substrings + assert.equal(await df({ type: 'word', text: 'lexow', prefix: false }), 0); + }); + + it("should count a prefix unit against every completion", async function () { + assert.equal(await Zotero.Lexical.getDocumentFrequency( + { type: 'word', text: 'lexow', prefix: true } + ), 2); + }); + + it("should count a phrase by adjacent occurrence in order", async function () { + let df = unit => Zotero.Lexical.getDocumentFrequency(unit); + assert.equal(await df( + { type: 'phrase', text: 'lexowl migration', tokens: ['lexowl', 'migration'] } + ), 1); + assert.equal(await df( + { type: 'phrase', text: 'migration lexowl', tokens: ['migration', 'lexowl'] } + ), 0); + }); + + it("should count CJK units against the 2-gram index", async function () { + await addDoc(4, "lexchinese document", '疫情 情控 控制'); + let df = unit => Zotero.Lexical.getDocumentFrequency(unit); + assert.equal(await df( + { type: 'cjk', text: '疫情控制', bigrams: '疫情 情控 控制' } + ), 1); + assert.equal(await df({ type: 'cjk', text: '疫情', bigrams: '疫情' }), 1); + // Not adjacent in the document + assert.equal(await df({ type: 'cjk', text: '控疫', bigrams: '控疫' }), 0); + // A single character approximates by the bigrams it starts... + assert.equal(await df({ type: 'cjk', text: '疫', bigrams: null }), 1); + // ...so one that only ever ends a run undercounts -- the + // documented blind spot of the approximation + assert.equal(await df({ type: 'cjk', text: '制', bigrams: null }), 0); + }); + + it("should measure corpus size from the index state", async function () { + let before = await Zotero.Lexical.getCorpusSize(); + await addDoc(5, "lexcorpus size probe"); + assert.equal(await Zotero.Lexical.getCorpusSize(), before + 1); + }); + }); + + // A unit from an analyzed list by its text, for asserting on one term + function unitByText(units, text) { + return units.find(unit => unit.text == text); + } + + describe("combined statistics", function () { + const BASE_ROWID = 920000000; + var inserted = []; + + async function addContentDoc(id, text) { + let rowid = BASE_ROWID + id; + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)", + [rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || ''] + ); + await Zotero.DB.queryAsync( + "REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)", + [rowid] + ); + inserted.push(rowid); + return rowid; + } + + after(async function () { + for (let rowid of inserted) { + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid); + } + }); + + it("should count document frequency across content and item text", async function () { + await addContentDoc(1, "lexcross appears in a document"); + await createDataObject('item', { title: 'Lexcross appears in a title' }); + assert.equal(await Zotero.Lexical.getDocumentFrequency( + { type: 'word', text: 'lexcross', prefix: false } + ), 2); + }); + + it("should count items toward the corpus size", async function () { + let before = await Zotero.Lexical.getCorpusSize(); + await createDataObject('item', { title: 'Lexcorpus member item' }); + assert.equal(await Zotero.Lexical.getCorpusSize(), before + 1); + }); + + it("should keep a term cheap for an annotation when the library is full of it", async function () { + this.timeout(60000); + // The term saturates the content corpus... + let corpusSize = await Zotero.Lexical.getCorpusSize(); + for (let i = 0; i < corpusSize + 10; i++) { + await addContentDoc(100 + i, `lexeverywhere document ${i}`); + } + // ...and appears once in an annotation + let item = await createDataObject('item'); + let attachment = await importPDFAttachment(item); + await createAnnotation('highlight', attachment, + { comment: 'lexeverywhere in a comment' }); + + let units = await Zotero.Lexical.analyzeQuery("lexeverywhere lexveryrare "); + let everywhere = unitByText(units, 'lexeverywhere'); + let rare = unitByText(units, 'lexveryrare'); + // Rarity is a property of the library, not of where the match + // lands: the saturated term is worth little anywhere, including + // in the annotation, and gets cut from retrieval + assert.isBelow(everywhere.weight, rare.weight * 0.2); + assert.isFalse(everywhere.informative); + assert.isTrue(rare.informative); + }); + + it("should keep the best unit informative in an all-common query", async function () { + let stubs = [ + sinon.stub(Zotero.Lexical, 'getCorpusSize').resolves(1000), + sinon.stub(Zotero.Lexical, 'getDocumentFrequency').resolves(950) + ]; + try { + let units = await Zotero.Lexical.analyzeQuery("common words only "); + assert.isTrue(units.every(unit => unit.informative)); + } + finally { + stubs.forEach(stub => stub.restore()); + } + }); + }); + + describe("#matchContent()", function () { + const BASE_ROWID = 930000000; + var inserted = []; + + async function addContentDoc(id, text, cjkBigrams) { + let rowid = BASE_ROWID + id; + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)", + [rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || ''] + ); + if (cjkBigrams) { + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContentCJK (rowid, text) VALUES (?, ?)", + [rowid, cjkBigrams] + ); + } + await Zotero.DB.queryAsync( + "REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)", + [rowid] + ); + inserted.push(rowid); + return rowid; + } + + after(async function () { + for (let rowid of inserted) { + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid); + } + }); + + it("should report presence for units a document contains", async function () { + let doc = await addContentDoc(1, "lexprobe migration study text"); + let other = await addContentDoc(2, "entirely unrelated content"); + let units = await Zotero.Lexical.analyzeQuery("lexprobe missingword "); + let strengths = await Zotero.Lexical.matchContent(units, [doc, other]); + assert.equal(strengths.get(doc).get(unitByText(units, 'lexprobe')), 1); + assert.isFalse(strengths.get(doc).has(unitByText(units, 'missingword'))); + assert.isFalse(strengths.has(other)); + }); + + it("should only report the requested candidates", async function () { + let units = await Zotero.Lexical.analyzeQuery("lexprobe "); + let strengths = await Zotero.Lexical.matchContent(units, [BASE_ROWID + 2]); + assert.isFalse(strengths.has(BASE_ROWID + 1)); + }); + + it("should match a prefix unit against its completions", async function () { + let units = await Zotero.Lexical.analyzeQuery("lexprob"); + assert.isTrue(units[0].prefix); + let strengths = await Zotero.Lexical.matchContent(units, [BASE_ROWID + 1]); + assert.equal(strengths.get(BASE_ROWID + 1).get(units[0]), 1); + }); + + it("should match CJK units against the 2-gram index", async function () { + let doc = await addContentDoc(3, "lexcjk carrier", '疫情 情控 控制'); + let units = await Zotero.Lexical.analyzeQuery("疫情控制 "); + let strengths = await Zotero.Lexical.matchContent(units, [doc]); + assert.equal(strengths.get(doc).get(units[0]), 1); + }); + + it("should verify a phrase against the document's stored text", async function () { + this.timeout(60000); + let item = await createDataObject('item'); + let attachment = await importPDFAttachment(item); + await Zotero.Fulltext.indexItems([attachment.id]); + + // "easy-to-use" in the document: hyphens and spaces separate the + // same words, so the phrase verifies + let units = await Zotero.Lexical.analyzeQuery('"easy to use" probe'); + let phrase = units.find(unit => unit.type == 'phrase'); + let strengths = await Zotero.Lexical.matchContent([phrase], [attachment.id]); + assert.equal(strengths.get(attachment.id).get(phrase), 1); + + // "collect, organize" in the document: adjacent for the index, + // but the comma has to match literally, so verification rejects it + units = await Zotero.Lexical.analyzeQuery('"collect organize" probe'); + phrase = units.find(unit => unit.type == 'phrase'); + strengths = await Zotero.Lexical.matchContent([phrase], [attachment.id]); + assert.isFalse(strengths.has(attachment.id)); + }); + }); + + describe("#matchFields()", function () { + it("should report title and abstract matches separately, at full strength", async function () { + let item = await createDataObject('item', { title: 'Owl migration atlas' }); + item.setField('abstractNote', 'Statistical methods for tracking studies'); + await item.saveTx(); + + let units = await Zotero.Lexical.analyzeQuery("migration statistical "); + let result = await Zotero.Lexical.matchFields(units, [item.id]); + assert.equal(result.title.get(item.id).get(unitByText(units, 'migration')), 1); + assert.isFalse(result.title.get(item.id).has(unitByText(units, 'statistical'))); + assert.equal(result.abstract.get(item.id).get(unitByText(units, 'statistical')), 1); + assert.isFalse(result.abstract.get(item.id).has(unitByText(units, 'migration'))); + }); + + it("should match whole words, with the trailing prefix matching completions", async function () { + let item = await createDataObject('item', { title: 'Rainfall patterns' }); + let units = await Zotero.Lexical.analyzeQuery("fall "); + let result = await Zotero.Lexical.matchFields(units, [item.id]); + assert.isFalse(result.title.has(item.id)); + units = await Zotero.Lexical.analyzeQuery("rain"); + result = await Zotero.Lexical.matchFields(units, [item.id]); + assert.equal(result.title.get(item.id).get(units[0]), 1); + }); + + it("should match titles diacritic-insensitively", async function () { + let item = await createDataObject('item', { title: 'Müller précis studies' }); + let units = await Zotero.Lexical.analyzeQuery("muller precis "); + let result = await Zotero.Lexical.matchFields(units, [item.id]); + assert.equal(result.title.get(item.id).get(unitByText(units, 'muller')), 1); + assert.equal(result.title.get(item.id).get(unitByText(units, 'precis')), 1); + }); + + it("should verify a phrase against the stored title", async function () { + // Adjacent for the index either way; only the hyphen reads as a + // space literally + let hyphenated = await createDataObject('item', + { title: 'Lexuno-lexdos analysis' }); + let punctuated = await createDataObject('item', + { title: 'Lexuno. Lexdos analysis' }); + let units = await Zotero.Lexical.analyzeQuery('"lexuno lexdos" probe'); + let phrase = units.find(unit => unit.type == 'phrase'); + let result = await Zotero.Lexical.matchFields( + [phrase], [hyphenated.id, punctuated.id]); + assert.equal(result.title.get(hyphenated.id).get(phrase), 1); + assert.isFalse(result.title.has(punctuated.id)); + }); + }); + + describe("#matchNotes()", function () { + it("should score a note that returns to a term above one passing mention", async function () { + let focused = new Zotero.Item('note'); + focused.setNote('

Lexowls hunt at night. Lexowls migrate. Lexowls return.

'); + await focused.saveTx(); + let passing = new Zotero.Item('note'); + let filler = Array.from({ length: 200 }, (x, i) => `note${i}`).join(' '); + passing.setNote(`

One mention of lexowls. ${filler}

`); + await passing.saveTx(); + + let units = await Zotero.Lexical.analyzeQuery("lexowls "); + let strengths = await Zotero.Lexical.matchNotes(units, [focused.id, passing.id]); + let unit = units[0]; + assert.isAbove(strengths.get(focused.id).get(unit), + strengths.get(passing.id).get(unit)); + assert.isAbove(strengths.get(passing.id).get(unit), 0); + assert.isBelow(strengths.get(focused.id).get(unit), 1); + }); + + it("should match whole words only", async function () { + let note = new Zotero.Item('note'); + note.setNote('

Heavy rainfall in the region

'); + await note.saveTx(); + let units = await Zotero.Lexical.analyzeQuery("fall "); + let strengths = await Zotero.Lexical.matchNotes(units, [note.id]); + assert.isFalse(strengths.has(note.id)); + }); + + it("should match an indexed note through the index", async function () { + let note = new Zotero.Item('note'); + note.setNote('

Lexindexed observations here today

'); + await note.saveTx(); + await Zotero.FullText.processNoteIndexQueue(); + let units = await Zotero.Lexical.analyzeQuery("lexindexed "); + let strengths = await Zotero.Lexical.matchNotes(units, [note.id]); + assert.isAbove(strengths.get(note.id).get(units[0]), 0); + }); + + it("should match a just-edited note by its current text", async function () { + let note = new Zotero.Item('note'); + note.setNote('

lexoldword only here

'); + await note.saveTx(); + await Zotero.FullText.processNoteIndexQueue(); + note.setNote('

lexnewword replaces it

'); + await note.saveTx(); + + let units = await Zotero.Lexical.analyzeQuery("lexoldword lexnewword "); + let strengths = await Zotero.Lexical.matchNotes(units, [note.id]); + assert.isTrue(strengths.get(note.id).has(unitByText(units, 'lexnewword'))); + assert.isFalse(strengths.get(note.id).has(unitByText(units, 'lexoldword'))); + }); + + it("should match a note the index doesn't have yet", async function () { + let note = new Zotero.Item('note'); + note.setNote('

lexunindexed content waiting for backfill

'); + await note.saveTx(); + // Simulate a note that predates the index + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextNoteIndexState WHERE itemID=?", note.id); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextItemText WHERE rowid=?", note.id); + let units = await Zotero.Lexical.analyzeQuery("lexunindexed "); + let strengths = await Zotero.Lexical.matchNotes(units, [note.id]); + assert.isAbove(strengths.get(note.id).get(units[0]), 0); + }); + + it("should count a phrase literally", async function () { + let hyphenated = new Zotero.Item('note'); + hyphenated.setNote('

The lexunited-states policy record

'); + await hyphenated.saveTx(); + let punctuated = new Zotero.Item('note'); + punctuated.setNote('

The lexunited. States policy record

'); + await punctuated.saveTx(); + + let units = await Zotero.Lexical.analyzeQuery('"lexunited states" probe'); + let phrase = units.find(unit => unit.type == 'phrase'); + let strengths = await Zotero.Lexical.matchNotes( + [phrase], [hyphenated.id, punctuated.id]); + assert.isAbove(strengths.get(hyphenated.id).get(phrase), 0); + assert.isFalse(strengths.has(punctuated.id)); + }); + }); + + describe("#scoreItemIDs()", function () { + const BASE_ROWID = 940000000; + var inserted = []; + + async function addContentDoc(id, text) { + let rowid = BASE_ROWID + id; + await Zotero.DB.queryAsync( + "INSERT INTO ftindex.fulltextContent (rowid, text) VALUES (?, ?)", + [rowid, Zotero.Utilities.Internal.normalizeForSearch(text) || ''] + ); + await Zotero.DB.queryAsync( + "REPLACE INTO ftindex.fulltextIndexState (itemID, version) VALUES (?, 1)", + [rowid] + ); + inserted.push(rowid); + return rowid; + } + + after(async function () { + for (let rowid of inserted) { + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextContent WHERE rowid=?", rowid); + await Zotero.DB.queryAsync( + "DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", rowid); + } + }); + + it("should rank coverage over partial matches, wherever they land", async function () { + // The walkthrough corpus: full coverage in a title, partial + // coverage in a title, partial coverage in a document, and noise + // containing none of the query's words. (Whether "in"/"the" count + // as informative depends on the corpus -- in this test library + // they can be rare enough to -- so the full title carries every + // query word.) + let full = await createDataObject('item', + { title: 'Lexsowl lexsmigration in the lexsunited lexsstates' }); + let census = await createDataObject('item', + { title: 'Lexsunited lexsstates census records' }); + let norway = await addContentDoc(1, + 'lexsowl lexsmigration routes across norway seasons'); + let noise = await addContentDoc(2, 'lexsother archive entry'); + + let scores = await Zotero.Lexical.scoreItemIDs( + 'lexsowl lexsmigration in the lexsunited lexsstates ', + [full.id, census.id, norway, noise] + ); + // Full coverage at full strength in the best source is the + // ceiling itself + assert.approximately(scores.get(full.id), 1, 0.001); + // Everything with an informative match is in, ranked under it... + assert.isAbove(scores.get(full.id), scores.get(census.id)); + assert.isAbove(scores.get(census.id), 0); + assert.isAbove(scores.get(norway), 0); + // ...and matching nothing the query asked for is no match at all + assert.isFalse(scores.has(noise)); + }); + + it("should count the same unit once, at its best source", async function () { + let item = await createDataObject('item', { title: 'Lexsboth appears here' }); + item.setField('abstractNote', 'Lexsboth appears in the abstract too'); + await item.saveTx(); + let scores = await Zotero.Lexical.scoreItemIDs('lexsboth ', [item.id]); + // Title and abstract both match; max, not sum -- a score of + // exactly 1 proves no double counting + assert.approximately(scores.get(item.id), 1, 0.001); + }); + + it("should grade document matches relative to each other", async function () { + let filler = Array.from({ length: 300 }, (x, i) => `lexsfill${i}`).join(' '); + let buried = await addContentDoc(3, `lexsdeep ${filler}`); + let focused = await addContentDoc(4, 'lexsdeep lexsdeep lexsdeep summary'); + let scores = await Zotero.Lexical.scoreItemIDs( + 'lexsdeep ', [buried, focused]); + // The strongest document anchors the unit's full strength: one + // unit at content boost against a title-boosted ceiling + assert.approximately(scores.get(focused), 0.5, 0.001); + if (scores.has(buried)) { + assert.isBelow(scores.get(buried), scores.get(focused)); + } + }); + + it("should drop scores below the floor", async function () { + let stubs = [ + sinon.stub(Zotero.Lexical, 'analyzeQuery').resolves([ + { type: 'word', text: 'big', prefix: false, df: 1, weight: 5, informative: true }, + { type: 'word', text: 'small', prefix: false, df: 1, weight: 1, informative: true } + ]), + sinon.stub(Zotero.Lexical, 'matchContent').callsFake( + async (units, itemIDs) => new Map([ + // One item barely touches the small unit; another + // matches the big one outright + [itemIDs[0], new Map([[units[1], 0.1]])], + [itemIDs[1], new Map([[units[0], 1]])] + ]) + ), + sinon.stub(Zotero.Lexical, 'matchFields').resolves( + { title: new Map(), abstract: new Map() }), + sinon.stub(Zotero.Lexical, 'matchNotes').resolves(new Map()), + sinon.stub(Zotero.Lexical, 'matchAnnotations').resolves(new Map()) + ]; + try { + let scores = await Zotero.Lexical.scoreItemIDs('anything', [1, 2]); + // (1 * 0.1) / 12 is under the floor; (5 * 1) / 12 is well over + assert.isFalse(scores.has(1)); + assert.isAbove(scores.get(2), 0.4); + } + finally { + stubs.forEach(stub => stub.restore()); + } + }); + + it("should return nothing for a query with no units", async function () { + assert.equal((await Zotero.Lexical.scoreItemIDs('', [1])).size, 0); + assert.equal((await Zotero.Lexical.scoreItemIDs('!!! ...', [1])).size, 0); + }); + + it("should abandon scoring when cancelled", async function () { + let item = await createDataObject('item', { title: 'Lexscancel target' }); + let e = await getPromiseError(Zotero.Lexical.scoreItemIDs( + 'lexscancel ', [item.id], { shouldCancel: () => true })); + assert.instanceOf(e, Zotero.Lexical.ScoringCancelledError); + }); + }); + + describe("#matchAnnotations()", function () { + it("should match an annotation's passage and comment at full strength", async function () { + this.timeout(60000); + let item = await createDataObject('item'); + let attachment = await importPDFAttachment(item); + let annotation = await createAnnotation('highlight', attachment, + { comment: 'lexanno methodology worry' }); + + let units = await Zotero.Lexical.analyzeQuery("lexanno missingword "); + let strengths = await Zotero.Lexical.matchAnnotations( + units, [annotation.id, attachment.id]); + assert.equal(strengths.get(annotation.id).get(unitByText(units, 'lexanno')), 1); + assert.isFalse(strengths.get(annotation.id).has(unitByText(units, 'missingword'))); + assert.isFalse(strengths.has(attachment.id)); + }); + + it("should match annotations diacritic-insensitively", async function () { + this.timeout(60000); + let item = await createDataObject('item'); + let attachment = await importPDFAttachment(item); + let annotation = await createAnnotation('highlight', attachment, + { comment: 'Müller réviewed this lexdiacritic passage' }); + + let units = await Zotero.Lexical.analyzeQuery("muller reviewed lexdiacritic "); + let strengths = await Zotero.Lexical.matchAnnotations(units, [annotation.id]); + for (let unit of units) { + assert.equal(strengths.get(annotation.id).get(unit), 1); + } + }); + }); +});