mirror of
https://github.com/zotero/zotero.git
synced 2026-08-28 05:25:31 +00:00
070ae8b615
Editing a note flagged it stale, and the background drain indexed it and then ran an FTS5 'optimize' -- a single statement that rewrites the content index and can hold the shared database for over a minute -- because the queue was empty again. Note saves waiting on the connection hit the transaction timeout and told the user to restart Zotero. Merge the index in bounded steps instead, so no statement runs long enough to keep other queries waiting, and only after enough items have been indexed to be worth it. https://forums.zotero.org/discussion/133298/
915 lines
38 KiB
JavaScript
915 lines
38 KiB
JavaScript
describe("Zotero.FullText", function () {
|
||
describe("Content database", function () {
|
||
async function createTextAttachment(content) {
|
||
let tmpDir = await getTempDirectory();
|
||
let path = OS.Path.join(tmpDir, Zotero.Utilities.randomString() + ".txt");
|
||
await Zotero.File.putContentsAsync(path, content);
|
||
return Zotero.Attachments.importFromFile({
|
||
file: path,
|
||
contentType: 'text/plain',
|
||
charset: 'utf-8'
|
||
});
|
||
}
|
||
|
||
function contentSearch(item, value) {
|
||
let s = new Zotero.Search();
|
||
s.libraryID = item.libraryID;
|
||
s.addCondition('fulltextContent', 'contains', value);
|
||
return s.search();
|
||
}
|
||
|
||
it("should match content case- and diacritic-insensitively", async function () {
|
||
let item = await createTextAttachment("The Séance was held at dawn");
|
||
assert.include(await contentSearch(item, 'seance'), item.id);
|
||
assert.include(await contentSearch(item, 'SÉANCE'), item.id);
|
||
});
|
||
|
||
it("should match other forms of a word by prefix", async function () {
|
||
let item = await createTextAttachment("old archives were archived");
|
||
assert.include(await contentSearch(item, 'archive'), item.id);
|
||
});
|
||
|
||
it("should not match the middle or end of a word", async function () {
|
||
let suffix = await createTextAttachment("the condition holds");
|
||
assert.notInclude(await contentSearch(suffix, 'ion'), suffix.id);
|
||
let prefixed = await createTextAttachment("postcolonialism studies");
|
||
assert.notInclude(await contentSearch(prefixed, 'colonialism'), prefixed.id);
|
||
});
|
||
|
||
it("should prefix-match a short term used as an explicit condition", async function () {
|
||
let item = await createTextAttachment("the quick brown fox");
|
||
// Short terms are only skipped by unquoted quick search (canSearchContent), so "fo"
|
||
// matches "fox" as a word prefix here
|
||
assert.include(await contentSearch(item, 'fo'), item.id);
|
||
});
|
||
|
||
it("should prefix-match accent-insensitively", async function () {
|
||
let item = await createTextAttachment("zzcafés in paris");
|
||
assert.include(await contentSearch(item, 'zzcafé'), item.id);
|
||
});
|
||
|
||
it("should match a phrase with the final word as a prefix", async function () {
|
||
let item = await createTextAttachment("global climate changes are accelerating");
|
||
assert.include(await contentSearch(item, 'climate chang'), item.id);
|
||
});
|
||
|
||
it("should match a phrase across whitespace and hyphens", async function () {
|
||
// Word separators vary with extraction layout and compound styling, so whitespace
|
||
// and hyphen runs in the phrase and the content match each other
|
||
let newline = await createTextAttachment("the climate\nchange debate");
|
||
assert.include(await contentSearch(newline, 'climate change'), newline.id);
|
||
let hyphen = await createTextAttachment("climate-change research");
|
||
assert.include(await contentSearch(hyphen, 'climate change'), hyphen.id);
|
||
let spaced = await createTextAttachment("zzdecision making processes");
|
||
assert.include(await contentSearch(spaced, 'zzdecision-making'), spaced.id);
|
||
});
|
||
|
||
it("should not match a phrase across punctuation", async function () {
|
||
// These are index candidates -- FTS5 ignores what separates adjacent tokens -- so
|
||
// they exercise the verification scan of the cached text
|
||
let punct = await createTextAttachment("the climate. Change is coming");
|
||
assert.notInclude(await contentSearch(punct, 'climate change'), punct.id);
|
||
let slash = await createTextAttachment("climate/change models");
|
||
assert.notInclude(await contentSearch(slash, 'climate change'), slash.id);
|
||
});
|
||
|
||
it("should require punctuation in a single-word term to match literally", async function () {
|
||
let cpp = await createTextAttachment("the c++ standard library");
|
||
assert.include(await contentSearch(cpp, 'c++'), cpp.id);
|
||
// "code" is a candidate for the "c" token prefix, but "c++" doesn't appear
|
||
let plain = await createTextAttachment("zzcat code samples");
|
||
assert.notInclude(await contentSearch(plain, 'c++'), plain.id);
|
||
});
|
||
|
||
it("should not match anything for a separator-only term", async function () {
|
||
let item = await createTextAttachment("plain text with spaces");
|
||
assert.notInclude(await contentSearch(item, '--'), item.id);
|
||
});
|
||
|
||
it("should match a 2-character CJK query", async function () {
|
||
let item = await createTextAttachment("中文搜索系统设计");
|
||
assert.include(await contentSearch(item, '搜索'), item.id);
|
||
});
|
||
|
||
it("should match a longer CJK substring within a run", async function () {
|
||
let item = await createTextAttachment("中文搜索系统设计");
|
||
assert.include(await contentSearch(item, '搜索系统'), item.id);
|
||
});
|
||
|
||
it("should distinguish voiced and unvoiced kana in CJK content", async function () {
|
||
let item = await createTextAttachment("研究がん");
|
||
assert.include(await contentSearch(item, 'がん'), item.id);
|
||
assert.notInclude(await contentSearch(item, 'かん'), item.id);
|
||
});
|
||
|
||
it("should match a mixed CJK and non-CJK term literally", async function () {
|
||
let mixed = await createTextAttachment("москва日本 report");
|
||
let cjkOnly = await createTextAttachment("日本語の研究");
|
||
// Neither index covers the term alone, so it falls back to a scan of the cached
|
||
// text; a document containing only the CJK part doesn't match
|
||
let results = await contentSearch(mixed, 'москва日本');
|
||
assert.include(results, mixed.id);
|
||
assert.notInclude(results, cjkOnly.id);
|
||
});
|
||
|
||
it("should not match an unrelated CJK document", async function () {
|
||
let item = await createTextAttachment("天气预报很准确");
|
||
let other = await createTextAttachment("中文搜索系统设计");
|
||
let results = await contentSearch(item, '搜索');
|
||
assert.notInclude(results, item.id);
|
||
assert.include(results, other.id);
|
||
});
|
||
|
||
it("should record an index-state row when indexing", async function () {
|
||
let item = await createTextAttachment("index state recorded here");
|
||
let version = await Zotero.DB.valueQueryAsync(
|
||
"SELECT version FROM ftindex.fulltextIndexState WHERE itemID=?", item.id
|
||
);
|
||
assert.equal(version, 1);
|
||
});
|
||
|
||
it("should index queued items missing from the content index", async function () {
|
||
let item = await createTextAttachment("queued jabberwocky content");
|
||
// Simulate an item full-text indexed before the content index existed: drop its entries
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
|
||
// It's now in the queue and not findable by content
|
||
assert.isAbove(await Zotero.FullText.getAttachmentIndexQueueCount(), 0);
|
||
assert.notInclude(await contentSearch(item, 'jabberwocky'), item.id);
|
||
|
||
// Draining the queue re-indexes it from the cached text
|
||
await Zotero.FullText.processAttachmentIndexQueue();
|
||
|
||
assert.include(await contentSearch(item, 'jabberwocky'), item.id);
|
||
let version = await Zotero.DB.valueQueryAsync(
|
||
"SELECT version FROM ftindex.fulltextIndexState WHERE itemID=?", item.id
|
||
);
|
||
assert.equal(version, 1);
|
||
});
|
||
|
||
it("should re-extract a queued item whose cache file is missing", async function () {
|
||
let item = await importFileAttachment('test.pdf');
|
||
let cacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
// Grab a real word from the extracted text to search for after re-extraction
|
||
let word = (await Zotero.File.getContentsAsync(cacheFile)).match(/[a-z]{5,}/i)[0];
|
||
// Simulate the storage folder being cleared after indexing (cache gone, file present),
|
||
// then re-queue the item
|
||
await OS.File.remove(cacheFile);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
|
||
// Draining the queue should re-extract from the file (not index empty)
|
||
await Zotero.FullText.processAttachmentIndexQueue();
|
||
|
||
assert.isTrue(await OS.File.exists(cacheFile));
|
||
assert.include(await contentSearch(item, word), item.id);
|
||
});
|
||
|
||
it("should not repeatedly re-extract a queued item with no extractable text", async function () {
|
||
let item = await importFileAttachment('test.pdf');
|
||
let cacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
await OS.File.remove(cacheFile);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContentCJK WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
|
||
let getFullText = sinon.stub(Zotero.PDFWorker, 'getFullText').resolves({
|
||
text: '',
|
||
extractedPages: 0,
|
||
totalPages: 1
|
||
});
|
||
try {
|
||
await Zotero.FullText.processAttachmentIndexQueue({ maxTime: 100 });
|
||
}
|
||
finally {
|
||
getFullText.restore();
|
||
}
|
||
|
||
assert.isTrue(getFullText.calledOnce);
|
||
let version = await Zotero.DB.valueQueryAsync(
|
||
"SELECT version FROM ftindex.fulltextIndexState WHERE itemID=?", item.id
|
||
);
|
||
assert.equal(version, 1);
|
||
assert.equal(await Zotero.FullText.getAttachmentIndexQueueCount(), 0);
|
||
});
|
||
|
||
it("should not count unindexable attachments as not indexed", async function () {
|
||
let before = (await Zotero.FullText.getIndexStats()).unindexedQueue;
|
||
// A Word document can't be full-text indexed, so it shouldn't inflate "not indexed"
|
||
let path = OS.Path.join(await getTempDirectory(), Zotero.Utilities.randomString() + ".doc");
|
||
await Zotero.File.putContentsAsync(path, "not really a word doc");
|
||
await Zotero.Attachments.importFromFile({
|
||
file: path, contentType: 'application/msword'
|
||
});
|
||
let after = (await Zotero.FullText.getIndexStats()).unindexedQueue;
|
||
assert.equal(after, before);
|
||
});
|
||
|
||
it("should index an unindexed attachment that has a local file", async function () {
|
||
let item = await createTextAttachment("filepresentunindexed content");
|
||
// Simulate a never-indexed attachment: no fulltextItems row, not in the index
|
||
await Zotero.DB.queryAsync("DELETE FROM fulltextItems WHERE itemID=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
assert.notInclude(await contentSearch(item, 'filepresentunindexed'), item.id);
|
||
await Zotero.FullText.processAttachmentExtractionQueue();
|
||
assert.include(await contentSearch(item, 'filepresentunindexed'), item.id);
|
||
});
|
||
|
||
it("should record an unindexed attachment with no local file as missing", async function () {
|
||
let item = await createTextAttachment("nolocalfile content");
|
||
await Zotero.DB.queryAsync("DELETE FROM fulltextItems WHERE itemID=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
// Remove the file so there's no local content to index
|
||
await OS.File.remove(await item.getFilePathAsync());
|
||
await Zotero.FullText.processAttachmentExtractionQueue();
|
||
let synced = await Zotero.DB.valueQueryAsync(
|
||
"SELECT synced FROM fulltextItems WHERE itemID=?", item.id
|
||
);
|
||
assert.equal(synced, Zotero.FullText.SYNC_STATE_MISSING);
|
||
});
|
||
|
||
it("should record an attachment as missing when its content can't be extracted", async function () {
|
||
// A file that isn't a valid EPUB: extraction fails, so the import leaves no fulltextItems
|
||
// row. The item must still leave the queue (recorded missing) rather than being retried
|
||
// endlessly.
|
||
let tmpDir = await getTempDirectory();
|
||
let path = OS.Path.join(tmpDir, Zotero.Utilities.randomString() + ".epub");
|
||
await Zotero.File.putContentsAsync(path, "not really an EPUB");
|
||
let item = await Zotero.Attachments.importFromFile({
|
||
file: path, contentType: 'application/epub+zip'
|
||
});
|
||
assert.isNotOk(await Zotero.DB.valueQueryAsync(
|
||
"SELECT 1 FROM fulltextItems WHERE itemID=?", item.id
|
||
));
|
||
await Zotero.FullText.processAttachmentExtractionQueue();
|
||
let synced = await Zotero.DB.valueQueryAsync(
|
||
"SELECT synced FROM fulltextItems WHERE itemID=?", item.id
|
||
);
|
||
assert.equal(synced, Zotero.FullText.SYNC_STATE_MISSING);
|
||
// No longer pending, so the drain won't keep retrying it
|
||
assert.equal(await Zotero.FullText.getAttachmentExtractionQueueCount(), 0);
|
||
});
|
||
|
||
it("should skip unindexed attachments while indexing is disabled and index them when re-enabled", async function () {
|
||
let item = await createTextAttachment("disabledpref content");
|
||
await Zotero.DB.queryAsync("DELETE FROM fulltextItems WHERE itemID=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextContent WHERE rowid=?", item.id);
|
||
await Zotero.DB.queryAsync("DELETE FROM ftindex.fulltextIndexState WHERE itemID=?", item.id);
|
||
Zotero.Prefs.set('fulltext.textMaxLength', 0);
|
||
try {
|
||
// Disabled: not counted as pending, and a pass records nothing (so it can't loop)
|
||
assert.equal(await Zotero.FullText.getAttachmentExtractionQueueCount(), 0);
|
||
await Zotero.FullText.processAttachmentExtractionQueue();
|
||
assert.isNotOk(await Zotero.DB.valueQueryAsync(
|
||
"SELECT 1 FROM fulltextItems WHERE itemID=?", item.id
|
||
));
|
||
}
|
||
finally {
|
||
Zotero.Prefs.clear('fulltext.textMaxLength');
|
||
}
|
||
// Re-enabled: eligible again and indexed
|
||
await Zotero.FullText.processAttachmentExtractionQueue();
|
||
assert.include(await contentSearch(item, 'disabledpref'), item.id);
|
||
});
|
||
|
||
it("should purge content index entries orphaned by item deletion", async function () {
|
||
let item = await createTextAttachment("orphanpurge content");
|
||
assert.include(await contentSearch(item, 'orphanpurge'), item.id);
|
||
// Simulate the FK cascade from a delete that bypassed Item.erase(): drop the
|
||
// fulltextItems row, leaving the content index entries orphaned
|
||
await Zotero.DB.queryAsync("DELETE FROM fulltextItems WHERE itemID=?", item.id);
|
||
await Zotero.FullText.purgeOrphanedContent();
|
||
assert.notInclude(await contentSearch(item, 'orphanpurge'), item.id);
|
||
});
|
||
|
||
it("should remove content from the index when cleared", async function () {
|
||
let item = await createTextAttachment("disposable singular content");
|
||
assert.include(await contentSearch(item, 'disposable'), item.id);
|
||
await Zotero.DB.executeTransaction(async function () {
|
||
await Zotero.FullText.clearItemWords(item.id);
|
||
});
|
||
assert.notInclude(await contentSearch(item, 'disposable'), item.id);
|
||
});
|
||
|
||
it("should keep content searchable after optimizing the index", async function () {
|
||
let item = await createTextAttachment("optimizable content");
|
||
// Failures are logged rather than thrown, so make sure the merge ran cleanly
|
||
let logError = sinon.spy(Zotero, 'logError');
|
||
try {
|
||
await Zotero.FullText.optimizeContentIndex({ force: true });
|
||
}
|
||
finally {
|
||
logError.restore();
|
||
}
|
||
assert.isTrue(logError.notCalled);
|
||
assert.include(await contentSearch(item, 'optimizable'), item.id);
|
||
});
|
||
|
||
it("should start each table's merge with a negative page count", async function () {
|
||
// A positive count only merges levels that already have enough segments of their own;
|
||
// SQLite requires a negative one to start a merge across all of them
|
||
await createTextAttachment("mergeprotocol content");
|
||
let queryAsync = sinon.spy(Zotero.DB, 'queryAsync');
|
||
try {
|
||
await Zotero.FullText.optimizeContentIndex({ force: true });
|
||
}
|
||
finally {
|
||
queryAsync.restore();
|
||
}
|
||
let merges = queryAsync.args.map(args => args[0])
|
||
.filter(sql => typeof sql == 'string' && sql.includes("VALUES('merge'"));
|
||
assert.isNotEmpty(merges);
|
||
for (let table of ['fulltextContent', 'fulltextContentCJK', 'fulltextNotes',
|
||
'fulltextNotesCJK']) {
|
||
let first = merges.find(sql => sql.includes(`ftindex.${table}(`));
|
||
assert.include(first, "VALUES('merge', -", table);
|
||
}
|
||
});
|
||
|
||
it("should keep content searchable after vacuuming the index", async function () {
|
||
let item = await createTextAttachment("vacuumable content");
|
||
await Zotero.FullText.vacuumContentIndex({ force: true });
|
||
assert.include(await contentSearch(item, 'vacuumable'), item.id);
|
||
});
|
||
|
||
it("should discard the index when it belongs to a different database instance", async function () {
|
||
let item = await createTextAttachment("instancemismatch content");
|
||
assert.include(await contentSearch(item, 'instancemismatch'), item.id);
|
||
// Simulate an index built against a different zotero.sqlite (e.g., the database was
|
||
// deleted and re-synced, reassigning local itemIDs) by changing the stored localUserKey
|
||
await Zotero.DB.queryAsync(
|
||
"UPDATE ftindex.fulltextIndexMeta SET value='xxxxxxxx' WHERE key='localUserKey'"
|
||
);
|
||
// Force a reconnect, which re-runs the content DB setup and detects the mismatch
|
||
await Zotero.DB.vacuum({ force: true });
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync("SELECT COUNT(*) FROM ftindex.fulltextIndexState"), 0
|
||
);
|
||
assert.notInclude(await contentSearch(item, 'instancemismatch'), item.id);
|
||
});
|
||
|
||
describe("Note indexing", function () {
|
||
async function createNote(noteText) {
|
||
let item = new Zotero.Item('note');
|
||
item.setNote(noteText);
|
||
await item.saveTx();
|
||
return item;
|
||
}
|
||
|
||
function noteSearch(item, value) {
|
||
let s = new Zotero.Search();
|
||
s.libraryID = item.libraryID;
|
||
s.addCondition('note', 'contains', value);
|
||
return s.search();
|
||
}
|
||
|
||
it("should match note content case- and diacritic-insensitively via the index", async function () {
|
||
let item = await createNote("<p>zqnnThe Séance was held at dawn</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.include(await noteSearch(item, 'zqnnthe seance'), item.id);
|
||
});
|
||
|
||
it("should match a just-edited note before it's re-indexed", async function () {
|
||
let item = await createNote("<p>zqnnoriginal content</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
item.setNote('<p style="color: red">zqnnréplacement content</p>');
|
||
await item.saveTx();
|
||
// The edit is matched right away, without waiting for background re-indexing, with
|
||
// the same accent-insensitive matching as the index (é folded)...
|
||
assert.include(await noteSearch(item, 'zqnnreplacement'), item.id);
|
||
// ...the old content no longer matches...
|
||
assert.notInclude(await noteSearch(item, 'zqnnoriginal'), item.id);
|
||
// ...and markup isn't matched
|
||
assert.notInclude(await noteSearch(item, 'color: red'), item.id);
|
||
});
|
||
|
||
it("should not match note markup", async function () {
|
||
let item = await createNote('<p style="color: red">zqnnstyled text</p>');
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.include(await noteSearch(item, 'zqnnstyled'), item.id);
|
||
assert.notInclude(await noteSearch(item, 'color: red'), item.id);
|
||
});
|
||
|
||
it("should scan plain text but not markup for a term too short for the index", async function () {
|
||
// "ai" and "re" are too short to index, so they fall back to scanning the note's
|
||
// normalized plain text. "ai" is in the text and matches; "re" appears only in "red"
|
||
// in the style attribute, so scanning plain text (not raw HTML) doesn't match it.
|
||
let item = await createNote('<p style="color: red">zqnn ai content</p>');
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.include(await noteSearch(item, 'ai'), item.id);
|
||
assert.notInclude(await noteSearch(item, 're'), item.id);
|
||
});
|
||
|
||
it("should match a mixed-script note term via the plain-text fallback", async function () {
|
||
// A mixed CJK/non-CJK term suits neither index, so it falls back to the plain-text scan
|
||
let item = await createNote("<p>zqnn研究ai</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.include(await noteSearch(item, '究ai'), item.id);
|
||
});
|
||
|
||
it("should match a mixed CJK and non-ASCII note term literally", async function () {
|
||
let mixed = await createNote("<p>zqnnмосква日本 report</p>");
|
||
let cjkOnly = await createNote("<p>zqnn日本語の研究</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
// The term suits neither index, so it scans the notes' plain text; a note
|
||
// containing only the CJK part doesn't match
|
||
let results = await noteSearch(mixed, 'москва日本');
|
||
assert.include(results, mixed.id);
|
||
assert.notInclude(results, cjkOnly.id);
|
||
});
|
||
|
||
it("shouldn't optimize the content index after indexing a note", async function () {
|
||
// Clear the count of items indexed since the last optimize
|
||
await Zotero.FullText.optimizeContentIndex({ force: true });
|
||
await createNote("<p>zqnnunoptimized content</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.isFalse(await Zotero.FullText.optimizeContentIndex());
|
||
});
|
||
|
||
it("should exclude a matching note with doesNotContain", async function () {
|
||
let item = await createNote("<p>zqnnexcludable content</p>");
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
let s = new Zotero.Search();
|
||
s.libraryID = item.libraryID;
|
||
s.addCondition('note', 'doesNotContain', 'zqnnexcludable');
|
||
assert.notInclude(await s.search(), item.id);
|
||
});
|
||
|
||
it("should fall back to scanning the HTML until the note backfill finishes", async function () {
|
||
let item = await createNote("<p>zqnnbackfill séance</p>");
|
||
// Rebuild the index (as after a delete-and-resync), leaving every note pending
|
||
// backfill
|
||
await Zotero.DB.queryAsync(
|
||
"UPDATE ftindex.fulltextIndexMeta SET value='xxxxxxxx' WHERE key='localUserKey'"
|
||
);
|
||
await Zotero.DB.vacuum({ force: true });
|
||
// The reconnect's index setup (which drops the mismatched tables) runs on the next
|
||
// query, as at startup before searches resume
|
||
await Zotero.DB.queryAsync("SELECT 1");
|
||
// The note isn't in the index yet, but an exact term still matches via the scan
|
||
assert.include(await noteSearch(item, 'zqnnbackfill'), item.id);
|
||
// Draining the queue enables diacritic-insensitive index matching
|
||
await Zotero.FullText.processNoteIndexQueue();
|
||
assert.include(await noteSearch(item, 'zqnnbackfill seance'), item.id);
|
||
});
|
||
});
|
||
});
|
||
|
||
describe("Indexing", function () {
|
||
beforeEach(function () {
|
||
Zotero.Prefs.clear('fulltext.textMaxLength');
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
});
|
||
after(function () {
|
||
Zotero.Prefs.clear('fulltext.textMaxLength');
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
});
|
||
|
||
describe("#indexItems()", function () {
|
||
it("should index a text file by default", async function () {
|
||
var item = await importFileAttachment('test.txt');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_INDEXED
|
||
);
|
||
})
|
||
|
||
it("should skip indexing of a text file if fulltext.textMaxLength is 0", async function () {
|
||
Zotero.Prefs.set('fulltext.textMaxLength', 0);
|
||
var item = await importFileAttachment('test.txt');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_UNINDEXED
|
||
);
|
||
})
|
||
|
||
it("should index a PDF by default", async function () {
|
||
var item = await importFileAttachment('test.pdf');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_INDEXED
|
||
);
|
||
})
|
||
|
||
it("should skip indexing of a PDF if fulltext.textMaxLength is 0", async function () {
|
||
Zotero.Prefs.set('fulltext.textMaxLength', 0);
|
||
var item = await importFileAttachment('test.pdf');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_UNINDEXED
|
||
);
|
||
})
|
||
|
||
it("should skip indexing of a PDF if fulltext.pdfMaxPages is 0", async function () {
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
|
||
var item = await importFileAttachment('test.pdf');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_UNINDEXED
|
||
);
|
||
})
|
||
|
||
it("should skip indexing of an EPUB if fulltext.textMaxLength is 0", async function () {
|
||
Zotero.Prefs.set('fulltext.textMaxLength', 0);
|
||
var item = await importFileAttachment('recognizeEPUB_test_content.epub');
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_UNINDEXED
|
||
);
|
||
});
|
||
|
||
it("should still work after the DB connection is reopened", async function () {
|
||
var item = await importFileAttachment('test.txt');
|
||
await Zotero.DB.vacuum({ force: true });
|
||
await Zotero.Fulltext.indexItems([item.id]);
|
||
assert.equal(
|
||
((await Zotero.Fulltext.getIndexedState(item))),
|
||
Zotero.Fulltext.INDEX_STATE_INDEXED
|
||
);
|
||
});
|
||
|
||
describe("Indexing with HiddenBrowser", () => {
|
||
it("should index attachment as its attachmentContentType when supported", async function () {
|
||
// Firefox would normally load this as text/x-shellscript, but we detect text/plain
|
||
let item = await importFileAttachment('test.sh');
|
||
assert.equal(item.attachmentContentType, 'text/plain');
|
||
assert.equal(await Zotero.Fulltext.getIndexedState(item), Zotero.Fulltext.INDEX_STATE_INDEXED);
|
||
});
|
||
|
||
it("should index attachment as text/plain when its text/* attachmentContentType is unsupported", async function () {
|
||
// Now we force text/x-shellscript, which the HiddenBrowser would normally refuse to load
|
||
// It should still load, because we fall back to text/plain from an unsupported text/* content type
|
||
let item = await importFileAttachment('test.sh', { contentType: 'text/x-shellscript' });
|
||
assert.equal(item.attachmentContentType, 'text/x-shellscript');
|
||
assert.equal(await Zotero.Fulltext.getIndexedState(item), Zotero.Fulltext.INDEX_STATE_INDEXED);
|
||
});
|
||
|
||
it("should not index attachment with non-text attachmentContentType", async function () {
|
||
let item = await importFileAttachment('test.txt', { contentType: 'image/png' });
|
||
assert.equal(item.attachmentContentType, 'image/png');
|
||
assert.equal(await Zotero.Fulltext.getIndexedState(item), Zotero.Fulltext.INDEX_STATE_UNINDEXED);
|
||
});
|
||
});
|
||
});
|
||
|
||
describe("#indexPDF()", function () {
|
||
it("should create cache files for linked attachments in storage directory", async function () {
|
||
var filename = 'test.pdf';
|
||
var file = OS.Path.join(getTestDataDirectory().path, filename);
|
||
var tempDir = await getTempDirectory();
|
||
var linkedFile = OS.Path.join(tempDir, filename);
|
||
await OS.File.copy(file, linkedFile);
|
||
|
||
var item = await Zotero.Attachments.linkFromFile({ file: linkedFile });
|
||
var storageDir = Zotero.Attachments.getStorageDirectory(item).path;
|
||
assert.isTrue(await OS.File.exists(storageDir));
|
||
assert.isTrue(await OS.File.exists(OS.Path.join(storageDir, '.zotero-ft-cache')));
|
||
assert.isFalse(await OS.File.exists(OS.Path.join(storageDir, filename)));
|
||
});
|
||
|
||
it("should preserve the SDT cache when reindexing a linked attachment", async function () {
|
||
var file = OS.Path.join(getTestDataDirectory().path, 'test.pdf');
|
||
var linkedFile = OS.Path.join(await getTempDirectory(), 'test.pdf');
|
||
await OS.File.copy(file, linkedFile);
|
||
var item = await Zotero.Attachments.linkFromFile({ file: linkedFile });
|
||
|
||
// The full-text cache of a linked file shares the item's
|
||
// storage directory with the SDT cache, so reindexing must
|
||
// not recreate the directory and destroy it
|
||
var storageDir = Zotero.Attachments.getStorageDirectory(item).path;
|
||
var sdtCacheFile = OS.Path.join(storageDir, '.zotero-sdt-cache');
|
||
await Zotero.File.putContentsAsync(sdtCacheFile, 'test');
|
||
|
||
assert.isTrue(await Zotero.Fulltext.indexPDF(linkedFile, item.id));
|
||
assert.isTrue(await OS.File.exists(sdtCacheFile));
|
||
});
|
||
});
|
||
});
|
||
|
||
describe("#getUnsyncedContent()", function () {
|
||
it("should get content that hasn't been uploaded", async function () {
|
||
var toSync = [];
|
||
var group = await getGroup();
|
||
|
||
var add = async function (options = {}) {
|
||
let item = await createDataObject('item', { libraryID: options.libraryID });
|
||
let attachment = new Zotero.Item('attachment');
|
||
if (options.libraryID) {
|
||
attachment.libraryID = options.libraryID;
|
||
}
|
||
attachment.parentItemID = item.id;
|
||
attachment.attachmentLinkMode = 'imported_file';
|
||
attachment.attachmentContentType = 'text/plain';
|
||
attachment.attachmentCharset = 'utf-8';
|
||
attachment.attachmentFilename = 'test.txt';
|
||
if (options.synced) {
|
||
attachment.synced = true;
|
||
}
|
||
await attachment.saveTx();
|
||
await Zotero.Attachments.createDirectoryForItem(attachment);
|
||
|
||
let path = attachment.getFilePath();
|
||
let content = new Array(10).fill("").map(x => Zotero.Utilities.randomString()).join(" ");
|
||
await Zotero.File.putContentsAsync(path, content);
|
||
|
||
if (!options.skip) {
|
||
toSync.push({
|
||
item: attachment,
|
||
content,
|
||
indexedChars: content.length,
|
||
indexedPages: 0
|
||
});
|
||
}
|
||
};
|
||
await add({ synced: true });
|
||
await add({ synced: true });
|
||
// Unsynced attachment shouldn't uploaded
|
||
await add({ skip: true });
|
||
// Attachment in another library shouldn't be uploaded
|
||
await add({ libraryID: group.libraryID, synced: true, skip: true });
|
||
// PDF attachment
|
||
var pdfAttachment = await importFileAttachment('test.pdf');
|
||
pdfAttachment.synced = true;
|
||
await pdfAttachment.saveTx();
|
||
toSync.push({
|
||
item: pdfAttachment,
|
||
content: "Zotero [zoh-TAIR-oh] is a free, easy-to-use tool to help you collect, "
|
||
+ "organize, cite, and share your research sources.",
|
||
indexedChars: 0,
|
||
indexedPages: 1
|
||
});
|
||
|
||
await Zotero.Fulltext.indexItems(toSync.map(x => x.item.id));
|
||
|
||
var data = await Zotero.FullText.getUnsyncedContent(Zotero.Libraries.userLibraryID);
|
||
assert.lengthOf(data, 3);
|
||
let contents = toSync.map(x => x.content);
|
||
|
||
for (let d of data) {
|
||
assert.include(contents, d.content);
|
||
let pos = contents.indexOf(d.content);
|
||
assert.equal(d.indexedChars, toSync[pos].indexedChars);
|
||
assert.equal(d.indexedPages, toSync[pos].indexedPages);
|
||
}
|
||
});
|
||
|
||
it("should mark PDF attachment content as missing if cache file doesn't exist", async function () {
|
||
var item = await importFileAttachment('test.pdf');
|
||
item.synced = true;
|
||
await item.saveTx();
|
||
|
||
await Zotero.Fulltext.indexItems([item.id]);
|
||
await OS.File.remove(Zotero.Fulltext.getItemCacheFile(item).path);
|
||
|
||
var sql = "SELECT synced FROM fulltextItems WHERE itemID=?";
|
||
var synced = await Zotero.DB.valueQueryAsync(sql, item.id);
|
||
assert.equal(synced, Zotero.Fulltext.SYNC_STATE_UNSYNCED);
|
||
var indexed = await Zotero.Fulltext.getIndexedState(item);
|
||
assert.equal(indexed, Zotero.Fulltext.INDEX_STATE_INDEXED);
|
||
|
||
await Zotero.Fulltext.getUnsyncedContent(item.libraryID);
|
||
|
||
synced = await Zotero.DB.valueQueryAsync(sql, item.id);
|
||
assert.equal(synced, Zotero.Fulltext.SYNC_STATE_MISSING);
|
||
indexed = await Zotero.Fulltext.getIndexedState(item);
|
||
assert.equal(indexed, Zotero.Fulltext.INDEX_STATE_UNINDEXED);
|
||
});
|
||
})
|
||
|
||
describe("#processSyncedContentNow()", function () {
|
||
before(() => {
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
|
||
});
|
||
after(() => {
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
});
|
||
|
||
it("should index sync-delivered content without waiting for idle", async function () {
|
||
var item = await importFileAttachment('test.pdf');
|
||
// Simulate content delivered by sync: stored as TO_PROCESS with a processor cache file,
|
||
// not yet in the search index (PDF indexing is disabled, so it wasn't indexed locally)
|
||
await Zotero.FullText.setItemContent(
|
||
item.libraryID,
|
||
item.key,
|
||
{ content: "zqpromptsync electrophoresis", indexedChars: 28, totalChars: 28 },
|
||
5
|
||
);
|
||
function search(term) {
|
||
let s = new Zotero.Search();
|
||
s.libraryID = item.libraryID;
|
||
s.addCondition('fulltextContent', 'contains', term);
|
||
return s.search();
|
||
}
|
||
assert.notInclude(await search('zqpromptsync'), item.id);
|
||
|
||
// Processing it promptly (as on sync completion) makes it searchable without idle
|
||
await Zotero.FullText.processSyncedContentNow();
|
||
|
||
assert.include(await search('zqpromptsync'), item.id);
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync("SELECT synced FROM fulltextItems WHERE itemID=?", item.id),
|
||
Zotero.FullText.SYNC_STATE_IN_SYNC
|
||
);
|
||
});
|
||
});
|
||
|
||
describe("#setItemContent()", function () {
|
||
before(() => {
|
||
// Disable PDF indexing
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
|
||
});
|
||
|
||
after(() => {
|
||
// Re-enable PDF indexing
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
});
|
||
|
||
it("should store data in .zotero-ft-unprocessed file", async function () {
|
||
var item = await importFileAttachment('test.pdf');
|
||
|
||
var processorCacheFile = Zotero.Fulltext.getSyncedContentCacheFile(item).path;
|
||
|
||
var version = 5;
|
||
await Zotero.Fulltext.setItemContent(
|
||
item.libraryID,
|
||
item.key,
|
||
{
|
||
content: "Test",
|
||
indexedPages: 4,
|
||
totalPages: 4
|
||
},
|
||
version
|
||
);
|
||
|
||
assert.equal(await Zotero.Fulltext.getItemVersion(item.id), 0);
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync("SELECT synced FROM fulltextItems WHERE itemID=?", item.id),
|
||
Zotero.FullText.SYNC_STATE_TO_PROCESS
|
||
);
|
||
assert.isTrue(await OS.File.exists(processorCacheFile));
|
||
});
|
||
|
||
|
||
it("should update the version if the local version is 0 but the text matches", async function () {
|
||
var item = await importFileAttachment('test.pdf');
|
||
|
||
await Zotero.DB.queryAsync(
|
||
"REPLACE INTO fulltextItems (itemID, version, indexedPages, totalPages, synced) "
|
||
+ "VALUES (?, 0, 4, 4, ?)",
|
||
[item.id, Zotero.FullText.SYNC_STATE_UNSYNCED]
|
||
);
|
||
|
||
var processorCacheFile = Zotero.FullText.getSyncedContentCacheFile(item).path;
|
||
var itemCacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
await Zotero.File.putContentsAsync(itemCacheFile, "Test");
|
||
|
||
var version = 5;
|
||
await Zotero.FullText.setItemContent(
|
||
item.libraryID,
|
||
item.key,
|
||
{
|
||
content: "Test",
|
||
indexedPages: 4,
|
||
totalPages: 4
|
||
},
|
||
version
|
||
);
|
||
|
||
assert.equal(await Zotero.FullText.getItemVersion(item.id), version);
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync("SELECT synced FROM fulltextItems WHERE itemID=?", item.id),
|
||
Zotero.FullText.SYNC_STATE_IN_SYNC
|
||
);
|
||
var { indexedPages, total } = await Zotero.FullText.getPages(item.id);
|
||
assert.equal(indexedPages, 4);
|
||
assert.equal(total, 4);
|
||
assert.isFalse(await OS.File.exists(processorCacheFile));
|
||
});
|
||
});
|
||
|
||
describe("#rebuildIndex()", function () {
|
||
afterEach(() => {
|
||
// Re-enable PDF indexing
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
});
|
||
|
||
it("should process queued full-text content in indexedOnly mode", async function () {
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
|
||
var item = await importFileAttachment('test.pdf');
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
|
||
var version = 5;
|
||
await Zotero.FullText.setItemContent(
|
||
item.libraryID,
|
||
item.key,
|
||
{
|
||
content: "Test",
|
||
indexedPages: 4,
|
||
totalPages: 4
|
||
},
|
||
version
|
||
);
|
||
|
||
var processorCacheFile = Zotero.FullText.getSyncedContentCacheFile(item).path;
|
||
var itemCacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
|
||
assert.isTrue(await OS.File.exists(processorCacheFile));
|
||
|
||
await Zotero.FullText.rebuildIndex(true);
|
||
|
||
// .zotero-ft-unprocessed should have been deleted
|
||
assert.isFalse(await OS.File.exists(processorCacheFile));
|
||
// .zotero-ft-cache should now exist
|
||
assert.isTrue(await OS.File.exists(itemCacheFile));
|
||
|
||
assert.equal(await Zotero.FullText.getItemVersion(item.id), version);
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync("SELECT synced FROM fulltextItems WHERE itemID=?", item.id),
|
||
Zotero.FullText.SYNC_STATE_IN_SYNC
|
||
);
|
||
var { indexedPages, total } = await Zotero.FullText.getPages(item.id);
|
||
assert.equal(indexedPages, 4);
|
||
assert.equal(total, 4);
|
||
});
|
||
|
||
it("should ignore queued full-text content in non-indexedOnly mode", async function () {
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 0);
|
||
var item = await importFileAttachment('test.pdf');
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
|
||
var version = 5;
|
||
await Zotero.FullText.setItemContent(
|
||
item.libraryID,
|
||
item.key,
|
||
{
|
||
content: "Test",
|
||
indexedPages: 4,
|
||
totalPages: 4
|
||
},
|
||
version
|
||
);
|
||
|
||
var processorCacheFile = Zotero.FullText.getSyncedContentCacheFile(item).path;
|
||
var itemCacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
|
||
assert.isTrue(await OS.File.exists(processorCacheFile));
|
||
|
||
await Zotero.FullText.rebuildIndex();
|
||
|
||
// .zotero-ft-unprocessed should have been deleted
|
||
assert.isFalse(await OS.File.exists(processorCacheFile));
|
||
// .zotero-ft-cache should now exist
|
||
assert.isTrue(await OS.File.exists(itemCacheFile));
|
||
|
||
// Processor cache file shouldn't have been used, and full text should be marked for
|
||
// syncing
|
||
assert.equal(await Zotero.FullText.getItemVersion(item.id), 0);
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync(
|
||
"SELECT synced FROM fulltextItems WHERE itemID=?",
|
||
item.id
|
||
),
|
||
Zotero.FullText.SYNC_STATE_UNSYNCED
|
||
);
|
||
var { indexedPages, total } = await Zotero.FullText.getPages(item.id);
|
||
assert.equal(indexedPages, 1);
|
||
assert.equal(total, 1);
|
||
});
|
||
|
||
// This shouldn't happen, but before 5.0.85 items reindexed elsewhere could clear local stats
|
||
it("shouldn't clear indexed items with missing file and no stats", async function () {
|
||
Zotero.Prefs.set('fulltext.pdfMaxPages', 1);
|
||
var item = await importFileAttachment('test.pdf');
|
||
Zotero.Prefs.clear('fulltext.pdfMaxPages');
|
||
|
||
var itemCacheFile = Zotero.FullText.getItemCacheFile(item).path;
|
||
assert.isTrue(await OS.File.exists(itemCacheFile));
|
||
|
||
var { indexedPages, total } = await Zotero.FullText.getPages(item.id);
|
||
assert.equal(indexedPages, 1);
|
||
assert.equal(total, 1);
|
||
await Zotero.DB.queryAsync(
|
||
"UPDATE fulltextItems SET indexedPages=NULL, totalPages=NULL WHERE itemID=?",
|
||
item.id
|
||
);
|
||
|
||
await Zotero.FullText.rebuildIndex();
|
||
|
||
// .zotero-ft-cache should still exist
|
||
assert.isTrue(await OS.File.exists(itemCacheFile));
|
||
|
||
assert.equal(
|
||
await Zotero.DB.valueQueryAsync(
|
||
"SELECT COUNT(*) FROM fulltextItems WHERE itemID=?",
|
||
item.id
|
||
),
|
||
1
|
||
);
|
||
});
|
||
});
|
||
})
|