Don't index items with too little text to say anything

A one-character title carries no signal but still scores as a moderate
match against any query. A single ideograph can be a whole word, so
those are kept.

chrome/content/zotero/xpcom/embeddings.js

@@ -1110,7 +1110,7 @@ Zotero.Embeddings.Indexing = new function () {
 			...Zotero.ItemFields.getTypeFieldsFromBase('title')
 		])];
 		let rows = await Zotero.DB.queryAsync(
-			"SELECT DISTINCT libraryID, itemID FROM itemData "
+			"SELECT libraryID, itemID, value FROM itemData "
 				+ "JOIN itemDataValues USING (valueID) "
 				+ "JOIN items USING (itemID) "
 				+ "WHERE fieldID IN (" + fieldIDs.join(',') + ") "
@@ -1118,7 +1118,12 @@ Zotero.Embeddings.Indexing = new function () {
 			Zotero.ItemTypes.getID('attachment')
 		);
 		let byLibrary = new Map();
+		let seen = new Set();
 		for (let row of rows) {
+			if (seen.has(row.itemID) || !_hasEmbeddableText(row.value)) {
+				continue;
+			}
+			seen.add(row.itemID);
 			let ids = byLibrary.get(row.libraryID);
 			if (!ids) {
 				ids = [];
@@ -1195,6 +1200,20 @@ Zotero.Embeddings.Indexing = new function () {
 		);
 	}
 
+	// Text with too little in it to say anything: a one-character title is
+	// noise in an alphabetic script, while a single ideograph can be a whole
+	// word, so those count.
+	function _hasEmbeddableText(text) {
+		text = (text || '').trim();
+		if (!text) {
+			return false;
+		}
+		if (/[\p{Ideographic}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u.test(text)) {
+			return true;
+		}
+		return [...text].length > 1;
+	}
+
 	// Text we embed for an item: its title and abstract. A title alone is
 	// enough -- it's useful signal even without an abstract. Returns null only
 	// for items with neither.
@@ -1210,8 +1229,7 @@ Zotero.Embeddings.Indexing = new function () {
 
 	// Compute and store embeddings for the given items, skipping any whose
 	// stored embedding is already up to date (via sourceHash). Items with no
-	// embeddable text (neither a title nor an abstract) have any existing
-	// embedding removed.
+	// embeddable text have any existing embedding removed.
 	//
 	// @param {Zotero.Item[]} items
 	// @param {Object} [options]
@@ -1255,7 +1273,7 @@ Zotero.Embeddings.Indexing = new function () {
 		let toDelete = [];
 		for (let item of items) {
 			let text = _getItemText(item);
-			if (!text) {
+			if (!_hasEmbeddableText(text)) {
 				if (storedHashes.has(item.id)) {
 					toDelete.push(item.id);
 				}

test/tests/embeddingsTest.js

@@ -248,6 +248,37 @@ describe("Zotero.Embeddings", function () {
 			}
 		});
 
+		it("should skip items with too little text to say anything", async function () {
+			this.timeout(60000);
+			await createDataObject('item', { title: 'C' });
+			await createDataObject('item', { title: '猫' });
+			await createDataObject('item', { title: 'A study of feline behavior' });
+
+			let vector = new Float32Array(4).fill(0.5);
+			let texts = [];
+			let stubs = [
+				sinon.stub(Zotero.Embeddings, 'embedPassages').callsFake(async (passages) => {
+					texts.push(...passages);
+					return passages.map(() => vector);
+				}),
+				sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true),
+				sinon.stub(Zotero.Embeddings, 'getModelVersion').returns('test-model/1'),
+				sinon.stub(Zotero.Embeddings, 'isDownloaded').resolves(true),
+				sinon.stub(Zotero.Embeddings, 'preloadModel').resolves()
+			];
+			try {
+				await Zotero.Embeddings.Indexing.startIndexing();
+			}
+			finally {
+				stubs.forEach(stub => stub.restore());
+			}
+
+			assert.include(texts, 'A study of feline behavior');
+			// A single ideograph is a word; a single letter isn't
+			assert.include(texts, '猫');
+			assert.notInclude(texts, 'C');
+		});
+
 		it("should look up stored hashes without a query per item", async function () {
 			this.timeout(60000);
 			for (let i = 0; i < 5; i++) {