minor fixes

- cleanup chunking flow

- no not create embeddings db on purely lexical run

- fix preview preloading not in display order

- fix stale test

chrome/content/zotero/xpcom/bestMatch.js

@@ -62,7 +62,7 @@ Zotero.BestMatch = new function () {
 	// the item has to offer at a glance; the rest are still derived --
 	// they're read whole rather than quoted, which needs no line chosen.
 	const MAX_QUOTED_PASSAGES = 3;
-	// Previews derived before scoring resolves, best-scored first: enough to
+	// Previews derived before scoring resolves, in screen order: enough to
 	// cover the top of the results. The rest follow in the background, since
 	// reading every matched item's text takes far longer than the ranking.
 	const PRELOADED_MATCH_PREVIEWS = 10;
@@ -299,9 +299,9 @@ Zotero.BestMatch = new function () {
 	 * A best-match search session: one query's scoring pass plus the
 	 * previews explaining its matches.
 	 *
-	 * score() ranks candidates and derives the best-scored few previews
+	 * score() ranks candidates and derives the first few previews on screen
 	 * (PRELOADED_MATCH_PREVIEWS) before it resolves. The rest derive in the
-	 * background, in score order and paced to stay out of the user's way
+	 * background, in the same order and paced to stay out of the user's way
 	 * (see PREVIEW_PAUSE_RATIO), reported through onPreviewsFilled and
 	 * awaitable through previewsSettled. An item's entries hold both
 	 * engines' evidence, merged, deduplicated and ordered by strength (see
@@ -313,6 +313,7 @@ Zotero.BestMatch = new function () {
 		constructor(queryText) {
 			this._queryText = queryText;
 			this._previews = new Map();
+			this._effectiveScores = new Map();
 			this._disposed = false;
 			// Bumped per background pass, so a re-score abandons the last one
 			this._derivation = 0;
@@ -331,7 +332,7 @@ Zotero.BestMatch = new function () {
 		 * Score candidates for this session's query (see
 		 * Zotero.BestMatch.scoreItemIDs()), rebuild the preview set from the
 		 * engines' match sets, recompute ranks and barFractions, and derive
-		 * the best-scored pending previews before resolving. Items still
+		 * the first pending previews on screen before resolving. Items still
 		 * matched keep their settled previews -- a re-score doesn't re-derive
 		 * kept text -- and items no longer matched lose theirs.
 		 *
@@ -383,10 +384,15 @@ Zotero.BestMatch = new function () {
 			}
 			this._previews = previews;
 			this._rank(scores);
-			let pending = [...scores.entries()]
-				.sort((a, b) => b[1] - a[1])
-				.map(([id]) => id)
-				.filter(id => previews.get(id)?.state == 'pending');
+			// Derive in the order rows appear on screen: an item's preview
+			// rows render under its top-level ancestor, which is ranked by
+			// the best match anywhere beneath it -- an item's own score can
+			// sit far below its position
+			let pending = [...scores.keys()]
+				.filter(id => previews.get(id)?.state == 'pending')
+				.map(id => [id, this._topLevelScore(id), this._effectiveScores.get(id) ?? 0])
+				.sort((a, b) => (b[1] - a[1]) || (b[2] - a[2]) || (a[0] - b[0]))
+				.map(([id]) => id);
 			for (let itemID of pending.slice(0, PRELOADED_MATCH_PREVIEWS)) {
 				if (this._disposed) {
 					return scores;
@@ -411,11 +417,11 @@ Zotero.BestMatch = new function () {
 			return this._previewsSettled;
 		}
 
-		// Derive the previews score() left behind, best-scored first,
-		// reporting them in batches (see onPreviewsFilled). Left unawaited by
-		// score(), so a consumer draws the ranking while the explanations
-		// behind it fill in. A newer pass -- another score() -- or dispose()
-		// abandons this one.
+		// Derive the previews score() left behind, in screen order (see
+		// score()), reporting them in batches (see onPreviewsFilled). Left
+		// unawaited by score(), so a consumer draws the ranking while the
+		// explanations behind it fill in. A newer pass -- another score() --
+		// or dispose() abandons this one.
 		async _deriveRest(itemIDs) {
 			let derivation = ++this._derivation;
 			let filled = [];
@@ -511,6 +517,18 @@ Zotero.BestMatch = new function () {
 			}
 			this._ranks = ranks;
 			this._barFractions = fractions;
+			this._effectiveScores = effectiveScores;
+		}
+
+		// The effective score of an item's top-level ancestor -- what places
+		// the row subtree the item's preview rows render in
+		_topLevelScore(itemID) {
+			let id = itemID;
+			let parentItemID;
+			while ((parentItemID = Zotero.Items.get(id)?.parentItemID)) {
+				id = parentItemID;
+			}
+			return this._effectiveScores.get(id) ?? 0;
 		}
 
 		/**
@@ -709,16 +727,21 @@ Zotero.BestMatch = new function () {
 					}
 				}
 			}
-			// Indexed, but with no model weighing the chunks -- either it
-			// isn't ranking, or it found nothing here worth ranking. They
-			// still say how the item divides, for the words to be found in.
 			if (!this._lexicalApplies(itemID)) {
 				return [];
 			}
-			let indexed = await Zotero.Embeddings.getChunks(itemID);
-			indexed = indexed.filter(chunk => chunk.text);
-			if (indexed.length) {
-				return indexed;
+			// Indexed, but with no model weighing the chunks -- either it
+			// found nothing here worth ranking, or its index isn't ready. The
+			// chunks still say how the item divides, for the words to be
+			// found in. Only asked while a model is enabled: with semantic
+			// search off there's no chunk store, and asking would create the
+			// embeddings database for a purely lexical search.
+			if (Zotero.Embeddings.isEnabled()) {
+				let indexed = await Zotero.Embeddings.getChunks(itemID);
+				indexed = indexed.filter(chunk => chunk.text);
+				if (indexed.length) {
+					return indexed;
+				}
 			}
 			return this._cutPassages(itemID);
 		}
@@ -732,25 +755,19 @@ Zotero.BestMatch = new function () {
 			if (!item) {
 				return [];
 			}
-			let chunking = Zotero.Utilities.Internal.Chunking;
 			// Only structure already extracted: generating it costs seconds,
 			// which is not a price a preview may charge (see
 			// Zotero.SDT.getPack()). Without it the flat text still divides,
 			// just without knowing where its passages sit.
 			let structure = await Zotero.SDT.getSections(itemID, { cachedOnly: true });
 			if (structure.ok && structure.sections.length) {
-				// The metrics only read the text for its script, so a sample
-				// of the opening sections says as much as all of them
-				let sample = structure.sections.slice(0, 5)
-					.map(section => section.text).join('\n\n');
-				return chunking.chunkSections(
-					structure.sections, chunking.getCharacterMetrics(sample));
+				return Zotero.Utilities.Internal.Chunking.chunkSections(structure.sections);
 			}
 			let text = await item.attachmentText;
 			if (!text) {
 				return [];
 			}
-			return chunking.chunkText(text, chunking.getCharacterMetrics(text));
+			return Zotero.Utilities.Internal.Chunking.chunkText(text);
 		}
 
 		/**
@@ -767,10 +784,8 @@ Zotero.BestMatch = new function () {
 		 * @param {Object[]} entries - Set in place
 		 */
 		async _pickSnippets(entries) {
-			let chunking = Zotero.Utilities.Internal.Chunking;
 			for (let entry of entries) {
-				let sentences = chunking.splitSentences(
-					entry.text, chunking.getCharacterMetrics(entry.text));
+				let sentences = Zotero.Utilities.Internal.Chunking.splitSentences(entry.text);
 				// The passage's opening, for a passage with no words to quote
 				// around
 				entry.snippet = sentences.length

chrome/content/zotero/xpcom/embeddings.js

@@ -1502,60 +1502,48 @@ Zotero.Embeddings = new function () {
  * each be embedded on their own, so a long text's later paragraphs are
  * searchable instead of being averaged into one vector or truncated away by the
  * pipeline. The indexer applies this to notes (chunkText()) and to attachment
- * full text (chunkSections()). Chunk size is bounded by CHUNK_MAX_TOKENS,
- * capped by the model's window, and counted in the model's own tokens.
+ * full text (chunkSections()). Chunk size is bounded by the shared passage
+ * geometry's BUDGET_TOKENS (see Zotero.Utilities.Internal.Chunking), capped
+ * by the model's window, and counted in the model's own tokens.
  */
 Zotero.Embeddings.Chunking = new function () {
-	// Tokens carried over from the end of one chunk into the start of the
-	// next, so a thought spanning a boundary is searchable in both. Carried
-	// only between pieces of the same paragraph block.
-	const CHUNK_OVERLAP_TOKENS = 48;
-	// Fewest tokens worth embedding on their own. An item scores as its best
-	// chunk, so every chunk is another draw at that maximum: splitting text
-	// into fragments inflates the score without adding information, and a
-	// fragment loses the context that gave it meaning. Paragraphs below this
-	// (headings, dates, list items) are combined with their neighbors rather
-	// than becoming chunks of their own.
-	const CHUNK_MIN_TOKENS = 120;
-	// Most tokens a chunk may reach. A ceiling rather than a target: chunks come
-	// out paragraph-sized, so this decides only how long a text has to be before
-	// it's split at all, and how far a single oversized paragraph is split.
-	const CHUNK_MAX_TOKENS = 768;
-	// Most characters to feed the tokenizer in one encode() call. The
-	// multilingual models' SentencePiece Unigram tokenizer is quadratic in
-	// input length -- its Metaspace pre-tokenizer doesn't split at whitespace,
-	// so the whole input goes through the Viterbi lattice as one string, and a
-	// single long encode can take seconds. Segments this size keep every call
-	// in the tokenizer's linear regime, and token counts are additive across
-	// whitespace boundaries, so the segmented measurement is exact.
+	// The shared passage geometry (see Zotero.Utilities.Internal.Chunking),
+	// counted here in the model's own tokens: BUDGET_TOKENS caps a chunk,
+	// within what the model's window allows (see _getMetrics()); MIN_TOKENS
+	// is the least worth embedding alone -- an item scores as its best chunk,
+	// so fragments would inflate the score while losing their context; and
+	// OVERLAP_TOKENS is carried across a split, so a thought spanning the
+	// boundary is searchable in both halves.
+	const { BUDGET_TOKENS, MIN_TOKENS, OVERLAP_TOKENS } = Zotero.Utilities.Internal.Chunking;
+	// Most characters per encode() call. The multilingual models'
+	// SentencePiece Unigram tokenizer is quadratic in input length (its
+	// Metaspace pre-tokenizer doesn't split at whitespace), so a single long
+	// encode can take seconds. Segments this size stay in the linear regime,
+	// and token counts are additive across whitespace boundaries, so the
+	// segmented measurement is exact.
 	const TOKENIZER_SEGMENT_CHARS = 1000;
 
 	// Tokenizer instances by model name. Failures aren't cached, so a later
 	// indexing run retries the load.
 	let _tokenizers = new Map();
 
-	// Metrics per tokenizer (see _getMetrics()), and the token counter they're
-	// built on (see _getCounter()). Both cost a handful of encode() calls to
-	// derive, which is nothing alongside a long text but real when many short
-	// texts are measured one at a time -- and both depend only on the
-	// tokenizer, so one derivation serves every text measured against it.
+	// Counters and metrics cached per tokenizer: each costs a few encode()
+	// calls to derive, which adds up when many short texts are measured one
+	// at a time. A tokenizer belongs to one model (see getTokenizer()), so
+	// the model facts baked into its metrics can't go stale.
 	let _metrics = new WeakMap();
 	let _counters = new WeakMap();
 
 	/**
-	 * The active model's tokenizer, constructed from the tokenizer files in
-	 * the runtime's model cache using the transformers.js implementation
-	 * Firefox ships -- the same code, over the same files, that the inference
-	 * process tokenizes with.
+	 * The active model's tokenizer, built from the files in the runtime's
+	 * model cache with the transformers.js implementation Firefox ships --
+	 * the same code, over the same files, that the inference process
+	 * tokenizes with.
 	 *
-	 * The tokenizer files are part of the downloaded model, so with the model
-	 * present this can't fail short of a broken download -- and then inference
-	 * couldn't tokenize either, so nothing would embed anyway. A failure
-	 * therefore throws, aborting the indexing run and surfacing as its error,
-	 * rather than falling back to an approximate count: an approximation would
-	 * only produce chunks that then fail to embed, and chunk boundaries are
-	 * persistent -- an item's source hash covers its text, not the chunker, so
-	 * a run that guessed at them would never be corrected.
+	 * A failure throws rather than falling back to an approximate count:
+	 * approximated chunk boundaries would persist uncorrected (an item's
+	 * source hash covers its text, not the chunker), and with the tokenizer
+	 * files broken nothing would embed anyway.
 	 *
 	 * @return {Promise<Object>}
 	 */
@@ -1587,27 +1575,10 @@ Zotero.Embeddings.Chunking = new function () {
 		return _tokenizers.get(name);
 	};
 
-	// How to measure text against the active model, and how much of its window a
-	// chunk's own text may use.
-	//
-	// Counts leave out the special tokens the tokenizer wraps every input in, so
-	// that the counts of several pieces of text add up to the count of those
-	// pieces joined. That additivity is what the whole chunking flow leans on:
-	// a text is tokenized exactly once, at paragraph granularity
-	// (_measureParagraphs()), and every level above -- blocks, sections, groups
-	// of sections -- is sized by summing those counts (_sumTokens()) instead of
-	// re-tokenizing the same text. `joinTokens` is what one '\n\n' join adds
-	// once paragraphs are put back together, measured rather than assumed
-	// (a SentencePiece model can spend a token on collapsed whitespace), so
-	// sums charge it per join.
-	//
-	// Those special tokens come off the window instead, together with the
-	// passage prefix embedPassages() prepends -- neither is part of the text this
-	// code sees, but both take up room once the chunk is embedded.
-	// Counts a text's tokens the way the chunking flow counts them, leaving out
-	// the special tokens the tokenizer wraps every input in so that counts are
-	// additive. Depends on the tokenizer alone -- not on which model is
-	// configured -- so text can be measured wherever a tokenizer is available.
+	// A counter of a text's tokens that leaves out the special tokens the
+	// tokenizer wraps every input in, so counts of several pieces add up to
+	// the count of the pieces joined -- the additivity that lets the shared
+	// chunker measure each paragraph once and size every grouping by summing.
 	function _getCounter(tokenizer) {
 		let counter = _counters.get(tokenizer);
 		if (!counter) {
@@ -1624,6 +1595,12 @@ Zotero.Embeddings.Chunking = new function () {
 		return counter;
 	}
 
+	// The shared chunker's metrics for the active model. joinSize is what one
+	// '\n\n' join adds, measured rather than assumed (a SentencePiece model
+	// can spend a token on collapsed whitespace). The budget is the shared
+	// ceiling, capped by the model's window, less the special tokens and the
+	// passage prefix embedPassages() prepends -- neither is part of the text
+	// the chunker sees, but both take up window once the chunk is embedded.
 	function _getMetrics(tokenizer) {
 		let cached = _metrics.get(tokenizer);
 		if (cached) {
@@ -1635,10 +1612,10 @@ Zotero.Embeddings.Chunking = new function () {
 		let metrics = {
 			count,
 			joinSize: Math.max(0, count('a\n\na') - 2 * count('a')),
-			budget: Math.min(CHUNK_MAX_TOKENS, Zotero.Embeddings.getModelMaxTokens())
+			budget: Math.min(BUDGET_TOKENS, Zotero.Embeddings.getModelMaxTokens())
 				- specialTokens - (prefix ? count(prefix) : 0),
-			minSize: CHUNK_MIN_TOKENS,
-			overlap: CHUNK_OVERLAP_TOKENS
+			minSize: MIN_TOKENS,
+			overlap: OVERLAP_TOKENS
 		};
 		_metrics.set(tokenizer, metrics);
 		return metrics;
@@ -1686,10 +1663,9 @@ Zotero.Embeddings.Chunking = new function () {
 	};
 	
 	/**
-	 * Token count of a text against the active model, leaving out the special
-	 * tokens the tokenizer wraps every input in, so that counts of several
-	 * pieces of text add up to the count of those pieces joined -- the same
-	 * measure the chunkers report on the chunks they return.
+	 * Token count of a text against the active model, without the special
+	 * tokens the tokenizer wraps every input in -- the same additive measure
+	 * the chunkers report on their chunks.
 	 *
 	 * @param {String} text
 	 * @return {Promise<Number>}

chrome/content/zotero/xpcom/utilities_internal.js

@@ -3354,29 +3354,41 @@ Zotero.Utilities.Internal.onDragItems = function (event, itemIDs, dragImage = ev
 
 
 /**
- * Zotero.Utilities.Internal.Chunking -- splitting a text into passages of a
- * size the caller sets, measured however the caller measures (see
- * Zotero.Embeddings.Chunking, which measures in a model's tokens).
+ * Zotero.Utilities.Internal.Chunking -- splitting a text into passages.
  *
- * Metrics: { count(text), joinSize (charged per join, so pieces' sizes add up
+ * Passages are measured in characters unless the caller supplies its own
+ * metrics (see Zotero.Embeddings.Chunking, which measures in a model's
+ * tokens): { count(text), joinSize (charged per join, so pieces' sizes add up
  * to the joined size), budget (most a passage may reach), minSize (least
  * worth standing alone), overlap (carried into the next passage where one is
- * split mid-thought) }.
+ * split mid-thought) }. The character default (see getCharacterMetrics())
+ * approximates the same geometry, so it suits cuts made and discarded in one
+ * sitting; boundaries that get stored should be measured exactly, with the
+ * caller's own metrics.
  *
  * Every passage is a slice of its source, located by its start/end extent.
  */
 Zotero.Utilities.Internal.Chunking = new function () {
-	// The token budgets the character measure mirrors, and what a token is
-	// worth in characters in either kind of script
+	// The passage geometry, in tokens: the most a passage may hold
+	// (BUDGET_TOKENS), the least text worth standing alone rather than being
+	// combined with a neighbor (MIN_TOKENS), and how much of a split
+	// passage's tail is carried into the next one (OVERLAP_TOKENS). Exported
+	// as the single definition of passage size, whichever measure a set of
+	// metrics counts it in.
 	const BUDGET_TOKENS = 768;
 	const MIN_TOKENS = 120;
 	const OVERLAP_TOKENS = 48;
+	this.BUDGET_TOKENS = BUDGET_TOKENS;
+	this.MIN_TOKENS = MIN_TOKENS;
+	this.OVERLAP_TOKENS = OVERLAP_TOKENS;
+	// What a token is worth in characters in either kind of script
 	const CHARS_PER_TOKEN = 4;
 	const CJK_CHARS_PER_TOKEN = 1;
 
 	/**
-	 * A measure counting characters, for a consumer with no tokenizer. The
-	 * budgets mirror the token ones, so passages come out about the size a
+	 * A measure counting characters, for a consumer with no tokenizer -- what
+	 * the chunkers here default to when no metrics are given. The budgets
+	 * mirror the token geometry above, so passages come out about the size a
 	 * model-driven chunker would make them.
 	 *
 	 * @param {String} text - Read for the script it's written in
@@ -3394,6 +3406,12 @@ Zotero.Utilities.Internal.Chunking = new function () {
 		};
 	};
 
+	// The character measure over the text a call is about to chunk, for a
+	// call that gave no metrics of its own
+	function _defaultMetrics(text) {
+		return Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text);
+	}
+
 	// CJK is written without spaces, so a character of it carries about what
 	// a word of an alphabetic script does
 	function _isMostlyCJK(text) {
@@ -3503,11 +3521,13 @@ Zotero.Utilities.Internal.Chunking = new function () {
 	 * truncation rather than as a passage.
 	 *
 	 * @param {String} text
-	 * @param {Object} metrics - See getCharacterMetrics(); only `count` is read
+	 * @param {Object} [metrics] - Only `count` is read; the character measure
+	 *     when omitted
 	 * @return {Object[]} - { text, size, start, end } per sentence, trimmed,
 	 *     with whitespace-only segments dropped
 	 */
 	this.splitSentences = function (text, metrics) {
+		metrics = metrics || _defaultMetrics(text);
 		return _segmentSentences(text, 0, text.length, metrics.count);
 	};
 
@@ -3595,10 +3615,11 @@ Zotero.Utilities.Internal.Chunking = new function () {
 	 * pieces at sentence boundaries.
 	 *
 	 * @param {String} text
-	 * @param {Object} metrics
+	 * @param {Object} [metrics] - The character measure when omitted
 	 * @return {Object[]} - [{ text, size, start, end }]
 	 */
 	this.chunkText = function (text, metrics) {
+		metrics = metrics || _defaultMetrics(text);
 		let paragraphs = _measureParagraphs(text, metrics.count);
 		return _chunkParagraphs(text, paragraphs, metrics.budget, metrics);
 	};
@@ -3684,12 +3705,16 @@ Zotero.Utilities.Internal.Chunking = new function () {
 	 * @param {Object[]} sections - [{ text, outlinePath, startBlock,
 	 *     auxiliary, blocks: [{ index, text, pageIndex, pageLabel,
 	 *     position }] }]
+	 * @param {Object} [metrics] - The character measure when omitted
 	 * @return {Object[]} - [{ text, embedText, size, outlinePath,
 	 *     startBlock, endBlock, startOffset, endOffset, pageIndex, pageLabel,
 	 *     position, sectionPart, sectionParts, auxiliary }], where size
 	 *     counts embedText
 	 */
 	this.chunkSections = function (sections, metrics) {
+		if (!metrics) {
+			metrics = _defaultMetrics(sections.map(section => section.text).join('\n\n'));
+		}
 		let { count, joinSize, budget } = metrics;
 
 		// Group sections into chunk-worthy units, combining any too small to

test/tests/bestMatchTest.js

@@ -552,6 +552,22 @@ describe("Zotero.BestMatch", function () {
 			assert.lengthOf(await session.getMatchingExcerpts(attachment.id), 1);
 		});
 
+		it("shouldn't touch the chunk store while no model is enabled", async function () {
+			stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(false));
+			let chunksStub = sinon.stub(Zotero.Embeddings, 'getChunks');
+			stubs.push(chunksStub);
+			stubs.push(sinon.stub(Zotero.SDT, 'getSections').resolves({ ok: false, reason: 'none' }));
+			stubs.push(sinon.stub(Zotero.Items, 'getAsync')
+				.resolves({ attachmentText: Promise.resolve('A paragraph about the owl.') }));
+
+			let session = await sessionFor();
+			let excerpts = await session.getMatchingExcerpts(attachment.id);
+			// Reading stored chunks would attach the embeddings database,
+			// which a purely lexical search shouldn't create
+			assert.isFalse(chunksStub.called);
+			assert.isAbove(excerpts.length, 0);
+		});
+
 		it("should rethrow an unexpected semantic failure", async function () {
 			stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true));
 			stubs.push(sinon.stub(Zotero.Embeddings, 'getMatchingChunks')
@@ -833,6 +849,32 @@ describe("Zotero.BestMatch", function () {
 			assert.sameMembers(reported, atts.slice(PRELOADED).map(att => att.id));
 		});
 
+		it("should preload previews by where rows appear, not by items' own scores", async function () {
+			let parent = await createDataObject('item');
+			let child = await importFileAttachment('test.pdf', { parentItemID: parent.id });
+			// The child scores worst of every match, but its preview rows
+			// render under the top-ranked parent, at the top of the list
+			let scores = new Map(atts.map((att, i) => [att.id, 0.9 - i / 100]));
+			scores.set(parent.id, 0.99);
+			scores.set(child.id, 0.01);
+			stubs.push(sinon.stub(Zotero.BestMatch, 'scoreItemIDs').resolves({
+				scores,
+				matches: { lexical: new Set(scores.keys()), semantic: new Set() }
+			}));
+			let { release, advanceTo } = stubDerive();
+			let session = Zotero.BestMatch.createSession('owl');
+			let scored = session.score([...scores.keys()]);
+			await advanceTo(PRELOADED + 1);
+			await scored;
+
+			assert.equal(session.getPreviews(child.id).state, 'filled');
+			// The weakest standalone attachment waited instead
+			assert.equal(session.getPreviews(atts[atts.length - 1].id).state, 'pending');
+
+			session.dispose();
+			release();
+		});
+
 		it("should stop the background pass when the session is disposed", async function () {
 			stubScore();
 			let { stub, release, advanceTo } = stubDerive();

test/tests/embeddingsTest.js

@@ -478,7 +478,7 @@ describe("Zotero.Embeddings", function () {
 		// that wrap every input is what a chunk's own text gets (see MODELS).
 		// bge's window is under the chunking ceiling, so the window governs here.
 		const BUDGET = 512 - 2;
-		// CHUNK_MAX_TOKENS less those same special tokens
+		// The shared BUDGET_TOKENS geometry less those same special tokens
 		const CEILING = 768 - 2;
 		var fakeTokenizer = wordTokenizer();
 		// A chunk's own tokens, the way chunking counts them
@@ -595,7 +595,7 @@ describe("Zotero.Embeddings", function () {
 			// A block only ever closes on a sentence boundary, so each piece
 			// lands a little under its target. Without spreading that slack over
 			// the pieces still to come, it accumulates into an extra runt piece
-			// -- which is what CHUNK_MIN_TOKENS exists to prevent.
+			// -- which is what the MIN_TOKENS geometry exists to prevent.
 			for (let count of [45, 64, 83, 97, 140]) {
 				let sentences = Array.from({ length: count },
 					(x, i) => `Sentence ${i} has a few more words in it about subject ${i}.`);

test/tests/utilities_internalTest.js

@@ -758,8 +758,27 @@ describe("Zotero.Utilities.Internal", function () {
 		});
 	});
 	describe("Chunking", function () {
-		var chunk = text => Zotero.Utilities.Internal.Chunking.chunkText(
-			text, Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text));
+		var chunk = text => Zotero.Utilities.Internal.Chunking.chunkText(text);
+
+		it("should chunk with the character measure when no metrics are given", function () {
+			let text = ('word '.repeat(60) + '\n').repeat(40);
+			let explicit = Zotero.Utilities.Internal.Chunking.chunkText(
+				text, Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text));
+			assert.deepEqual(chunk(text), explicit);
+		});
+
+		it("should mirror the exported geometry in its character defaults", function () {
+			let chunking = Zotero.Utilities.Internal.Chunking;
+			// One scale for all three numbers, so the character measure can't
+			// drift from the geometry it stands in for
+			for (let text of ['climate change', '気候変動'.repeat(10)]) {
+				let metrics = chunking.getCharacterMetrics(text);
+				let scale = metrics.budget / chunking.BUDGET_TOKENS;
+				assert.isAbove(scale, 0);
+				assert.equal(metrics.minSize, chunking.MIN_TOKENS * scale);
+				assert.equal(metrics.overlap, chunking.OVERLAP_TOKENS * scale);
+			}
+		});
 
 		it("should leave a text within the budget whole", function () {
 			let chunks = chunk('A short paragraph.');