diff --git a/chrome/content/zotero/xpcom/bestMatch.js b/chrome/content/zotero/xpcom/bestMatch.js index ba27e835d3..d3b54ae618 100644 --- a/chrome/content/zotero/xpcom/bestMatch.js +++ b/chrome/content/zotero/xpcom/bestMatch.js @@ -62,7 +62,7 @@ Zotero.BestMatch = new function () { // the item has to offer at a glance; the rest are still derived -- // they're read whole rather than quoted, which needs no line chosen. const MAX_QUOTED_PASSAGES = 3; - // Previews derived before scoring resolves, best-scored first: enough to + // Previews derived before scoring resolves, in screen order: enough to // cover the top of the results. The rest follow in the background, since // reading every matched item's text takes far longer than the ranking. const PRELOADED_MATCH_PREVIEWS = 10; @@ -299,9 +299,9 @@ Zotero.BestMatch = new function () { * A best-match search session: one query's scoring pass plus the * previews explaining its matches. * - * score() ranks candidates and derives the best-scored few previews + * score() ranks candidates and derives the first few previews on screen * (PRELOADED_MATCH_PREVIEWS) before it resolves. The rest derive in the - * background, in score order and paced to stay out of the user's way + * background, in the same order and paced to stay out of the user's way * (see PREVIEW_PAUSE_RATIO), reported through onPreviewsFilled and * awaitable through previewsSettled. An item's entries hold both * engines' evidence, merged, deduplicated and ordered by strength (see @@ -313,6 +313,7 @@ Zotero.BestMatch = new function () { constructor(queryText) { this._queryText = queryText; this._previews = new Map(); + this._effectiveScores = new Map(); this._disposed = false; // Bumped per background pass, so a re-score abandons the last one this._derivation = 0; @@ -331,7 +332,7 @@ Zotero.BestMatch = new function () { * Score candidates for this session's query (see * Zotero.BestMatch.scoreItemIDs()), rebuild the preview set from the * engines' match sets, recompute ranks and barFractions, and derive - * the best-scored pending previews before resolving. Items still + * the first pending previews on screen before resolving. Items still * matched keep their settled previews -- a re-score doesn't re-derive * kept text -- and items no longer matched lose theirs. * @@ -383,10 +384,15 @@ Zotero.BestMatch = new function () { } this._previews = previews; this._rank(scores); - let pending = [...scores.entries()] - .sort((a, b) => b[1] - a[1]) - .map(([id]) => id) - .filter(id => previews.get(id)?.state == 'pending'); + // Derive in the order rows appear on screen: an item's preview + // rows render under its top-level ancestor, which is ranked by + // the best match anywhere beneath it -- an item's own score can + // sit far below its position + let pending = [...scores.keys()] + .filter(id => previews.get(id)?.state == 'pending') + .map(id => [id, this._topLevelScore(id), this._effectiveScores.get(id) ?? 0]) + .sort((a, b) => (b[1] - a[1]) || (b[2] - a[2]) || (a[0] - b[0])) + .map(([id]) => id); for (let itemID of pending.slice(0, PRELOADED_MATCH_PREVIEWS)) { if (this._disposed) { return scores; @@ -411,11 +417,11 @@ Zotero.BestMatch = new function () { return this._previewsSettled; } - // Derive the previews score() left behind, best-scored first, - // reporting them in batches (see onPreviewsFilled). Left unawaited by - // score(), so a consumer draws the ranking while the explanations - // behind it fill in. A newer pass -- another score() -- or dispose() - // abandons this one. + // Derive the previews score() left behind, in screen order (see + // score()), reporting them in batches (see onPreviewsFilled). Left + // unawaited by score(), so a consumer draws the ranking while the + // explanations behind it fill in. A newer pass -- another score() -- + // or dispose() abandons this one. async _deriveRest(itemIDs) { let derivation = ++this._derivation; let filled = []; @@ -511,6 +517,18 @@ Zotero.BestMatch = new function () { } this._ranks = ranks; this._barFractions = fractions; + this._effectiveScores = effectiveScores; + } + + // The effective score of an item's top-level ancestor -- what places + // the row subtree the item's preview rows render in + _topLevelScore(itemID) { + let id = itemID; + let parentItemID; + while ((parentItemID = Zotero.Items.get(id)?.parentItemID)) { + id = parentItemID; + } + return this._effectiveScores.get(id) ?? 0; } /** @@ -709,16 +727,21 @@ Zotero.BestMatch = new function () { } } } - // Indexed, but with no model weighing the chunks -- either it - // isn't ranking, or it found nothing here worth ranking. They - // still say how the item divides, for the words to be found in. if (!this._lexicalApplies(itemID)) { return []; } - let indexed = await Zotero.Embeddings.getChunks(itemID); - indexed = indexed.filter(chunk => chunk.text); - if (indexed.length) { - return indexed; + // Indexed, but with no model weighing the chunks -- either it + // found nothing here worth ranking, or its index isn't ready. The + // chunks still say how the item divides, for the words to be + // found in. Only asked while a model is enabled: with semantic + // search off there's no chunk store, and asking would create the + // embeddings database for a purely lexical search. + if (Zotero.Embeddings.isEnabled()) { + let indexed = await Zotero.Embeddings.getChunks(itemID); + indexed = indexed.filter(chunk => chunk.text); + if (indexed.length) { + return indexed; + } } return this._cutPassages(itemID); } @@ -732,25 +755,19 @@ Zotero.BestMatch = new function () { if (!item) { return []; } - let chunking = Zotero.Utilities.Internal.Chunking; // Only structure already extracted: generating it costs seconds, // which is not a price a preview may charge (see // Zotero.SDT.getPack()). Without it the flat text still divides, // just without knowing where its passages sit. let structure = await Zotero.SDT.getSections(itemID, { cachedOnly: true }); if (structure.ok && structure.sections.length) { - // The metrics only read the text for its script, so a sample - // of the opening sections says as much as all of them - let sample = structure.sections.slice(0, 5) - .map(section => section.text).join('\n\n'); - return chunking.chunkSections( - structure.sections, chunking.getCharacterMetrics(sample)); + return Zotero.Utilities.Internal.Chunking.chunkSections(structure.sections); } let text = await item.attachmentText; if (!text) { return []; } - return chunking.chunkText(text, chunking.getCharacterMetrics(text)); + return Zotero.Utilities.Internal.Chunking.chunkText(text); } /** @@ -767,10 +784,8 @@ Zotero.BestMatch = new function () { * @param {Object[]} entries - Set in place */ async _pickSnippets(entries) { - let chunking = Zotero.Utilities.Internal.Chunking; for (let entry of entries) { - let sentences = chunking.splitSentences( - entry.text, chunking.getCharacterMetrics(entry.text)); + let sentences = Zotero.Utilities.Internal.Chunking.splitSentences(entry.text); // The passage's opening, for a passage with no words to quote // around entry.snippet = sentences.length diff --git a/chrome/content/zotero/xpcom/embeddings.js b/chrome/content/zotero/xpcom/embeddings.js index 551bab23b1..81c0d33f52 100644 --- a/chrome/content/zotero/xpcom/embeddings.js +++ b/chrome/content/zotero/xpcom/embeddings.js @@ -1502,60 +1502,48 @@ Zotero.Embeddings = new function () { * each be embedded on their own, so a long text's later paragraphs are * searchable instead of being averaged into one vector or truncated away by the * pipeline. The indexer applies this to notes (chunkText()) and to attachment - * full text (chunkSections()). Chunk size is bounded by CHUNK_MAX_TOKENS, - * capped by the model's window, and counted in the model's own tokens. + * full text (chunkSections()). Chunk size is bounded by the shared passage + * geometry's BUDGET_TOKENS (see Zotero.Utilities.Internal.Chunking), capped + * by the model's window, and counted in the model's own tokens. */ Zotero.Embeddings.Chunking = new function () { - // Tokens carried over from the end of one chunk into the start of the - // next, so a thought spanning a boundary is searchable in both. Carried - // only between pieces of the same paragraph block. - const CHUNK_OVERLAP_TOKENS = 48; - // Fewest tokens worth embedding on their own. An item scores as its best - // chunk, so every chunk is another draw at that maximum: splitting text - // into fragments inflates the score without adding information, and a - // fragment loses the context that gave it meaning. Paragraphs below this - // (headings, dates, list items) are combined with their neighbors rather - // than becoming chunks of their own. - const CHUNK_MIN_TOKENS = 120; - // Most tokens a chunk may reach. A ceiling rather than a target: chunks come - // out paragraph-sized, so this decides only how long a text has to be before - // it's split at all, and how far a single oversized paragraph is split. - const CHUNK_MAX_TOKENS = 768; - // Most characters to feed the tokenizer in one encode() call. The - // multilingual models' SentencePiece Unigram tokenizer is quadratic in - // input length -- its Metaspace pre-tokenizer doesn't split at whitespace, - // so the whole input goes through the Viterbi lattice as one string, and a - // single long encode can take seconds. Segments this size keep every call - // in the tokenizer's linear regime, and token counts are additive across - // whitespace boundaries, so the segmented measurement is exact. + // The shared passage geometry (see Zotero.Utilities.Internal.Chunking), + // counted here in the model's own tokens: BUDGET_TOKENS caps a chunk, + // within what the model's window allows (see _getMetrics()); MIN_TOKENS + // is the least worth embedding alone -- an item scores as its best chunk, + // so fragments would inflate the score while losing their context; and + // OVERLAP_TOKENS is carried across a split, so a thought spanning the + // boundary is searchable in both halves. + const { BUDGET_TOKENS, MIN_TOKENS, OVERLAP_TOKENS } = Zotero.Utilities.Internal.Chunking; + // Most characters per encode() call. The multilingual models' + // SentencePiece Unigram tokenizer is quadratic in input length (its + // Metaspace pre-tokenizer doesn't split at whitespace), so a single long + // encode can take seconds. Segments this size stay in the linear regime, + // and token counts are additive across whitespace boundaries, so the + // segmented measurement is exact. const TOKENIZER_SEGMENT_CHARS = 1000; // Tokenizer instances by model name. Failures aren't cached, so a later // indexing run retries the load. let _tokenizers = new Map(); - // Metrics per tokenizer (see _getMetrics()), and the token counter they're - // built on (see _getCounter()). Both cost a handful of encode() calls to - // derive, which is nothing alongside a long text but real when many short - // texts are measured one at a time -- and both depend only on the - // tokenizer, so one derivation serves every text measured against it. + // Counters and metrics cached per tokenizer: each costs a few encode() + // calls to derive, which adds up when many short texts are measured one + // at a time. A tokenizer belongs to one model (see getTokenizer()), so + // the model facts baked into its metrics can't go stale. let _metrics = new WeakMap(); let _counters = new WeakMap(); /** - * The active model's tokenizer, constructed from the tokenizer files in - * the runtime's model cache using the transformers.js implementation - * Firefox ships -- the same code, over the same files, that the inference - * process tokenizes with. + * The active model's tokenizer, built from the files in the runtime's + * model cache with the transformers.js implementation Firefox ships -- + * the same code, over the same files, that the inference process + * tokenizes with. * - * The tokenizer files are part of the downloaded model, so with the model - * present this can't fail short of a broken download -- and then inference - * couldn't tokenize either, so nothing would embed anyway. A failure - * therefore throws, aborting the indexing run and surfacing as its error, - * rather than falling back to an approximate count: an approximation would - * only produce chunks that then fail to embed, and chunk boundaries are - * persistent -- an item's source hash covers its text, not the chunker, so - * a run that guessed at them would never be corrected. + * A failure throws rather than falling back to an approximate count: + * approximated chunk boundaries would persist uncorrected (an item's + * source hash covers its text, not the chunker), and with the tokenizer + * files broken nothing would embed anyway. * * @return {Promise} */ @@ -1587,27 +1575,10 @@ Zotero.Embeddings.Chunking = new function () { return _tokenizers.get(name); }; - // How to measure text against the active model, and how much of its window a - // chunk's own text may use. - // - // Counts leave out the special tokens the tokenizer wraps every input in, so - // that the counts of several pieces of text add up to the count of those - // pieces joined. That additivity is what the whole chunking flow leans on: - // a text is tokenized exactly once, at paragraph granularity - // (_measureParagraphs()), and every level above -- blocks, sections, groups - // of sections -- is sized by summing those counts (_sumTokens()) instead of - // re-tokenizing the same text. `joinTokens` is what one '\n\n' join adds - // once paragraphs are put back together, measured rather than assumed - // (a SentencePiece model can spend a token on collapsed whitespace), so - // sums charge it per join. - // - // Those special tokens come off the window instead, together with the - // passage prefix embedPassages() prepends -- neither is part of the text this - // code sees, but both take up room once the chunk is embedded. - // Counts a text's tokens the way the chunking flow counts them, leaving out - // the special tokens the tokenizer wraps every input in so that counts are - // additive. Depends on the tokenizer alone -- not on which model is - // configured -- so text can be measured wherever a tokenizer is available. + // A counter of a text's tokens that leaves out the special tokens the + // tokenizer wraps every input in, so counts of several pieces add up to + // the count of the pieces joined -- the additivity that lets the shared + // chunker measure each paragraph once and size every grouping by summing. function _getCounter(tokenizer) { let counter = _counters.get(tokenizer); if (!counter) { @@ -1624,6 +1595,12 @@ Zotero.Embeddings.Chunking = new function () { return counter; } + // The shared chunker's metrics for the active model. joinSize is what one + // '\n\n' join adds, measured rather than assumed (a SentencePiece model + // can spend a token on collapsed whitespace). The budget is the shared + // ceiling, capped by the model's window, less the special tokens and the + // passage prefix embedPassages() prepends -- neither is part of the text + // the chunker sees, but both take up window once the chunk is embedded. function _getMetrics(tokenizer) { let cached = _metrics.get(tokenizer); if (cached) { @@ -1635,10 +1612,10 @@ Zotero.Embeddings.Chunking = new function () { let metrics = { count, joinSize: Math.max(0, count('a\n\na') - 2 * count('a')), - budget: Math.min(CHUNK_MAX_TOKENS, Zotero.Embeddings.getModelMaxTokens()) + budget: Math.min(BUDGET_TOKENS, Zotero.Embeddings.getModelMaxTokens()) - specialTokens - (prefix ? count(prefix) : 0), - minSize: CHUNK_MIN_TOKENS, - overlap: CHUNK_OVERLAP_TOKENS + minSize: MIN_TOKENS, + overlap: OVERLAP_TOKENS }; _metrics.set(tokenizer, metrics); return metrics; @@ -1686,10 +1663,9 @@ Zotero.Embeddings.Chunking = new function () { }; /** - * Token count of a text against the active model, leaving out the special - * tokens the tokenizer wraps every input in, so that counts of several - * pieces of text add up to the count of those pieces joined -- the same - * measure the chunkers report on the chunks they return. + * Token count of a text against the active model, without the special + * tokens the tokenizer wraps every input in -- the same additive measure + * the chunkers report on their chunks. * * @param {String} text * @return {Promise} diff --git a/chrome/content/zotero/xpcom/utilities_internal.js b/chrome/content/zotero/xpcom/utilities_internal.js index ddf03f12a5..375730a961 100644 --- a/chrome/content/zotero/xpcom/utilities_internal.js +++ b/chrome/content/zotero/xpcom/utilities_internal.js @@ -3354,29 +3354,41 @@ Zotero.Utilities.Internal.onDragItems = function (event, itemIDs, dragImage = ev /** - * Zotero.Utilities.Internal.Chunking -- splitting a text into passages of a - * size the caller sets, measured however the caller measures (see - * Zotero.Embeddings.Chunking, which measures in a model's tokens). + * Zotero.Utilities.Internal.Chunking -- splitting a text into passages. * - * Metrics: { count(text), joinSize (charged per join, so pieces' sizes add up + * Passages are measured in characters unless the caller supplies its own + * metrics (see Zotero.Embeddings.Chunking, which measures in a model's + * tokens): { count(text), joinSize (charged per join, so pieces' sizes add up * to the joined size), budget (most a passage may reach), minSize (least * worth standing alone), overlap (carried into the next passage where one is - * split mid-thought) }. + * split mid-thought) }. The character default (see getCharacterMetrics()) + * approximates the same geometry, so it suits cuts made and discarded in one + * sitting; boundaries that get stored should be measured exactly, with the + * caller's own metrics. * * Every passage is a slice of its source, located by its start/end extent. */ Zotero.Utilities.Internal.Chunking = new function () { - // The token budgets the character measure mirrors, and what a token is - // worth in characters in either kind of script + // The passage geometry, in tokens: the most a passage may hold + // (BUDGET_TOKENS), the least text worth standing alone rather than being + // combined with a neighbor (MIN_TOKENS), and how much of a split + // passage's tail is carried into the next one (OVERLAP_TOKENS). Exported + // as the single definition of passage size, whichever measure a set of + // metrics counts it in. const BUDGET_TOKENS = 768; const MIN_TOKENS = 120; const OVERLAP_TOKENS = 48; + this.BUDGET_TOKENS = BUDGET_TOKENS; + this.MIN_TOKENS = MIN_TOKENS; + this.OVERLAP_TOKENS = OVERLAP_TOKENS; + // What a token is worth in characters in either kind of script const CHARS_PER_TOKEN = 4; const CJK_CHARS_PER_TOKEN = 1; /** - * A measure counting characters, for a consumer with no tokenizer. The - * budgets mirror the token ones, so passages come out about the size a + * A measure counting characters, for a consumer with no tokenizer -- what + * the chunkers here default to when no metrics are given. The budgets + * mirror the token geometry above, so passages come out about the size a * model-driven chunker would make them. * * @param {String} text - Read for the script it's written in @@ -3394,6 +3406,12 @@ Zotero.Utilities.Internal.Chunking = new function () { }; }; + // The character measure over the text a call is about to chunk, for a + // call that gave no metrics of its own + function _defaultMetrics(text) { + return Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text); + } + // CJK is written without spaces, so a character of it carries about what // a word of an alphabetic script does function _isMostlyCJK(text) { @@ -3503,11 +3521,13 @@ Zotero.Utilities.Internal.Chunking = new function () { * truncation rather than as a passage. * * @param {String} text - * @param {Object} metrics - See getCharacterMetrics(); only `count` is read + * @param {Object} [metrics] - Only `count` is read; the character measure + * when omitted * @return {Object[]} - { text, size, start, end } per sentence, trimmed, * with whitespace-only segments dropped */ this.splitSentences = function (text, metrics) { + metrics = metrics || _defaultMetrics(text); return _segmentSentences(text, 0, text.length, metrics.count); }; @@ -3595,10 +3615,11 @@ Zotero.Utilities.Internal.Chunking = new function () { * pieces at sentence boundaries. * * @param {String} text - * @param {Object} metrics + * @param {Object} [metrics] - The character measure when omitted * @return {Object[]} - [{ text, size, start, end }] */ this.chunkText = function (text, metrics) { + metrics = metrics || _defaultMetrics(text); let paragraphs = _measureParagraphs(text, metrics.count); return _chunkParagraphs(text, paragraphs, metrics.budget, metrics); }; @@ -3684,12 +3705,16 @@ Zotero.Utilities.Internal.Chunking = new function () { * @param {Object[]} sections - [{ text, outlinePath, startBlock, * auxiliary, blocks: [{ index, text, pageIndex, pageLabel, * position }] }] + * @param {Object} [metrics] - The character measure when omitted * @return {Object[]} - [{ text, embedText, size, outlinePath, * startBlock, endBlock, startOffset, endOffset, pageIndex, pageLabel, * position, sectionPart, sectionParts, auxiliary }], where size * counts embedText */ this.chunkSections = function (sections, metrics) { + if (!metrics) { + metrics = _defaultMetrics(sections.map(section => section.text).join('\n\n')); + } let { count, joinSize, budget } = metrics; // Group sections into chunk-worthy units, combining any too small to diff --git a/test/tests/bestMatchTest.js b/test/tests/bestMatchTest.js index f350ff29b7..6814789bb3 100644 --- a/test/tests/bestMatchTest.js +++ b/test/tests/bestMatchTest.js @@ -552,6 +552,22 @@ describe("Zotero.BestMatch", function () { assert.lengthOf(await session.getMatchingExcerpts(attachment.id), 1); }); + it("shouldn't touch the chunk store while no model is enabled", async function () { + stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(false)); + let chunksStub = sinon.stub(Zotero.Embeddings, 'getChunks'); + stubs.push(chunksStub); + stubs.push(sinon.stub(Zotero.SDT, 'getSections').resolves({ ok: false, reason: 'none' })); + stubs.push(sinon.stub(Zotero.Items, 'getAsync') + .resolves({ attachmentText: Promise.resolve('A paragraph about the owl.') })); + + let session = await sessionFor(); + let excerpts = await session.getMatchingExcerpts(attachment.id); + // Reading stored chunks would attach the embeddings database, + // which a purely lexical search shouldn't create + assert.isFalse(chunksStub.called); + assert.isAbove(excerpts.length, 0); + }); + it("should rethrow an unexpected semantic failure", async function () { stubs.push(sinon.stub(Zotero.Embeddings, 'isEnabled').returns(true)); stubs.push(sinon.stub(Zotero.Embeddings, 'getMatchingChunks') @@ -833,6 +849,32 @@ describe("Zotero.BestMatch", function () { assert.sameMembers(reported, atts.slice(PRELOADED).map(att => att.id)); }); + it("should preload previews by where rows appear, not by items' own scores", async function () { + let parent = await createDataObject('item'); + let child = await importFileAttachment('test.pdf', { parentItemID: parent.id }); + // The child scores worst of every match, but its preview rows + // render under the top-ranked parent, at the top of the list + let scores = new Map(atts.map((att, i) => [att.id, 0.9 - i / 100])); + scores.set(parent.id, 0.99); + scores.set(child.id, 0.01); + stubs.push(sinon.stub(Zotero.BestMatch, 'scoreItemIDs').resolves({ + scores, + matches: { lexical: new Set(scores.keys()), semantic: new Set() } + })); + let { release, advanceTo } = stubDerive(); + let session = Zotero.BestMatch.createSession('owl'); + let scored = session.score([...scores.keys()]); + await advanceTo(PRELOADED + 1); + await scored; + + assert.equal(session.getPreviews(child.id).state, 'filled'); + // The weakest standalone attachment waited instead + assert.equal(session.getPreviews(atts[atts.length - 1].id).state, 'pending'); + + session.dispose(); + release(); + }); + it("should stop the background pass when the session is disposed", async function () { stubScore(); let { stub, release, advanceTo } = stubDerive(); diff --git a/test/tests/embeddingsTest.js b/test/tests/embeddingsTest.js index d9c505607c..dc3fe49afa 100644 --- a/test/tests/embeddingsTest.js +++ b/test/tests/embeddingsTest.js @@ -478,7 +478,7 @@ describe("Zotero.Embeddings", function () { // that wrap every input is what a chunk's own text gets (see MODELS). // bge's window is under the chunking ceiling, so the window governs here. const BUDGET = 512 - 2; - // CHUNK_MAX_TOKENS less those same special tokens + // The shared BUDGET_TOKENS geometry less those same special tokens const CEILING = 768 - 2; var fakeTokenizer = wordTokenizer(); // A chunk's own tokens, the way chunking counts them @@ -595,7 +595,7 @@ describe("Zotero.Embeddings", function () { // A block only ever closes on a sentence boundary, so each piece // lands a little under its target. Without spreading that slack over // the pieces still to come, it accumulates into an extra runt piece - // -- which is what CHUNK_MIN_TOKENS exists to prevent. + // -- which is what the MIN_TOKENS geometry exists to prevent. for (let count of [45, 64, 83, 97, 140]) { let sentences = Array.from({ length: count }, (x, i) => `Sentence ${i} has a few more words in it about subject ${i}.`); diff --git a/test/tests/utilities_internalTest.js b/test/tests/utilities_internalTest.js index 60d9e9d193..488f10376c 100644 --- a/test/tests/utilities_internalTest.js +++ b/test/tests/utilities_internalTest.js @@ -758,8 +758,27 @@ describe("Zotero.Utilities.Internal", function () { }); }); describe("Chunking", function () { - var chunk = text => Zotero.Utilities.Internal.Chunking.chunkText( - text, Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text)); + var chunk = text => Zotero.Utilities.Internal.Chunking.chunkText(text); + + it("should chunk with the character measure when no metrics are given", function () { + let text = ('word '.repeat(60) + '\n').repeat(40); + let explicit = Zotero.Utilities.Internal.Chunking.chunkText( + text, Zotero.Utilities.Internal.Chunking.getCharacterMetrics(text)); + assert.deepEqual(chunk(text), explicit); + }); + + it("should mirror the exported geometry in its character defaults", function () { + let chunking = Zotero.Utilities.Internal.Chunking; + // One scale for all three numbers, so the character measure can't + // drift from the geometry it stands in for + for (let text of ['climate change', '気候変動'.repeat(10)]) { + let metrics = chunking.getCharacterMetrics(text); + let scale = metrics.budget / chunking.BUDGET_TOKENS; + assert.isAbove(scale, 0); + assert.equal(metrics.minSize, chunking.MIN_TOKENS * scale); + assert.equal(metrics.overlap, chunking.OVERLAP_TOKENS * scale); + } + }); it("should leave a text within the budget whole", function () { let chunks = chunk('A short paragraph.');