From 1e9ef98aa453225ea84ca9f1041a6c19fca67f23 Mon Sep 17 00:00:00 2001 From: Parafee41 Date: Wed, 1 Jul 2026 15:27:09 +0800 Subject: [PATCH] fix(indexing): keep full text file content searchable (#2323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(indexing): keep full text file content searchable * fix(indexing): flush CSV chunks by byte size * fix(fts): flatten newlines/tabs in indexed content so multiline files are searchable (#2317) The end-to-end FTS test (review follow-up F2) exposed that removing the 10KB cap alone does NOT fix #2317: Ladybug's FTS tokenizer splits ONLY on the space character — \n, \r, and \t are not delimiters. So multiline file/symbol content indexes as a few giant cross-line tokens that no word query matches; full content is stored but stays unsearchable. (Verified: identical 8KB content is fully searchable when space-separated and entirely unsearchable when newline-separated.) The existing fts-description-search test never caught this because all its seed content is single-line. Collapse \r\n\t -> single space in the FTS-indexed text (extractContent's File and snippet content, plus the description column) via normalizeFtsText. This rewrites the stored column too, so File content returned via the graph API is space-flattened — an accepted trade for making file/symbol text searchable. Add the real end-to-end guard test/integration/fts-fullfile-search.test.ts: write a >16KB file, load it through the real streamAllCSVsToDisk -> COPY -> createSearchFTSIndexes path, and assert searchFTSFromLbug returns a needle past 10KB (plus a short-content no-regression and a stored-cell-not-truncated guard). It drives the COPY path a Cypher-seed test would bypass, reusing withTestLbugDB's FTS-availability gating via a new before-FTS load hook. * docs(lbug): note the deliberate File-unbounded / snippet-capped asymmetry The File branch returns full content (whitespace-normalized for FTS, bounded upstream by the walker cap) while the symbol snippet path 11 lines down stays MAX_SNIPPET-capped. Comment the intent so the uncapped File branch doesn't read as a forgotten guard. No behavior change. * test(lbug): update #2203 overlap round-trip for FTS whitespace normalization The newline/tab→space normalization (a170915a, #2317) flattens stored File content, so the #2203 overlap test's "File content == original multiline source" assertion no longer holds. The test's actual invariant — overlap path == serial path, byte-for-byte — is unchanged and still asserted; BasicBlock text (not FTS-indexed) still round-trips raw. Update only the File-content expectation to the whitespace-flattened form and document why. * fix(lbug): collapse CSV flush to a single byte threshold BufferedCSVWriter flushed on row-count (FLUSH_EVERY=500) OR byte-count (FLUSH_BYTES=8MB) — two independent triggers for one job. Byte count is the only one tied to the actual risk (an unbounded buffer.join('\n') string), so drop FLUSH_EVERY and make shouldFlushCSVBuffer single-arg. Rather than tune FLUSH_BYTES by guesswork or expose it as an env knob, derive its safety margin from constants the codebase already hard-enforces: a single row is capped at TREE_SITTER_MAX_BUFFER (32MB, clamped regardless of GITNEXUS_MAX_FILE_SIZE) and at most doubled by escapeCSVField's quote-escaping, so the worst-case joined chunk (FLUSH_BYTES + 2 * TREE_SITTER_MAX_BUFFER ≈ 72MB) sits >7x under Node's MAX_STRING_LENGTH (~512MB) — the ceiling that throws RangeError: Invalid string length. A new test pins that margin numerically so it can't erode unnoticed, which covers the "configurable" alternative better than a knob would: there's no evidence any deployment needs a different value, and an unbounded env var would let an operator silently walk the margin back into the danger zone. Also updates the two tests tied to the removed row-count path: the FLUSH_EVERY-boundary integration test now crosses FLUSH_BYTES with real oversized File content instead of relying on row count, and the shouldFlushCSVBuffer unit test drops to the new single-arg signature. --------- Co-authored-by: Gergő Magyar --- gitnexus/src/core/lbug/csv-generator.ts | 87 +++++++++++++++---- gitnexus/test/helpers/test-indexed-db.ts | 19 +++- .../test/integration/csv-pipeline.test.ts | 69 ++++++++++++--- .../integration/fts-fullfile-search.test.ts | 86 ++++++++++++++++++ .../integration/lbug-load-overlap.test.ts | 12 ++- gitnexus/vitest.config.ts | 2 + 6 files changed, 242 insertions(+), 33 deletions(-) create mode 100644 gitnexus/test/integration/fts-fullfile-search.test.ts diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 04bf0fd5d1..b1f5f602aa 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -3,7 +3,7 @@ * * Streams CSV rows directly to disk files in a single pass over graph nodes. * File contents are lazy-read from disk per-node to avoid holding the entire - * repo in RAM. Rows are buffered (FLUSH_EVERY) before writing to minimize + * repo in RAM. Rows are buffered (FLUSH_BYTES) before writing to minimize * per-row Promise overhead. * * RFC 4180 Compliant: @@ -44,8 +44,33 @@ const orderedRelationships = ( ): Iterable => sorted ? [...graph.iterRelationships()].sort(byGraphId) : graph.iterRelationships(); -/** Flush buffered rows to disk every N rows */ -const FLUSH_EVERY = 500; +/** + * Flush buffered rows to disk once the buffered chunk reaches this many bytes. + * Byte-bounded rather than row-count-bounded: row size ranges from a few dozen + * bytes (typical symbol/relationship rows) up to a full File's content + * (#2317/#2323), so a row-count-only cap lets a handful of huge rows build an + * unbounded `buffer.join('\n')` string before ever tripping it. + * + * Not an env knob — fixed by a safety margin, not a preference. The one worst + * case that matters: one more oversized row lands right after the buffer was + * just under this threshold, before the flush fires. That row is capped at + * TREE_SITTER_MAX_BUFFER (32MB, hard-clamped — GITNEXUS_MAX_FILE_SIZE cannot + * raise it), and escapeCSVField's quote-doubling can at most double it. So the + * peak joined-string size is bounded by + * FLUSH_BYTES + 2 * TREE_SITTER_MAX_BUFFER ≈ 8MB + 64MB = 72MB, + * versus Node's `buffer.constants.MAX_STRING_LENGTH` (~512MB) that throws + * `RangeError: Invalid string length` past it — a >7x margin (see the + * `shouldFlushCSVBuffer stays within the V8 string-length ceiling` test, + * which fails loudly if either constant ever moves this margin the wrong + * way). Raising FLUSH_BYTES trades fewer/larger flushes for less margin; + * lowering it trades the reverse for lower peak transient memory. Change the + * constant directly if a real workload needs a different point on that + * curve — a per-host env var would let the margin get silently reintroduced + * by an operator with no way to know why 512MB is dangerous. + */ +export const FLUSH_BYTES = 8 * 1024 * 1024; + +export const shouldFlushCSVBuffer = (byteCount: number): boolean => byteCount >= FLUSH_BYTES; /** * Yield the event loop every N relationship rows during the emit pass (#2226 F4) @@ -148,6 +173,24 @@ class FileContentCache { } } +/** + * Flatten newlines and tabs to single spaces for FTS-indexed text columns + * (`content`, `description`) — the real fix for #2317. + * + * Ladybug's full-text-search tokenizer splits ONLY on the space character — + * `\n`, `\r`, and `\t` are NOT token delimiters. So multiline text indexes as + * a handful of giant tokens (each whole line, joined across lines), and a + * word query matches none of them: `searchFTSFromLbug('foo')` misses a file + * whose content is `... \nfoo\n ...`. Removing the 10KB cap (#2333/#2317) + * stores the full body but leaves it unsearchable; collapsing intra-text + * whitespace to spaces is what actually makes every word searchable. + * + * This rewrites the STORED column too (the same value is COPYed in), so File + * content returned via the graph API is space-flattened — an accepted trade + * for making file/symbol text searchable. Leading/trailing/empty are no-ops. + */ +const normalizeFtsText = (text: string): string => text.replace(/[\r\n\t]+/g, ' '); + const extractContent = async (node: GraphNode, contentCache: FileContentCache): Promise => { const filePath = node.properties.filePath; const content = await contentCache.get(filePath); @@ -155,11 +198,13 @@ const extractContent = async (node: GraphNode, contentCache: FileContentCache): if (node.label === 'Folder') return ''; if (isBinaryContent(content)) return '[Binary file - content not stored]'; + // File content is stored in full — intentionally NOT length-capped here, so + // text past the old 10KB cutoff stays FTS-searchable (#2317). It is already + // bounded upstream by the walker's max-file-size cap (512KB default / 32MB), + // and only whitespace-normalized for the tokenizer. The symbol snippet path + // below, by contrast, deliberately stays capped at MAX_SNIPPET. if (node.label === 'File') { - const MAX_FILE_CONTENT = 10000; - return content.length > MAX_FILE_CONTENT - ? content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]' - : content; + return normalizeFtsText(content); } const startLine = node.properties.startLine; @@ -171,9 +216,9 @@ const extractContent = async (node: GraphNode, contentCache: FileContentCache): const end = Math.min(lines.length - 1, endLine + 2); const snippet = lines.slice(start, end + 1).join('\n'); const MAX_SNIPPET = 5000; - return snippet.length > MAX_SNIPPET - ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]' - : snippet; + const capped = + snippet.length > MAX_SNIPPET ? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]' : snippet; + return normalizeFtsText(capped); }; // ============================================================================ @@ -183,6 +228,7 @@ const extractContent = async (node: GraphNode, contentCache: FileContentCache): class BufferedCSVWriter { private ws: WriteStream; private buffer: string[] = []; + private bufferedBytes = 0; rows = 0; constructor(filePath: string, header: string) { @@ -190,10 +236,11 @@ class BufferedCSVWriter { // Large repos flush many times — raise listener cap to avoid MaxListenersExceededWarning this.ws.setMaxListeners(50); this.buffer.push(header); + this.bufferedBytes = Buffer.byteLength(header) + 1; } /** - * Buffer a row. Returns a promise ONLY when the buffer crossed FLUSH_EVERY + * Buffer a row. Returns a promise ONLY when the buffer crossed FLUSH_BYTES * and a disk write was issued; otherwise returns `undefined` so the caller * can skip awaiting (#2203 U3) — avoiding a microtask tick on every buffered * row (millions at scale). The flush promise still resolves on drain, so @@ -201,8 +248,9 @@ class BufferedCSVWriter { */ addRow(row: string): Promise | undefined { this.buffer.push(row); + this.bufferedBytes += Buffer.byteLength(row) + 1; this.rows++; - if (this.buffer.length >= FLUSH_EVERY) { + if (shouldFlushCSVBuffer(this.bufferedBytes)) { return this.flush(); } return undefined; @@ -212,6 +260,7 @@ class BufferedCSVWriter { if (this.buffer.length === 0) return Promise.resolve(); const chunk = this.buffer.join('\n') + '\n'; this.buffer.length = 0; + this.bufferedBytes = 0; return new Promise((resolve, reject) => { this.ws.once('error', reject); const ok = this.ws.write(chunk); @@ -451,7 +500,7 @@ export const streamAllCSVsToDisk = async ( // addRow returns a promise only when it flushes; awaiting it once after the // switch (instead of `await`-ing every addRow) skips a per-row microtask - // tick on the ~FLUSH_EVERY-1 buffered rows between flushes (#2203 U3). + // tick on the rows buffered between byte-bounded flushes (#2203 U3). let pending: Promise | undefined; switch (node.label) { case 'File': { @@ -484,7 +533,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVField(node.properties.name || ''), escapeCSVField(node.properties.heuristicLabel || ''), keywordsStr, - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), escapeCSVField(node.properties.enrichedBy || 'heuristic'), escapeCSVNumber(node.properties.cohesion, 0), escapeCSVNumber(node.properties.symbolCount, 0), @@ -520,7 +569,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.endLine, -1), node.properties.isExported ? 'true' : 'false', escapeCSVField(content), - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), escapeCSVNumber(node.properties.parameterCount, 0), escapeCSVField(node.properties.returnType || ''), ].join(','), @@ -538,7 +587,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.endLine, -1), escapeCSVNumber(node.properties.level, 1), escapeCSVField(content), - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), ].join(','), ); break; @@ -572,7 +621,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVField(node.id), escapeCSVField(node.properties.name || ''), escapeCSVField(node.properties.filePath || ''), - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), ].join(','), ); break; @@ -593,7 +642,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.endLine, -1), node.properties.isExported ? 'true' : 'false', escapeCSVField(content), - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), ].join(','), ); } else { @@ -609,7 +658,7 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.startLine, -1), escapeCSVNumber(node.properties.endLine, -1), escapeCSVField(content), - escapeCSVField(node.properties.description || ''), + escapeCSVField(normalizeFtsText(node.properties.description || '')), ...(node.label === 'Property' ? [escapeCSVField(node.properties.declaredType || '')] : []), diff --git a/gitnexus/test/helpers/test-indexed-db.ts b/gitnexus/test/helpers/test-indexed-db.ts index c6cca74625..1d7db5217c 100644 --- a/gitnexus/test/helpers/test-indexed-db.ts +++ b/gitnexus/test/helpers/test-indexed-db.ts @@ -37,12 +37,21 @@ export interface FTSIndexDef { /** * Options for withTestLbugDB lifecycle. * - * Lifecycle: initLbug → loadFTS → dropFTS → clearData → seed + * Lifecycle: initLbug → loadFTS → dropFTS → clearData → seed → beforeFTS * → createFTS → [closeCoreLbug + poolInitLbug] → afterSetup */ export interface WithTestLbugDBOptions { /** Cypher CREATE queries to insert seed data (runs before core adapter opens). */ seed?: string[]; + /** + * Custom load step run after Cypher `seed` and BEFORE the gated FTS build, so + * `createFTSIndex` indexes whatever this loads. Use it to exercise the real + * CSV→COPY path (`loadGraphToLbug`) instead of Cypher CREATE. Receives the + * core adapter's `dbPath`; colocate scratch files under `path.dirname(dbPath)` + * to inherit the suite's temp-dir cleanup. Runs unconditionally (no FTS + * needed); the FTS build below stays gated on extension availability. + */ + beforeFTS?: (dbPath: string) => Promise; /** FTS indexes to create after seeding. */ ftsIndexes?: FTSIndexDef[]; /** Close core adapter and open pool adapter (read-only) after FTS setup. */ @@ -141,6 +150,14 @@ export function withTestLbugDB( } } + // 4b. Custom load step (e.g. loadGraphToLbug COPY path) before the FTS + // build, so createFTSIndex below indexes the loaded rows. Runs + // unconditionally — no FTS extension needed to COPY — while the FTS + // build stays gated on ftsAvailable. + if (options?.beforeFTS) { + await options.beforeFTS(dbPath); + } + // 5. Create FTS indexes on fresh data (only when the extension loaded; // otherwise the suite is skipped via beforeEach below). if (options?.ftsIndexes?.length && ftsAvailable) { diff --git a/gitnexus/test/integration/csv-pipeline.test.ts b/gitnexus/test/integration/csv-pipeline.test.ts index 692ffe00f4..e0799d88d5 100644 --- a/gitnexus/test/integration/csv-pipeline.test.ts +++ b/gitnexus/test/integration/csv-pipeline.test.ts @@ -9,16 +9,20 @@ import fs from 'fs/promises'; import { readdirSync } from 'node:fs'; import { finished } from 'stream/promises'; import path from 'path'; +import { constants as bufferConstants } from 'node:buffer'; import { createTempDir, type TestDBHandle } from '../helpers/test-db.js'; import { buildTestGraph, type TestNodeInput, type TestRelInput } from '../helpers/test-graph.js'; import { streamAllCSVsToDisk, buildRelRow, REL_CSV_HEADER, + shouldFlushCSVBuffer, + FLUSH_BYTES, } from '../../src/core/lbug/csv-generator.js'; import { splitRelCsvByLabelPair } from '../../src/core/lbug/lbug-adapter.js'; import { getNodeLabel } from '../../src/core/lbug/rel-pair-routing.js'; import { NODE_TABLES } from '../../src/core/lbug/schema.js'; +import { TREE_SITTER_MAX_BUFFER } from '../../src/core/ingestion/constants.js'; let tmpHandle: TestDBHandle; let csvDir: string; @@ -166,6 +170,30 @@ describe('streamAllCSVsToDisk', () => { expect(content).toContain('"index.ts"'); }); + it('keeps full text file content searchable past 10KB', async () => { + const lateNeedle = 'late_text_file_needle_after_10kb'; + await fs.writeFile( + path.join(repoDir, 'src', 'large.txt'), + `${'filler line for large text indexing\n'.repeat(400)}${lateNeedle}\n`, + ); + const graph = buildTestGraph([ + { + id: 'file:src/large.txt', + label: 'File', + name: 'large.txt', + filePath: 'src/large.txt', + }, + ]); + + const result = await streamAllCSVsToDisk(graph, repoDir, csvDir); + const fileCsv = result.nodeFiles.get('File'); + expect(fileCsv).toBeDefined(); + + const content = await fs.readFile(fileCsv!.csvPath, 'utf-8'); + expect(content).toContain(lateNeedle); + expect(content).not.toContain('[truncated]'); + }); + it('handles community nodes with keywords', async () => { const graph = buildTestGraph([ { @@ -255,17 +283,21 @@ describe('streamAllCSVsToDisk', () => { expect(fileCsv!.rows).toBe(1); }); - it('crosses the BufferedCSVWriter FLUSH_EVERY boundary without losing rows', async () => { - // FLUSH_EVERY=500; a >500-node graph forces ≥1 mid-stream flush, exercising - // addRow's flush-promise return + the loop's `if (pending) await pending` - // path that the small fixtures above never reach (only the bench did). - const N = 600; - const nodes = Array.from({ length: N }, (_, i) => ({ - id: `File:src/f${i}.ts`, - label: 'File' as const, - name: `f${i}.ts`, - filePath: `src/f${i}.ts`, - })); + it('crosses the BufferedCSVWriter FLUSH_BYTES boundary without losing rows', async () => { + // FLUSH_BYTES=8MB; real File content totalling >8MB forces ≥1 mid-stream + // flush, exercising addRow's flush-promise return + the loop's + // `if (pending) await pending` path that the small fixtures above never + // reach (only the bench did). + const N = 10; + const CONTENT_SIZE = 1024 * 1024; // 1MB/file, 10MB total > FLUSH_BYTES + const bigContent = 'x'.repeat(CONTENT_SIZE); + await fs.mkdir(path.join(repoDir, 'src', 'big'), { recursive: true }); + const nodes: TestNodeInput[] = []; + for (let i = 0; i < N; i++) { + const filePath = `src/big/f${i}.ts`; + await fs.writeFile(path.join(repoDir, filePath), bigContent); + nodes.push({ id: `File:${filePath}`, label: 'File', name: `f${i}.ts`, filePath }); + } const result = await streamAllCSVsToDisk(buildTestGraph(nodes), repoDir, csvDir); const fileCsv = result.nodeFiles.get('File'); @@ -275,6 +307,21 @@ describe('streamAllCSVsToDisk', () => { expect(dataRows).toHaveLength(N); expect(new Set(dataRows).size).toBe(N); // all distinct — no flush-boundary corruption }); + + it('flushes the buffered CSV chunk once the byte threshold is reached', () => { + expect(shouldFlushCSVBuffer(FLUSH_BYTES - 1)).toBe(false); + expect(shouldFlushCSVBuffer(FLUSH_BYTES)).toBe(true); + }); + + it('shouldFlushCSVBuffer stays within the V8 string-length ceiling', () => { + // One more max-size row (TREE_SITTER_MAX_BUFFER, hard-clamped — see + // max-file-size.ts) can land right after the buffer was just under + // FLUSH_BYTES; escapeCSVField's quote-doubling can at most double it. + // The resulting join() must stay well under Node's MAX_STRING_LENGTH, + // or BufferedCSVWriter.flush() throws `RangeError: Invalid string length`. + const worstCaseJoinSize = FLUSH_BYTES + 2 * TREE_SITTER_MAX_BUFFER; + expect(worstCaseJoinSize).toBeLessThan(bufferConstants.MAX_STRING_LENGTH / 2); + }); }); /** diff --git a/gitnexus/test/integration/fts-fullfile-search.test.ts b/gitnexus/test/integration/fts-fullfile-search.test.ts new file mode 100644 index 0000000000..7bed2fcaea --- /dev/null +++ b/gitnexus/test/integration/fts-fullfile-search.test.ts @@ -0,0 +1,86 @@ +/** + * End-to-end FTS searchability for full File content (#2317 / PR #2323). + * + * PR #2323 removed the 10KB `MAX_FILE_CONTENT` cap so full text-file content + * reaches `file_fts`. The PR's own test proves the late needle lands in the + * generated `file.csv`; it does NOT prove an FTS *search* returns content past + * 10KB. This closes that gap through the REAL pipeline: + * + * write >10KB file on disk → loadGraphToLbug (streamAllCSVsToDisk → COPY of + * the multiline quoted cell) → createFTSIndex(file_fts) → searchFTSFromLbug. + * + * A Cypher CREATE seed would bypass COPY and pass even if COPY truncated the + * cell — the exact thing #2317 must guarantee — so this uses `loadGraphToLbug` + * via the harness's `beforeFTS` hook (which runs before the gated FTS build), + * reusing `withTestLbugDB`'s offline-skip / GITNEXUS_REQUIRE_FTS gating. + */ +import { describe, it, expect } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; +import { buildTestGraph } from '../helpers/test-graph.js'; +import { searchFTSFromLbug } from '../../src/core/search/bm25-index.js'; + +// A token near the top (< 10KB) and a distinctive token past ~20KB. Both are +// unique lowercase-alphabetic non-stopwords so they tokenize cleanly under the +// `porter` stemmer and never collide with the printable-ASCII filler (keeping +// the first 1000 chars text, so isBinaryContent does not swap in its sentinel). +const earlyWord = 'sentinelalpha'; +const lateNeedle = 'zarquonbeacon'; +const FILLER = 'filler line for full file content indexing\n'; // ~43 chars +const FILE_BODY = + `${earlyWord} appears near the very top of the file\n` + + FILLER.repeat(500) + // ~21.5KB of filler → lateNeedle lands well past 10KB + `${lateNeedle} appears far past the old ten kilobyte cutoff\n`; + +withTestLbugDB( + 'fts-fullfile-search', + () => { + describe('full File content past 10KB is FTS-searchable (#2317)', () => { + it('returns the file for a needle located past the old 10KB cutoff', async () => { + const { results } = await searchFTSFromLbug(lateNeedle, 20); + expect(results.map((r) => r.filePath)).toContain('large.txt'); + }); + + it('persists the full multiline cell through COPY — past 10KB, not the binary sentinel', async () => { + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const rows = await adapter.executeQuery( + "MATCH (f:File {filePath: 'large.txt'}) RETURN f.content AS content", + ); + const stored = String(rows[0].content); + expect(stored.length).toBeGreaterThan(10240); + expect(stored).not.toContain('[Binary file'); + expect(stored).toContain(lateNeedle); + }); + + it('still finds a token within the first 10KB (no short-content regression)', async () => { + const { results } = await searchFTSFromLbug(earlyWord, 20); + expect(results.map((r) => r.filePath)).toContain('large.txt'); + }); + }); + }, + { + // Triggers the FTS-availability probe + offline-skip / GITNEXUS_REQUIRE_FTS + // gating, and builds file_fts over the COPY'd File rows (after beforeFTS). + ftsIndexes: [{ table: 'File', indexName: 'file_fts', columns: ['name', 'content'] }], + // No Cypher `seed`; no pool adapter → searchFTSFromLbug routes through the + // core-adapter connection loadGraphToLbug + createFTSIndex wrote to. + beforeFTS: async (dbPath) => { + // Colocate scratch dirs under the suite temp root so they're auto-cleaned. + const root = path.dirname(dbPath); + const repoDir = path.join(root, 'repo'); + const storageDir = path.join(root, 'storage'); + await fs.mkdir(repoDir, { recursive: true }); + await fs.mkdir(storageDir, { recursive: true }); + await fs.writeFile(path.join(repoDir, 'large.txt'), FILE_BODY); + + // extractContent reads File content from disk, so the on-disk file is the + // source of the COPY'd cell. loadGraphToLbug runs the real emit + COPY. + const graph = buildTestGraph([ + { id: 'file:large.txt', label: 'File', name: 'large.txt', filePath: 'large.txt' }, + ]); + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.loadGraphToLbug(graph, repoDir, storageDir); + }, + }, +); diff --git a/gitnexus/test/integration/lbug-load-overlap.test.ts b/gitnexus/test/integration/lbug-load-overlap.test.ts index e44d94745d..27477a7719 100644 --- a/gitnexus/test/integration/lbug-load-overlap.test.ts +++ b/gitnexus/test/integration/lbug-load-overlap.test.ts @@ -191,10 +191,18 @@ describe('node-COPY ‖ rel-emit overlap persists identical content (#2203)', () }); it('multiline content/text fields round-trip identically (byte-for-byte)', () => { + // The #2203 invariant — overlap path == serial path — holds byte-for-byte + // for both fields regardless of any content transform. expect(overlapSnap.bbText).toBe(serialSnap.bbText); - expect(overlapSnap.bbText).toBe(BB_TEXT); expect(overlapSnap.fileContent).toBe(serialSnap.fileContent); - expect(overlapSnap.fileContent).toBe(FILE_SRC); + + // BasicBlock text is NOT FTS-indexed, so it round-trips raw (newlines kept). + expect(overlapSnap.bbText).toBe(BB_TEXT); + // File content IS FTS-indexed and is whitespace-normalized for the + // space-only tokenizer (#2317): newlines/tabs collapse to single spaces. + // So it round-trips as the source with intra-text whitespace flattened, + // not byte-identical to the original multiline source. + expect(overlapSnap.fileContent).toBe(FILE_SRC.replace(/[\r\n\t]+/g, ' ')); }); it('loadGraphToLbug accounting (insertedRels/skippedRels/warnings) is identical', () => { diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 4b7355b27c..4ffdcfe48a 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -58,6 +58,7 @@ export default defineConfig({ 'test/integration/search-core.test.ts', 'test/integration/search-pool.test.ts', 'test/integration/fts-description-search.test.ts', + 'test/integration/fts-fullfile-search.test.ts', 'test/integration/augmentation.test.ts', 'test/integration/staleness-and-stability.test.ts', 'test/integration/lbug-lock-retry.test.ts', @@ -95,6 +96,7 @@ export default defineConfig({ 'test/integration/search-core.test.ts', 'test/integration/search-pool.test.ts', 'test/integration/fts-description-search.test.ts', + 'test/integration/fts-fullfile-search.test.ts', 'test/integration/augmentation.test.ts', 'test/integration/staleness-and-stability.test.ts', 'test/integration/lbug-lock-retry.test.ts',