diff --git a/README.md b/README.md index 55e5a03a7..f19ad996c 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | | `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | +| `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | | `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` | `8388608` (8 MB) | Per-job byte budget the pool will send to a worker in one `postMessage`. | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure. | | `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT` | `3` | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot. | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped. | diff --git a/gitnexus/README.md b/gitnexus/README.md index 92cf4593c..62a329b75 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -363,6 +363,7 @@ Configure the behavior with two environment variables: | -------------------------------------------- | ---------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | | `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process `INSTALL` child before it is killed. | +| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | ```bash @@ -371,6 +372,9 @@ GITNEXUS_LBUG_EXTENSION_INSTALL=load-only npx gitnexus analyze # Slow network: give extension downloads more time GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS=30000 npx gitnexus analyze + +# CJK-heavy codebase: rebuild keyword indexes without English stemming +GITNEXUS_FTS_STEMMER=none npx gitnexus analyze --repair-fts ``` ### Analysis runs out of memory diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 698c98fdd..d40943e32 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -2334,6 +2334,13 @@ export const loadVectorExtension = async ( if (loaded && useModuleState) vectorExtensionLoaded = true; return loaded; }; +/** + * Default stemmer for FTS indexes. Single source so the analyze path + * (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex` + * defaults can never silently diverge. + */ +export const DEFAULT_FTS_STEMMER = 'porter'; + /** * Create a full-text search index on a table * @param tableName - The node table name (e.g., 'File', 'CodeSymbol') @@ -2345,7 +2352,7 @@ export const createFTSIndex = async ( tableName: string, indexName: string, properties: string[], - stemmer: string = 'porter', + stemmer: string = DEFAULT_FTS_STEMMER, ): Promise => { if (!conn) { throw new Error('LadybugDB not initialized. Call initLbug first.'); @@ -2441,7 +2448,7 @@ export const ensureFTSIndex = async ( tableName: string, indexName: string, properties: string[], - stemmer: string = 'porter', + stemmer: string = DEFAULT_FTS_STEMMER, ): Promise => { const key = ftsIndexKey(tableName, indexName); if (ensuredFTSIndexes.has(key)) return; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 1c968c523..8ba5673b3 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -30,7 +30,11 @@ import { queryImporters, loadFTSExtension, } from './lbug/lbug-adapter.js'; -import { createSearchFTSIndexes, verifySearchFTSIndexes } from './search/fts-indexes.js'; +import { + createSearchFTSIndexes, + initialiseSearchFTSStemmer, + verifySearchFTSIndexes, +} from './search/fts-indexes.js'; import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js'; import { startWalCheckpointDriver, @@ -547,6 +551,11 @@ export async function runFullAnalysis( const progress = (phase: string, percent: number, message: string) => callbacks.onProgress(phase, percent, message); + // Resolve + validate operator-provided FTS config once, before the expensive + // parse/load phases. A typo fails here in ms; createSearchFTSIndexes reuses + // the cached value via getSearchFTSStemmer. + initialiseSearchFTSStemmer(); + // Scope the degraded-parse log throttle to this run. On a reused process // (e.g. tests, or any host that calls runFullAnalysis more than once) the // module-level counter would otherwise stay saturated and suppress every diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index cc3d7a638..d24c6ac36 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -1,14 +1,85 @@ -import { createFTSIndex, dropFTSIndex } from '../lbug/lbug-adapter.js'; +import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js'; import { FTS_INDEXES } from './fts-schema.js'; +// Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token +// set in the extension bundled with @ladybugdb/core 0.17.x (see package.json). +// Keep in sync on a LadybugDB minor bump — a value here that the installed +// extension rejects would pass validation but fail at CREATE_FTS_INDEX. +const SUPPORTED_FTS_STEMMERS = new Set([ + 'arabic', + 'basque', + 'catalan', + 'danish', + 'dutch', + 'english', + 'finnish', + 'french', + 'german', + 'greek', + 'hindi', + 'hungarian', + 'indonesian', + 'irish', + 'italian', + 'lithuanian', + 'nepali', + 'norwegian', + 'none', + 'porter', + 'portuguese', + 'romanian', + 'russian', + 'serbian', + 'spanish', + 'swedish', + 'tamil', + 'turkish', +]); + export interface CreateSearchFTSIndexesOptions { onIndexStart?: (table: string, indexName: string) => void; onIndexReady?: (table: string, indexName: string) => void; } +let resolvedStemmer: string | undefined; + +/** Read + validate `GITNEXUS_FTS_STEMMER`. Throws on an unsupported value. */ +function resolveFTSStemmer(): string { + const raw = process.env.GITNEXUS_FTS_STEMMER?.trim().toLowerCase(); + if (!raw) return DEFAULT_FTS_STEMMER; + if (SUPPORTED_FTS_STEMMERS.has(raw)) return raw; + + throw new Error( + `Invalid GITNEXUS_FTS_STEMMER "${process.env.GITNEXUS_FTS_STEMMER}". ` + + `Expected one of: ${[...SUPPORTED_FTS_STEMMERS].sort().join(', ')}.`, + ); +} + +/** + * Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup, + * and cache it. An invalid value throws here — in milliseconds — instead of + * ~85% into a run (after the expensive parse/scope-resolution work). The cached + * value is what {@link getSearchFTSStemmer} returns for the rest of the run, so + * config is read and validated in exactly one place. + */ +export function initialiseSearchFTSStemmer(): string { + resolvedStemmer = resolveFTSStemmer(); + return resolvedStemmer; +} + +/** + * Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back + * to resolving on demand when init was never called (read-only hosts, unit + * tests) so validation always applies. + */ +export function getSearchFTSStemmer(): string { + return resolvedStemmer ?? resolveFTSStemmer(); +} + export async function createSearchFTSIndexes( options?: CreateSearchFTSIndexesOptions, ): Promise { + const stemmer = getSearchFTSStemmer(); for (const { table, indexName, properties } of FTS_INDEXES) { options?.onIndexStart?.(table, indexName); // Drop first so the live `properties` always win. `createFTSIndex` is @@ -23,7 +94,7 @@ export async function createSearchFTSIndexes( // runs inside the existing FTS phase. Gate on a stored schema fingerprint if // this rebuild cost ever shows up in analyze profiles. await dropFTSIndex(table, indexName); - await createFTSIndex(table, indexName, [...properties]); + await createFTSIndex(table, indexName, [...properties], stemmer); options?.onIndexReady?.(table, indexName); } } diff --git a/gitnexus/test/unit/bm25-search.test.ts b/gitnexus/test/unit/bm25-search.test.ts index a9a65725c..579870590 100644 --- a/gitnexus/test/unit/bm25-search.test.ts +++ b/gitnexus/test/unit/bm25-search.test.ts @@ -35,7 +35,7 @@ describe('BM25 search', () => { await createSearchFTSIndexes(); expect(vi.mocked(createFTSIndex).mock.calls).toEqual( - FTS_INDEXES.map((i) => [i.table, i.indexName, [...i.properties]]), + FTS_INDEXES.map((i) => [i.table, i.indexName, [...i.properties], 'porter']), ); }); diff --git a/gitnexus/test/unit/fts-indexes.test.ts b/gitnexus/test/unit/fts-indexes.test.ts index f94819f14..c104d742b 100644 --- a/gitnexus/test/unit/fts-indexes.test.ts +++ b/gitnexus/test/unit/fts-indexes.test.ts @@ -4,20 +4,25 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; const { calls } = vi.hoisted(() => ({ calls: [] as string[] })); vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + DEFAULT_FTS_STEMMER: 'porter', dropFTSIndex: vi.fn(async (table: string, indexName: string) => { calls.push(`drop:${table}.${indexName}`); }), - createFTSIndex: vi.fn(async (table: string, indexName: string) => { - calls.push(`create:${table}.${indexName}`); - }), + createFTSIndex: vi.fn( + async (table: string, indexName: string, _props: string[], stemmer: string) => { + calls.push(`create:${table}.${indexName}:${stemmer}`); + }, + ), })); -const { createSearchFTSIndexes } = await import('../../src/core/search/fts-indexes.js'); +const { createSearchFTSIndexes, getSearchFTSStemmer, initialiseSearchFTSStemmer } = + await import('../../src/core/search/fts-indexes.js'); const { FTS_INDEXES } = await import('../../src/core/search/fts-schema.js'); afterEach(() => { calls.length = 0; vi.clearAllMocks(); + vi.unstubAllEnvs(); }); describe('createSearchFTSIndexes', () => { @@ -25,7 +30,7 @@ describe('createSearchFTSIndexes', () => { await createSearchFTSIndexes(); const expected = FTS_INDEXES.flatMap((i) => [ `drop:${i.table}.${i.indexName}`, - `create:${i.table}.${i.indexName}`, + `create:${i.table}.${i.indexName}:porter`, ]); expect(calls).toEqual(expected); }); @@ -41,4 +46,50 @@ describe('createSearchFTSIndexes', () => { expect(started).toEqual(expectedNames); expect(ready).toEqual(expectedNames); }); + + it('passes the configured FTS stemmer to every index', async () => { + vi.stubEnv('GITNEXUS_FTS_STEMMER', ' none '); + + await createSearchFTSIndexes(); + + expect(calls.filter((call) => call.startsWith('create:'))).toEqual( + FTS_INDEXES.map((i) => `create:${i.table}.${i.indexName}:none`), + ); + }); + + it('rejects unsupported stemmer names before creating indexes', async () => { + vi.stubEnv('GITNEXUS_FTS_STEMMER', "none'); DROP TABLE File; --"); + + await expect(createSearchFTSIndexes()).rejects.toThrow('Invalid GITNEXUS_FTS_STEMMER'); + expect(calls).toEqual([]); + }); +}); + +describe('getSearchFTSStemmer', () => { + it('defaults to porter when unset', () => { + expect(getSearchFTSStemmer()).toBe('porter'); + }); + + it('normalizes configured stemmer names', () => { + vi.stubEnv('GITNEXUS_FTS_STEMMER', ' German '); + + expect(getSearchFTSStemmer()).toBe('german'); + }); +}); + +// Caches module state via initialise; keep last so no later test reads it. +describe('initialiseSearchFTSStemmer', () => { + it('throws on an unsupported stemmer', () => { + vi.stubEnv('GITNEXUS_FTS_STEMMER', 'porterr'); + + expect(() => initialiseSearchFTSStemmer()).toThrow('Invalid GITNEXUS_FTS_STEMMER'); + }); + + it('resolves once so later reads ignore a changed env', () => { + vi.stubEnv('GITNEXUS_FTS_STEMMER', 'german'); + expect(initialiseSearchFTSStemmer()).toBe('german'); + + vi.stubEnv('GITNEXUS_FTS_STEMMER', 'french'); + expect(getSearchFTSStemmer()).toBe('german'); + }); }); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 4d8a0c15c..6e151aad2 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -22,6 +22,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { vi.doUnmock('../../src/storage/repo-manager.js'); vi.resetModules(); vi.clearAllMocks(); + vi.unstubAllEnvs(); }); it('fails repair mode when no base meta exists', async () => { @@ -43,6 +44,35 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { } }); + it('validates configured FTS stemmer before full analyze pipeline work', async () => { + const runPipelineFromRepo = vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo, + })); + vi.stubEnv('GITNEXUS_FTS_STEMMER', 'porterr'); + + const tmpRepo = await createTempDir('gitnexus-run-analyze-invalid-fts-stemmer-'); + try { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { force: true }, + { + onProgress: () => {}, + }, + ), + ).rejects.toThrow(/Invalid GITNEXUS_FTS_STEMMER/i); + expect(runPipelineFromRepo).not.toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }); + it('fails repair mode when graph store is missing', async () => { const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-missing-store-'); try { @@ -121,6 +151,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { loadFTSExtension: vi.fn(async () => true), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), createSearchFTSIndexes: vi.fn(async () => undefined), verifySearchFTSIndexes: vi.fn(async () => [SIMULATED_MISSING_FTS_INDEX_NAME]), })); @@ -170,6 +201,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { loadFTSExtension: vi.fn(async () => true), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), createSearchFTSIndexes: vi.fn(async () => { throw new Error('FTS extension unavailable'); }), @@ -225,6 +257,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { loadFTSExtension: vi.fn(async () => false), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), createSearchFTSIndexes, verifySearchFTSIndexes: vi.fn(async () => []), })); @@ -269,6 +302,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { loadFTSExtension: vi.fn(async () => true), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), createSearchFTSIndexes: vi.fn(async () => undefined), verifySearchFTSIndexes: vi.fn(async () => ['Function.function_fts']), })); @@ -318,6 +352,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { loadFTSExtension: vi.fn(async () => false), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), createSearchFTSIndexes, verifySearchFTSIndexes, }));