fix(search): make FTS stemmer configurable (#2307)

README.md

@@ -327,6 +327,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max
 | `PROF_LBUG_LOAD`                       | unset                     | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. |
 | `GITNEXUS_MAX_FILE_SIZE`               | `512` (KB)                | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size <kb>`.                                       | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed.                     |
 | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000`                   | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout <seconds>` × 1000.                                              | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s.                                        |
+| `GITNEXUS_FTS_STEMMER`                 | `porter`                  | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming.                                             |
 | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD`       | `67108864` (64 MiB)       | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold <bytes>`. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload.                                                         |
 | `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES`  | `8388608` (8 MB)          | Per-job byte budget the pool will send to a worker in one `postMessage`.                                                                                   | Very large individual files; mostly diagnostic — bumping past 8 MB risks structured-clone memory pressure.                                  |
 | `GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT`        | `3`                       | Max replacement spawns per worker slot before the slot is dropped from the active rotation. Bounds respawn loops on a chronically-crashing slot.           | Hosts where a flaky worker should retry more (raise) or fail-fast (lower) before the slot is dropped.                                       |

gitnexus/README.md

@@ -363,6 +363,7 @@ Configure the behavior with two environment variables:
 | -------------------------------------------- | ---------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `GITNEXUS_LBUG_EXTENSION_INSTALL`            | `auto`, `load-only`, `never` | `auto`              | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely.                                                                                       |
 | `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer             | `15000`             | Wall-clock budget for the out-of-process `INSTALL` child before it is killed.                                                                                                                                                                                                            |
+| `GITNEXUS_FTS_STEMMER`                       | supported LadybugDB stemmer  | `porter`            | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it.              |
 | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD`          | integer `>= -1`              | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
 
 ```bash
@@ -371,6 +372,9 @@ GITNEXUS_LBUG_EXTENSION_INSTALL=load-only npx gitnexus analyze
 
 # Slow network: give extension downloads more time
 GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS=30000 npx gitnexus analyze
+
+# CJK-heavy codebase: rebuild keyword indexes without English stemming
+GITNEXUS_FTS_STEMMER=none npx gitnexus analyze --repair-fts
 ```
 
 ### Analysis runs out of memory

gitnexus/src/core/lbug/lbug-adapter.ts

@@ -2334,6 +2334,13 @@ export const loadVectorExtension = async (
   if (loaded && useModuleState) vectorExtensionLoaded = true;
   return loaded;
 };
+/**
+ * Default stemmer for FTS indexes. Single source so the analyze path
+ * (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex`
+ * defaults can never silently diverge.
+ */
+export const DEFAULT_FTS_STEMMER = 'porter';
+
 /**
  * Create a full-text search index on a table
  * @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
@@ -2345,7 +2352,7 @@ export const createFTSIndex = async (
   tableName: string,
   indexName: string,
   properties: string[],
-  stemmer: string = 'porter',
+  stemmer: string = DEFAULT_FTS_STEMMER,
 ): Promise<void> => {
   if (!conn) {
     throw new Error('LadybugDB not initialized. Call initLbug first.');
@@ -2441,7 +2448,7 @@ export const ensureFTSIndex = async (
   tableName: string,
   indexName: string,
   properties: string[],
-  stemmer: string = 'porter',
+  stemmer: string = DEFAULT_FTS_STEMMER,
 ): Promise<void> => {
   const key = ftsIndexKey(tableName, indexName);
   if (ensuredFTSIndexes.has(key)) return;

gitnexus/src/core/run-analyze.ts

@@ -30,7 +30,11 @@ import {
   queryImporters,
   loadFTSExtension,
 } from './lbug/lbug-adapter.js';
-import { createSearchFTSIndexes, verifySearchFTSIndexes } from './search/fts-indexes.js';
+import {
+  createSearchFTSIndexes,
+  initialiseSearchFTSStemmer,
+  verifySearchFTSIndexes,
+} from './search/fts-indexes.js';
 import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
 import {
   startWalCheckpointDriver,
@@ -547,6 +551,11 @@ export async function runFullAnalysis(
   const progress = (phase: string, percent: number, message: string) =>
     callbacks.onProgress(phase, percent, message);
 
+  // Resolve + validate operator-provided FTS config once, before the expensive
+  // parse/load phases. A typo fails here in ms; createSearchFTSIndexes reuses
+  // the cached value via getSearchFTSStemmer.
+  initialiseSearchFTSStemmer();
+
   // Scope the degraded-parse log throttle to this run. On a reused process
   // (e.g. tests, or any host that calls runFullAnalysis more than once) the
   // module-level counter would otherwise stay saturated and suppress every

gitnexus/src/core/search/fts-indexes.ts

@@ -1,14 +1,85 @@
-import { createFTSIndex, dropFTSIndex } from '../lbug/lbug-adapter.js';
+import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js';
 import { FTS_INDEXES } from './fts-schema.js';
 
+// Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token
+// set in the extension bundled with @ladybugdb/core 0.17.x (see package.json).
+// Keep in sync on a LadybugDB minor bump — a value here that the installed
+// extension rejects would pass validation but fail at CREATE_FTS_INDEX.
+const SUPPORTED_FTS_STEMMERS = new Set<string>([
+  'arabic',
+  'basque',
+  'catalan',
+  'danish',
+  'dutch',
+  'english',
+  'finnish',
+  'french',
+  'german',
+  'greek',
+  'hindi',
+  'hungarian',
+  'indonesian',
+  'irish',
+  'italian',
+  'lithuanian',
+  'nepali',
+  'norwegian',
+  'none',
+  'porter',
+  'portuguese',
+  'romanian',
+  'russian',
+  'serbian',
+  'spanish',
+  'swedish',
+  'tamil',
+  'turkish',
+]);
+
 export interface CreateSearchFTSIndexesOptions {
   onIndexStart?: (table: string, indexName: string) => void;
   onIndexReady?: (table: string, indexName: string) => void;
 }
 
+let resolvedStemmer: string | undefined;
+
+/** Read + validate `GITNEXUS_FTS_STEMMER`. Throws on an unsupported value. */
+function resolveFTSStemmer(): string {
+  const raw = process.env.GITNEXUS_FTS_STEMMER?.trim().toLowerCase();
+  if (!raw) return DEFAULT_FTS_STEMMER;
+  if (SUPPORTED_FTS_STEMMERS.has(raw)) return raw;
+
+  throw new Error(
+    `Invalid GITNEXUS_FTS_STEMMER "${process.env.GITNEXUS_FTS_STEMMER}". ` +
+      `Expected one of: ${[...SUPPORTED_FTS_STEMMERS].sort().join(', ')}.`,
+  );
+}
+
+/**
+ * Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
+ * and cache it. An invalid value throws here — in milliseconds — instead of
+ * ~85% into a run (after the expensive parse/scope-resolution work). The cached
+ * value is what {@link getSearchFTSStemmer} returns for the rest of the run, so
+ * config is read and validated in exactly one place.
+ */
+export function initialiseSearchFTSStemmer(): string {
+  resolvedStemmer = resolveFTSStemmer();
+  return resolvedStemmer;
+}
+
+/**
+ * Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back
+ * to resolving on demand when init was never called (read-only hosts, unit
+ * tests) so validation always applies.
+ */
+export function getSearchFTSStemmer(): string {
+  return resolvedStemmer ?? resolveFTSStemmer();
+}
+
 export async function createSearchFTSIndexes(
   options?: CreateSearchFTSIndexesOptions,
 ): Promise<void> {
+  const stemmer = getSearchFTSStemmer();
   for (const { table, indexName, properties } of FTS_INDEXES) {
     options?.onIndexStart?.(table, indexName);
     // Drop first so the live `properties` always win. `createFTSIndex` is
@@ -23,7 +94,7 @@ export async function createSearchFTSIndexes(
     // runs inside the existing FTS phase. Gate on a stored schema fingerprint if
     // this rebuild cost ever shows up in analyze profiles.
     await dropFTSIndex(table, indexName);
-    await createFTSIndex(table, indexName, [...properties]);
+    await createFTSIndex(table, indexName, [...properties], stemmer);
     options?.onIndexReady?.(table, indexName);
   }
 }

gitnexus/test/unit/bm25-search.test.ts

@@ -35,7 +35,7 @@ describe('BM25 search', () => {
       await createSearchFTSIndexes();
 
       expect(vi.mocked(createFTSIndex).mock.calls).toEqual(
-        FTS_INDEXES.map((i) => [i.table, i.indexName, [...i.properties]]),
+        FTS_INDEXES.map((i) => [i.table, i.indexName, [...i.properties], 'porter']),
       );
     });
 

gitnexus/test/unit/fts-indexes.test.ts

@@ -4,20 +4,25 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
 const { calls } = vi.hoisted(() => ({ calls: [] as string[] }));
 
 vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({
+  DEFAULT_FTS_STEMMER: 'porter',
   dropFTSIndex: vi.fn(async (table: string, indexName: string) => {
     calls.push(`drop:${table}.${indexName}`);
   }),
-  createFTSIndex: vi.fn(async (table: string, indexName: string) => {
-    calls.push(`create:${table}.${indexName}`);
-  }),
+  createFTSIndex: vi.fn(
+    async (table: string, indexName: string, _props: string[], stemmer: string) => {
+      calls.push(`create:${table}.${indexName}:${stemmer}`);
+    },
+  ),
 }));
 
-const { createSearchFTSIndexes } = await import('../../src/core/search/fts-indexes.js');
+const { createSearchFTSIndexes, getSearchFTSStemmer, initialiseSearchFTSStemmer } =
+  await import('../../src/core/search/fts-indexes.js');
 const { FTS_INDEXES } = await import('../../src/core/search/fts-schema.js');
 
 afterEach(() => {
   calls.length = 0;
   vi.clearAllMocks();
+  vi.unstubAllEnvs();
 });
 
 describe('createSearchFTSIndexes', () => {
@@ -25,7 +30,7 @@ describe('createSearchFTSIndexes', () => {
     await createSearchFTSIndexes();
     const expected = FTS_INDEXES.flatMap((i) => [
       `drop:${i.table}.${i.indexName}`,
-      `create:${i.table}.${i.indexName}`,
+      `create:${i.table}.${i.indexName}:porter`,
     ]);
     expect(calls).toEqual(expected);
   });
@@ -41,4 +46,50 @@ describe('createSearchFTSIndexes', () => {
     expect(started).toEqual(expectedNames);
     expect(ready).toEqual(expectedNames);
   });
+
+  it('passes the configured FTS stemmer to every index', async () => {
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', ' none ');
+
+    await createSearchFTSIndexes();
+
+    expect(calls.filter((call) => call.startsWith('create:'))).toEqual(
+      FTS_INDEXES.map((i) => `create:${i.table}.${i.indexName}:none`),
+    );
+  });
+
+  it('rejects unsupported stemmer names before creating indexes', async () => {
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', "none'); DROP TABLE File; --");
+
+    await expect(createSearchFTSIndexes()).rejects.toThrow('Invalid GITNEXUS_FTS_STEMMER');
+    expect(calls).toEqual([]);
+  });
+});
+
+describe('getSearchFTSStemmer', () => {
+  it('defaults to porter when unset', () => {
+    expect(getSearchFTSStemmer()).toBe('porter');
+  });
+
+  it('normalizes configured stemmer names', () => {
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', ' German ');
+
+    expect(getSearchFTSStemmer()).toBe('german');
+  });
+});
+
+// Caches module state via initialise; keep last so no later test reads it.
+describe('initialiseSearchFTSStemmer', () => {
+  it('throws on an unsupported stemmer', () => {
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', 'porterr');
+
+    expect(() => initialiseSearchFTSStemmer()).toThrow('Invalid GITNEXUS_FTS_STEMMER');
+  });
+
+  it('resolves once so later reads ignore a changed env', () => {
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', 'german');
+    expect(initialiseSearchFTSStemmer()).toBe('german');
+
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', 'french');
+    expect(getSearchFTSStemmer()).toBe('german');
+  });
 });

gitnexus/test/unit/run-analyze-fts-repair.test.ts

@@ -22,6 +22,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
     vi.doUnmock('../../src/storage/repo-manager.js');
     vi.resetModules();
     vi.clearAllMocks();
+    vi.unstubAllEnvs();
   });
 
   it('fails repair mode when no base meta exists', async () => {
@@ -43,6 +44,35 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
     }
   });
 
+  it('validates configured FTS stemmer before full analyze pipeline work', async () => {
+    const runPipelineFromRepo = vi.fn(async (repoPath: string) => ({
+      repoPath,
+      graph: { forEachNode: () => undefined },
+    }));
+    vi.doMock('../../src/core/ingestion/pipeline.js', () => ({
+      runPipelineFromRepo,
+    }));
+    vi.stubEnv('GITNEXUS_FTS_STEMMER', 'porterr');
+
+    const tmpRepo = await createTempDir('gitnexus-run-analyze-invalid-fts-stemmer-');
+    try {
+      const { runFullAnalysis } = await import('../../src/core/run-analyze.js');
+
+      await expect(
+        runFullAnalysis(
+          tmpRepo.dbPath,
+          { force: true },
+          {
+            onProgress: () => {},
+          },
+        ),
+      ).rejects.toThrow(/Invalid GITNEXUS_FTS_STEMMER/i);
+      expect(runPipelineFromRepo).not.toHaveBeenCalled();
+    } finally {
+      await tmpRepo.cleanup();
+    }
+  });
+
   it('fails repair mode when graph store is missing', async () => {
     const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-missing-store-');
     try {
@@ -121,6 +151,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
       loadFTSExtension: vi.fn(async () => true),
     }));
     vi.doMock('../../src/core/search/fts-indexes.js', () => ({
+      initialiseSearchFTSStemmer: vi.fn(() => 'porter'),
       createSearchFTSIndexes: vi.fn(async () => undefined),
       verifySearchFTSIndexes: vi.fn(async () => [SIMULATED_MISSING_FTS_INDEX_NAME]),
     }));
@@ -170,6 +201,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
       loadFTSExtension: vi.fn(async () => true),
     }));
     vi.doMock('../../src/core/search/fts-indexes.js', () => ({
+      initialiseSearchFTSStemmer: vi.fn(() => 'porter'),
       createSearchFTSIndexes: vi.fn(async () => {
         throw new Error('FTS extension unavailable');
       }),
@@ -225,6 +257,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
       loadFTSExtension: vi.fn(async () => false),
     }));
     vi.doMock('../../src/core/search/fts-indexes.js', () => ({
+      initialiseSearchFTSStemmer: vi.fn(() => 'porter'),
       createSearchFTSIndexes,
       verifySearchFTSIndexes: vi.fn(async () => []),
     }));
@@ -269,6 +302,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
       loadFTSExtension: vi.fn(async () => true),
     }));
     vi.doMock('../../src/core/search/fts-indexes.js', () => ({
+      initialiseSearchFTSStemmer: vi.fn(() => 'porter'),
       createSearchFTSIndexes: vi.fn(async () => undefined),
       verifySearchFTSIndexes: vi.fn(async () => ['Function.function_fts']),
     }));
@@ -318,6 +352,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => {
       loadFTSExtension: vi.fn(async () => false),
     }));
     vi.doMock('../../src/core/search/fts-indexes.js', () => ({
+      initialiseSearchFTSStemmer: vi.fn(() => 'porter'),
       createSearchFTSIndexes,
       verifySearchFTSIndexes,
     }));