diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 203089d93..e4a5041ba 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to GitNexus will be documented in this file. +## [Unreleased] + +### Performance + +- **`analyze` ~33% faster** — moved FTS index creation from the analyze pipeline to first-use lazy initialisation. The 5 `CREATE_FTS_INDEX` calls cost ~440 ms each in LadybugDB regardless of table size (≈2 s fixed overhead) and dominated runtime on small repos and slow CI runners. The cost now amortises across the first `query`/`context` call in a session via a new `ensureFTSIndex` helper. Mini-repo `analyze` measured locally on Windows: 6.4 s → 4.0 s warm; on CI Windows runners (≈3× slower) restores comfortable headroom against the 30 s e2e test budget. + ## [1.6.2] - 2026-04-18 ### Added diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index f019960c5..e4190c323 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -144,6 +144,18 @@ let currentDbPath: string | null = null; let ftsLoaded = false; let vectorExtensionLoaded = false; +/** + * In-process cache of FTS indexes that have been ensured against the current + * connection. Prevents repeated `CALL CREATE_FTS_INDEX` round-trips inside a + * single CLI/MCP session — the first call to `ensureFTSIndex` for a given + * `(tableName, indexName)` pays the LadybugDB cost (~440 ms even when the + * index already exists on disk), subsequent calls are a Set lookup. Cleared + * by `closeLbug` so a re-init starts fresh. + * + * Key format: `${tableName}:${indexName}`. + */ +const ensuredFTSIndexes = new Set(); + /** * Check if an error indicates a missing column or table (schema-level problem) * rather than a transient/connection error. Used for legacy DB fallback logic. @@ -1037,6 +1049,7 @@ export const closeLbug = async (): Promise => { currentDbPath = null; ftsLoaded = false; vectorExtensionLoaded = false; + ensuredFTSIndexes.clear(); }; export const isLbugReady = (): boolean => conn !== null && db !== null; @@ -1219,6 +1232,29 @@ export const createFTSIndex = async ( } }; +/** + * Lazy-create an FTS index, caching the fact in-process. + * + * Used by `queryFTS` so that `analyze` doesn't pay the ~440 ms × 5 fixed + * LadybugDB cost up-front (it dominates analyze on small repos). Instead, + * the cost is moved to the first `query`/`context` call in a session, + * where it's amortised across many lookups. + * + * Safe to call repeatedly — the in-process Set guarantees only the first + * call hits LadybugDB. `closeLbug` clears the cache so re-init starts fresh. + */ +export const ensureFTSIndex = async ( + tableName: string, + indexName: string, + properties: string[], + stemmer: string = 'porter', +): Promise => { + const key = `${tableName}:${indexName}`; + if (ensuredFTSIndexes.has(key)) return; + await createFTSIndex(tableName, indexName, properties, stemmer); + ensuredFTSIndexes.add(key); +}; + /** * Query a full-text search index * @param tableName - The node table name diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 5c2191003..e61c20f21 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -19,7 +19,6 @@ import { executeQuery, executeWithReusedStatement, closeLbug, - createFTSIndex, loadCachedEmbeddings, } from './lbug/lbug-adapter.js'; import { @@ -215,17 +214,12 @@ export async function runFullAnalysis( }); // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── - progress('fts', 85, 'Creating search indexes...'); - - try { - await createFTSIndex('File', 'file_fts', ['name', 'content']); - await createFTSIndex('Function', 'function_fts', ['name', 'content']); - await createFTSIndex('Class', 'class_fts', ['name', 'content']); - await createFTSIndex('Method', 'method_fts', ['name', 'content']); - await createFTSIndex('Interface', 'interface_fts', ['name', 'content']); - } catch { - // Non-fatal — FTS is best-effort - } + // FTS indexes are created lazily on first `query`/`context` call instead + // of eagerly here. On small repos / CI runners the LadybugDB + // CREATE_FTS_INDEX cost is ~440 ms × 5 (≈2 s) regardless of table size, + // which dominated `analyze` runtime and pushed Windows CI past its + // 30 s test budget. Lazy creation is implemented in + // `core/search/bm25-index.ts` via `ensureFTSIndex`. // ── Phase 3.5: Re-insert cached embeddings ──────────────────────── if (cachedEmbeddings.length > 0) { diff --git a/gitnexus/src/core/search/bm25-index.ts b/gitnexus/src/core/search/bm25-index.ts index ae433ad28..040440999 100644 --- a/gitnexus/src/core/search/bm25-index.ts +++ b/gitnexus/src/core/search/bm25-index.ts @@ -3,9 +3,15 @@ * * Uses LadybugDB's built-in full-text search indexes for keyword-based search. * Always reads from the database (no cached state to drift). + * + * FTS indexes are created lazily on first query (via `ensureFTSIndex`) — see + * `lbug-adapter.ts` for the rationale. This keeps `analyze` fast (the + * ~440 ms × 5 LadybugDB CREATE_FTS_INDEX cost dominates pipeline time on + * small repos / CI runners) at the cost of paying that overhead on the + * first `query`/`context` call in a session. */ -import { queryFTS } from '../lbug/lbug-adapter.js'; +import { queryFTS, ensureFTSIndex } from '../lbug/lbug-adapter.js'; export interface BM25SearchResult { filePath: string; @@ -13,6 +19,56 @@ export interface BM25SearchResult { rank: number; } +/** + * FTS schema served by `searchFTSFromLbug`. Centralised so that both the + * CLI/pipeline path and the MCP pool path use identical (table, index, + * properties) tuples and the lazy-create logic stays in one place. + */ +const FTS_INDEXES: ReadonlyArray<{ + table: string; + indexName: string; + properties: readonly string[]; +}> = [ + { table: 'File', indexName: 'file_fts', properties: ['name', 'content'] }, + { table: 'Function', indexName: 'function_fts', properties: ['name', 'content'] }, + { table: 'Class', indexName: 'class_fts', properties: ['name', 'content'] }, + { table: 'Method', indexName: 'method_fts', properties: ['name', 'content'] }, + { table: 'Interface', indexName: 'interface_fts', properties: ['name', 'content'] }, +]; + +/** + * Per-process cache for the MCP pool path: tracks which `(repoId, table)` + * pairs have been ensured. The CLI/pipeline path gets its own cache inside + * `lbug-adapter.ts` keyed by table/index, scoped to the singleton connection. + */ +const ensuredPoolFTS = new Set(); + +async function ensureFTSIndexViaExecutor( + executor: (cypher: string) => Promise, + repoId: string, + table: string, + indexName: string, + properties: readonly string[], +): Promise { + const key = `${repoId}:${table}:${indexName}`; + if (ensuredPoolFTS.has(key)) return; + const propList = properties.map((p) => `'${p}'`).join(', '); + try { + await executor( + `CALL CREATE_FTS_INDEX('${table}', '${indexName}', [${propList}], stemmer := 'porter')`, + ); + } catch (e: any) { + // 'already exists' is the happy path (index persists on disk between + // process invocations) — anything else we swallow because FTS is + // best-effort: queryFTS itself returns [] on missing-index errors. + const msg = String(e?.message ?? ''); + if (!msg.includes('already exists')) { + // Best-effort — continue without index, queryFTS will fall back to []. + } + } + ensuredPoolFTS.add(key); +} + /** * Execute a single FTS query via a custom executor (for MCP connection pool). * Returns the same shape as core queryFTS (from LadybugDB adapter). @@ -75,6 +131,13 @@ export const searchFTSFromLbug = async ( // The MCP pool supports multiple connections, but FTS is best run serially. const { executeQuery } = await import('../lbug/pool-adapter.js'); const executor = (cypher: string) => executeQuery(repoId, cypher); + + // Lazy-create FTS indexes on first query for this repo (analyze no longer + // creates them up-front, so we ensure them here). Cached per-process. + for (const { table, indexName, properties } of FTS_INDEXES) { + await ensureFTSIndexViaExecutor(executor, repoId, table, indexName, properties); + } + fileResults = await queryFTSViaExecutor(executor, 'File', 'file_fts', query, limit); functionResults = await queryFTSViaExecutor(executor, 'Function', 'function_fts', query, limit); classResults = await queryFTSViaExecutor(executor, 'Class', 'class_fts', query, limit); @@ -87,7 +150,12 @@ export const searchFTSFromLbug = async ( limit, ); } else { - // Use core lbug adapter (CLI / pipeline context) — also sequential for safety + // Use core lbug adapter (CLI / pipeline context) — also sequential for safety. + // Lazy-create FTS indexes on first query (analyze no longer does it). + for (const { table, indexName, properties } of FTS_INDEXES) { + await ensureFTSIndex(table, indexName, [...properties]).catch(() => {}); + } + fileResults = await queryFTS('File', 'file_fts', query, limit, false).catch(() => []); functionResults = await queryFTS('Function', 'function_fts', query, limit, false).catch( () => [],