From 98bb893d00d77a32db64ba7fe969d265585695db Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Sun, 10 May 2026 04:23:08 +0530 Subject: [PATCH] feat(lbug): loadGraphFromLbug, queryImporters, deleteAllCommunitiesAndProcesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new primitives in lbug-adapter.ts to support incremental indexing: * loadGraphFromLbug(graph, unchangedFilePaths) — streams all nodes for files in the set across every hydratable node table (excludes Community/Process — graph-wide, regenerated downstream). Then loads edges where both endpoints belong to loaded nodes, excluding MEMBER_OF / STEP_IN_PROCESS edges (also graph-wide). FilePaths chunked at 200 per query to keep statement size bounded on huge repos. Endpoint-level join filters by source-side filePath in the query, target-side checked JS-side via the loadedNodeIds set. * queryImporters(targetFilePath) — returns DISTINCT a.filePath where a -[IMPORTS]-> b and b.filePath = target. Powers closure expansion: when a changed file's surface signature changes, all its importers must be re-parsed. * deleteAllCommunitiesAndProcesses() — drops Community/Process nodes (and their edges via DETACH DELETE) at the start of each incremental run so the communities/processes phases regenerate them from the fully-merged graph. Required for the 'Leiden runs on full graph' correctness invariant. Co-Authored-By: Claude Opus 4.7 (1M context) --- gitnexus/src/core/lbug/lbug-adapter.ts | 239 +++++++++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index cb88986ca..f47d05e1b 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -1204,6 +1204,245 @@ export const deleteNodesForFile = async ( export const getEmbeddingTableName = (): string => EMBEDDING_TABLE_NAME; +// ============================================================================ +// Incremental indexing: DB → KnowledgeGraph hydration +// ============================================================================ + +/** + * Node tables that have a `filePath` column and are eligible for hydration. + * `Community` and `Process` are graph-wide (no filePath) and are ALWAYS + * regenerated by the communities/processes phases — we never load them back. + */ +const HYDRATABLE_NODE_TABLES: readonly NodeTableName[] = NODE_TABLES.filter( + (t) => t !== 'Community' && t !== 'Process', +); + +/** Per-table extra columns to project beyond the base (id, name, filePath). */ +const TABLE_EXTRA_COLUMNS: Record = { + File: ['content'], + Folder: [], + Function: ['startLine', 'endLine', 'isExported', 'content', 'description'], + Class: ['startLine', 'endLine', 'isExported', 'content', 'description'], + Interface: ['startLine', 'endLine', 'isExported', 'content', 'description'], + Method: [ + 'startLine', + 'endLine', + 'isExported', + 'content', + 'description', + 'parameterCount', + 'returnType', + ], + CodeElement: ['startLine', 'endLine', 'isExported', 'content', 'description'], + Section: ['startLine', 'endLine', 'level', 'content', 'description'], + Property: ['startLine', 'endLine', 'content', 'description', 'declaredType'], + Route: ['responseKeys', 'errorKeys', 'middleware'], + Tool: ['description'], +}; + +/** All other tables share the CODE_ELEMENT_BASE schema (startLine/endLine/content/description). */ +const DEFAULT_EXTRA_COLUMNS = ['startLine', 'endLine', 'content', 'description']; + +const escapeFilePath = (s: string): string => + s.replace(/\\/g, '\\\\').replace(/'/g, "''"); + +/** + * Hydrate `graph` with all nodes (and their relationships) belonging to files + * in `unchangedFilePaths`. Used by the incremental-indexing pipeline to + * pre-populate the in-memory graph with everything that didn't change, so + * downstream phases (mro, communities, processes) see a complete picture. + * + * Skips Community/Process labels and their MEMBER_OF / STEP_IN_PROCESS edges + * — those are graph-wide and will be regenerated from scratch by the + * pipeline's downstream phases. + * + * Returns counts for logging/diagnostics. + */ +export const loadGraphFromLbug = async ( + graph: KnowledgeGraph, + unchangedFilePaths: ReadonlySet, +): Promise<{ nodesLoaded: number; edgesLoaded: number }> => { + if (!conn) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + if (unchangedFilePaths.size === 0) { + return { nodesLoaded: 0, edgesLoaded: 0 }; + } + + let nodesLoaded = 0; + let edgesLoaded = 0; + + // Track which node IDs we successfully loaded so the relationship pass + // can verify both endpoints are present (cheaper than a DB-side join). + const loadedNodeIds = new Set(); + + // ── 1. Hydrate nodes per table ───────────────────────────────────────── + // Chunk filePaths to keep query size manageable on huge repos. + const CHUNK = 200; + const filePathArr = [...unchangedFilePaths]; + + for (const tableName of HYDRATABLE_NODE_TABLES) { + const t = escapeTableName(tableName); + const extras = TABLE_EXTRA_COLUMNS[tableName] ?? DEFAULT_EXTRA_COLUMNS; + // Always include base columns: id, name, filePath + const cols = ['id', 'name', 'filePath', ...extras]; + const projection = cols.map((c) => `n.${c} AS ${c}`).join(', '); + + for (let i = 0; i < filePathArr.length; i += CHUNK) { + const batch = filePathArr.slice(i, i + CHUNK); + const inList = batch.map((p) => `'${escapeFilePath(p)}'`).join(','); + const cypher = `MATCH (n:${t}) WHERE n.filePath IN [${inList}] RETURN ${projection}`; + + try { + const queryResult = await conn.query(cypher); + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + const rows = await result.getAll(); + for (const row of rows) { + const id = typeof row.id === 'string' ? row.id : String(row.id ?? ''); + if (!id) continue; + const properties: Record = {}; + for (const c of cols) { + const v = row[c]; + if (v !== undefined && v !== null) properties[c] = v; + } + // Required for downstream phases: ensure name/filePath always set. + if (properties.name === undefined) properties.name = ''; + graph.addNode({ + id, + label: tableName as unknown as import('gitnexus-shared').NodeLabel, + properties: properties as import('gitnexus-shared').NodeProperties, + }); + loadedNodeIds.add(id); + nodesLoaded++; + } + } catch { + // Some tables may not exist in the schema or may be empty — that's fine. + } + } + } + + // ── 2. Hydrate relationships ─────────────────────────────────────────── + // We pull all CodeRelation rows whose endpoints are nodes we just loaded. + // Using SRC.filePath IN [...] AND TGT.filePath IN [...] is precise but + // requires LadybugDB to traverse with both endpoint constraints. We + // exclude graph-wide edge types (MEMBER_OF, STEP_IN_PROCESS) entirely — + // they'll be regenerated. + // + // Strategy: for each (src filePath chunk × tgt filePath chunk) we'd risk + // O(n²) chunking. Instead, we fetch by source-chunk, then verify the + // target endpoint is in `loadedNodeIds` JS-side. This keeps the DB query + // bounded by source chunks while preserving correctness. + for (let i = 0; i < filePathArr.length; i += CHUNK) { + const batch = filePathArr.slice(i, i + CHUNK); + const inList = batch.map((p) => `'${escapeFilePath(p)}'`).join(','); + const cypher = ` + MATCH (a)-[r:${REL_TABLE_NAME}]->(b) + WHERE a.filePath IN [${inList}] + AND r.type <> 'MEMBER_OF' + AND r.type <> 'STEP_IN_PROCESS' + RETURN a.id AS src, b.id AS dst, r.type AS type, + r.confidence AS confidence, r.reason AS reason, r.step AS step + `; + + try { + const queryResult = await conn.query(cypher); + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + const rows = await result.getAll(); + for (const row of rows) { + const src = typeof row.src === 'string' ? row.src : ''; + const dst = typeof row.dst === 'string' ? row.dst : ''; + if (!src || !dst) continue; + // The other endpoint must be a node we loaded — otherwise the edge + // crosses into a closure file (will be re-emitted) or a dropped + // graph-wide node (Community/Process), and we skip it. + if (!loadedNodeIds.has(dst)) continue; + const relType = typeof row.type === 'string' ? row.type : ''; + if (!relType) continue; + const relId = `${src}_${relType}_${dst}`; + graph.addRelationship({ + id: relId, + sourceId: src, + targetId: dst, + type: relType as import('gitnexus-shared').RelationshipType, + confidence: typeof row.confidence === 'number' ? row.confidence : 1.0, + reason: typeof row.reason === 'string' ? row.reason : '', + step: + typeof row.step === 'number' && row.step !== 0 ? row.step : undefined, + }); + edgesLoaded++; + } + } catch { + // Continue on chunk failure — best-effort hydration. + } + } + + return { nodesLoaded, edgesLoaded }; +}; + +/** + * Query the IMPORTS edge table for all files that import `targetFilePath`. + * Used by the incremental-indexing closure-expansion logic to find files + * whose resolution may be stale when `targetFilePath`'s public surface + * changes. + * + * Returns repo-relative paths (i.e. the source file's `filePath`), de-duplicated. + */ +export const queryImporters = async (targetFilePath: string): Promise => { + if (!conn) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + const escaped = escapeFilePath(targetFilePath); + const cypher = ` + MATCH (a)-[r:${REL_TABLE_NAME}]->(b) + WHERE r.type = 'IMPORTS' AND b.filePath = '${escaped}' + RETURN DISTINCT a.filePath AS importer + `; + try { + const queryResult = await conn.query(cypher); + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + const rows = await result.getAll(); + const out: string[] = []; + for (const row of rows) { + const p = row.importer; + if (typeof p === 'string' && p.length > 0) out.push(p); + } + return out; + } catch { + return []; + } +}; + +/** + * Delete all Community and Process nodes and their MEMBER_OF / + * STEP_IN_PROCESS edges. Used at the start of an incremental run so the + * communities/processes phases can regenerate them on the merged graph. + */ +export const deleteAllCommunitiesAndProcesses = async (): Promise<{ + nodesDeleted: number; +}> => { + if (!conn) { + throw new Error('LadybugDB not initialized. Call initLbug first.'); + } + let nodesDeleted = 0; + for (const label of ['Community', 'Process']) { + try { + const countResult = await conn.query( + `MATCH (n:${label}) RETURN count(n) AS cnt`, + ); + const result = Array.isArray(countResult) ? countResult[0] : countResult; + const rows = await result.getAll(); + const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0); + if (count > 0) { + await conn.query(`MATCH (n:${label}) DETACH DELETE n`); + nodesDeleted += count; + } + } catch { + // table may not exist yet + } + } + return { nodesDeleted }; +}; + // ============================================================================ // Full-Text Search (FTS) Functions // ============================================================================