diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 26382632c..b3e913868 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -1,17 +1,17 @@ /** * Analyze Command - * + * * Indexes a repository and stores the knowledge graph in .gitnexus/ */ import path from 'path'; import cliProgress from 'cli-progress'; -import { runPipelineFromRepo } from '../core/ingestion/pipeline.js'; -import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex } from '../core/kuzu/kuzu-adapter.js'; +import { runPipelineFromRepo, runIncrementalPipeline } from '../core/ingestion/pipeline.js'; +import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex, loadExistingGraph } from '../core/kuzu/kuzu-adapter.js'; import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; import { disposeEmbedder } from '../core/embeddings/embedder.js'; import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, getGlobalRegistryPath, getGlobalDir } from '../storage/repo-manager.js'; -import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js'; +import { getCurrentCommit, isGitRepo, getGitRoot, getChangedFiles, getDeletedFiles, getUncommittedChanges } from '../storage/git.js'; import { generateAIContextFiles } from './ai-context.js'; import fs from 'fs/promises'; import { registerClaudeHook } from './claude-hooks.js'; @@ -21,6 +21,9 @@ export interface AnalyzeOptions { skipEmbeddings?: boolean; } +/** Threshold: auto-skip embeddings for repos with more nodes than this */ +const EMBEDDING_NODE_LIMIT = 50_000; + const PHASE_LABELS: Record = { extracting: 'Scanning files', structure: 'Building structure', @@ -30,7 +33,11 @@ const PHASE_LABELS: Record = { heritage: 'Extracting inheritance', communities: 'Detecting communities', processes: 'Detecting processes', - complete: 'Complete', + complete: 'Pipeline complete', + kuzu: 'Loading into KuzuDB', + fts: 'Creating search indexes', + embeddings: 'Generating embeddings', + done: 'Done', }; export const analyzeCommand = async ( @@ -45,7 +52,7 @@ export const analyzeCommand = async ( } else { const gitRoot = getGitRoot(process.cwd()); if (!gitRoot) { - console.log(' ✗ Not inside a git repository\n'); + console.log(' Not inside a git repository\n'); process.exitCode = 1; return; } @@ -53,7 +60,7 @@ export const analyzeCommand = async ( } if (!isGitRepo(repoPath)) { - console.log(' ✗ Not a git repository\n'); + console.log(' Not a git repository\n'); process.exitCode = 1; return; } @@ -63,41 +70,139 @@ export const analyzeCommand = async ( const existingMeta = await loadMeta(storagePath); if (existingMeta && !options?.force && existingMeta.lastCommit === currentCommit) { - console.log(' ✓ Repository already up to date\n'); + console.log(' Already up to date\n'); return; } - const multibar = new cliProgress.MultiBar({ + // ── Determine incremental vs full rebuild ────────────────────────── + const INCREMENTAL_THRESHOLD = 0.6; // Fall back to full if >60% files changed + let useIncremental = false; + let changedFileList: string[] = []; + let deletedFileList: string[] = []; + let cachedEmbeddingNodeIds = new Set(); + let cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }> = []; + + if (existingMeta?.lastCommit && !options?.force) { + changedFileList = getChangedFiles(existingMeta.lastCommit, currentCommit, repoPath); + deletedFileList = getDeletedFiles(existingMeta.lastCommit, currentCommit, repoPath); + + // Also include uncommitted working tree changes — the pipeline reads from disk, + // so any file modified in the working tree needs re-parsing even if not committed. + const uncommitted = getUncommittedChanges(repoPath); + if (uncommitted.length > 0) { + const changedSet = new Set(changedFileList); + for (const f of uncommitted) changedSet.add(f); + changedFileList = Array.from(changedSet); + } + + // We'll check the ratio after scanning total file count below + if (changedFileList.length > 0 || deletedFileList.length > 0) { + useIncremental = true; + } + } + + // Single progress bar for entire pipeline + const bar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {phase}', - barCompleteChar: '█', - barIncompleteChar: '░', + barCompleteChar: '\u2588', + barIncompleteChar: '\u2591', hideCursor: true, barGlue: '', autopadding: true, + clearOnComplete: false, + stopOnComplete: false, }, cliProgress.Presets.shades_grey); - const progressBar = multibar.create(100, 0, { phase: 'Initializing...' }); + bar.start(100, 0, { phase: 'Initializing...' }); - const pipelineResult = await runPipelineFromRepo(repoPath, (progress) => { - const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase; - progressBar.update(progress.percent, { phase: phaseLabel }); - }); + const t0Global = Date.now(); + let pipelineMode: 'full' | 'incremental' = 'full'; + + // ── Load existing graph for incremental mode ─────────────────────── + let preloadedData: Awaited> | null = null; + + if (useIncremental) { + try { + bar.update(0, { phase: 'Loading cached graph...' }); + await initKuzu(kuzuPath); + preloadedData = await loadExistingGraph( + new Set(changedFileList), + new Set(deletedFileList), + ); + await closeKuzu(); + + // Check threshold: if too many files changed, fall back to full rebuild + const totalCachedFiles = preloadedData.nodes.filter(n => n.label === 'File').length; + const totalFiles = totalCachedFiles + changedFileList.length; + const changeRatio = totalFiles > 0 ? changedFileList.length / totalFiles : 1; + + if (changeRatio > INCREMENTAL_THRESHOLD) { + preloadedData = null; + useIncremental = false; + } else { + pipelineMode = 'incremental'; + cachedEmbeddingNodeIds = preloadedData.embeddingNodeIds; + cachedEmbeddings = preloadedData.cachedEmbeddings; + } + } catch { + // KuzuDB load failed — fall back to full rebuild + preloadedData = null; + useIncremental = false; + try { await closeKuzu(); } catch {} + } + } + + // ── Phase 1: Pipeline (0–60%) ────────────────────────────────────── + let pipelineResult; + + if (pipelineMode === 'incremental' && preloadedData) { + bar.update(0, { phase: `Incremental: ${changedFileList.length} files changed` }); + pipelineResult = await runIncrementalPipeline( + repoPath, + new Set(changedFileList), + { + nodes: preloadedData.nodes, + relationships: preloadedData.relationships, + symbolEntries: preloadedData.symbolEntries, + }, + (progress) => { + const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase; + const scaled = Math.round(progress.percent * 0.6); + bar.update(scaled, { phase: phaseLabel }); + } + ); + } else { + pipelineResult = await runPipelineFromRepo(repoPath, (progress) => { + const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase; + const scaled = Math.round(progress.percent * 0.6); + bar.update(scaled, { phase: phaseLabel }); + }); + } + + // ── Phase 2: KuzuDB (60–85%) ────────────────────────────────────── + bar.update(60, { phase: 'Loading into KuzuDB...' }); - progressBar.update(100, { phase: 'Loading graph into KuzuDB...' }); - await closeKuzu(); - - const fsClean = await import('fs/promises'); const kuzuFiles = [kuzuPath, `${kuzuPath}.wal`, `${kuzuPath}.lock`]; for (const f of kuzuFiles) { - try { await fsClean.rm(f, { recursive: true, force: true }); } catch { /* may not exist */ } + try { await fs.rm(f, { recursive: true, force: true }); } catch {} } - + + const t0Kuzu = Date.now(); await initKuzu(kuzuPath); - await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath); + let kuzuMsgCount = 0; + const kuzuResult = await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath, (msg) => { + kuzuMsgCount++; + const progress = Math.min(84, 60 + Math.round((kuzuMsgCount / (kuzuMsgCount + 10)) * 24)); + bar.update(progress, { phase: msg }); + }); + const kuzuTime = ((Date.now() - t0Kuzu) / 1000).toFixed(1); + const kuzuWarnings = kuzuResult.warnings; - progressBar.update(100, { phase: 'Creating search indexes...' }); + // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── + bar.update(85, { phase: 'Creating search indexes...' }); + const t0Fts = Date.now(); try { await createFTSIndex('File', 'file_fts', ['name', 'content']); await createFTSIndex('Function', 'function_fts', ['name', 'content']); @@ -105,21 +210,60 @@ export const analyzeCommand = async ( await createFTSIndex('Method', 'method_fts', ['name', 'content']); await createFTSIndex('Interface', 'interface_fts', ['name', 'content']); } catch (e: any) { - console.error(' Note: Some FTS indexes may not have been created:', e.message); + // Non-fatal — FTS is best-effort + } + const ftsTime = ((Date.now() - t0Fts) / 1000).toFixed(1); + + // ── Phase 3.5: Re-insert cached embeddings (incremental) ─────────── + if (cachedEmbeddings.length > 0) { + bar.update(88, { phase: `Restoring ${cachedEmbeddings.length} cached embeddings...` }); + const EMBED_BATCH = 200; + for (let i = 0; i < cachedEmbeddings.length; i += EMBED_BATCH) { + const batch = cachedEmbeddings.slice(i, i + EMBED_BATCH); + const paramsList = batch.map(e => ({ nodeId: e.nodeId, embedding: e.embedding })); + try { + await executeWithReusedStatement( + `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`, + paramsList, + ); + } catch { /* some may fail if node was removed, that's fine */ } + } } - if (!options?.skipEmbeddings) { - progressBar.update(100, { phase: 'Generating embeddings...' }); + // ── Phase 4: Embeddings (90–98%) ────────────────────────────────── + const stats = await getKuzuStats(); + let embeddingTime = '0.0'; + let embeddingSkipped = false; + let embeddingSkipReason = ''; + + if (options?.skipEmbeddings) { + embeddingSkipped = true; + embeddingSkipReason = 'skipped (--skip-embeddings)'; + } else if (stats.nodes > EMBEDDING_NODE_LIMIT) { + embeddingSkipped = true; + embeddingSkipReason = `skipped (${stats.nodes.toLocaleString()} nodes > ${EMBEDDING_NODE_LIMIT.toLocaleString()} limit)`; + } + + if (!embeddingSkipped) { + bar.update(90, { phase: 'Loading embedding model...' }); + const t0Emb = Date.now(); await runEmbeddingPipeline( executeQuery, executeWithReusedStatement, (progress) => { - progressBar.update(progress.percent, { phase: `Embeddings ${progress.percent}%` }); - } + const scaled = 90 + Math.round((progress.percent / 100) * 8); + const label = progress.phase === 'loading-model' ? 'Loading embedding model...' : `Embedding ${progress.nodesProcessed || 0}/${progress.totalNodes || '?'}`; + bar.update(scaled, { phase: label }); + }, + {}, + cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined, ); + embeddingTime = ((Date.now() - t0Emb) / 1000).toFixed(1); } - const stats = await getKuzuStats(); + // ── Phase 5: Finalize (98–100%) ─────────────────────────────────── + bar.update(98, { phase: 'Saving metadata...' }); + const meta = { repoPath, lastCommit: currentCommit, @@ -133,14 +277,11 @@ export const analyzeCommand = async ( }, }; await saveMeta(storagePath, meta); - await registerRepo(repoPath, meta); - await addToGitignore(repoPath); - - // Auto-register Claude Code hook (idempotent) + const hookResult = await registerClaudeHook(); - + const projectName = path.basename(repoPath); let aggregatedClusterCount = 0; if (pipelineResult.communityResult?.communities) { @@ -160,25 +301,38 @@ export const analyzeCommand = async ( clusters: aggregatedClusterCount, processes: pipelineResult.processResult?.stats.totalProcesses, }); - - await closeKuzu(); + await closeKuzu(); await disposeEmbedder(); - multibar.stop(); + const totalTime = ((Date.now() - t0Global) / 1000).toFixed(1); + + bar.update(100, { phase: 'Done' }); + bar.stop(); + + // ── Summary ─────────────────────────────────────────────────────── + const modeLabel = pipelineMode === 'incremental' + ? `incremental — ${changedFileList.length} files re-parsed` + : 'full rebuild'; + console.log(`\n Repository indexed successfully (${totalTime}s) [${modeLabel}]\n`); + console.log(` ${stats.nodes.toLocaleString()} nodes | ${stats.edges.toLocaleString()} edges | ${pipelineResult.communityResult?.stats.totalCommunities || 0} clusters | ${pipelineResult.processResult?.stats.totalProcesses || 0} flows`); + console.log(` KuzuDB ${kuzuTime}s | FTS ${ftsTime}s | Embeddings ${embeddingSkipped ? embeddingSkipReason : embeddingTime + 's'}`); + console.log(` ${repoPath}`); - console.log('\n ✓ Repository indexed successfully\n'); - console.log(` Path: ${repoPath}`); - console.log(` Storage: ${storagePath}`); - console.log(` Registry: ${getGlobalDir()}`); - console.log(` Stats: ${stats.nodes} nodes, ${stats.edges} edges, ${pipelineResult.communityResult?.stats.totalCommunities || 0} clusters, ${pipelineResult.processResult?.stats.totalProcesses || 0} processes`); - if (aiContext.files.length > 0) { - console.log(` Context: ${aiContext.files.join(', ')}`); + console.log(` Context: ${aiContext.files.join(', ')}`); } - + if (hookResult.registered) { - console.log(` Hooks: ${hookResult.message}`); + console.log(` Hooks: ${hookResult.message}`); + } + + // Show warnings (missing schema pairs, etc.) after the clean output + if (kuzuWarnings.length > 0) { + console.log(`\n Warnings (${kuzuWarnings.length}):`); + for (const w of kuzuWarnings) { + console.log(` ${w}`); + } } try { diff --git a/gitnexus/src/core/kuzu/kuzu-adapter.ts b/gitnexus/src/core/kuzu/kuzu-adapter.ts index 675251047..0620e774a 100644 --- a/gitnexus/src/core/kuzu/kuzu-adapter.ts +++ b/gitnexus/src/core/kuzu/kuzu-adapter.ts @@ -1,7 +1,7 @@ import fs from 'fs/promises'; import path from 'path'; import kuzu from 'kuzu'; -import { KnowledgeGraph } from '../graph/types.js'; +import { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types.js'; import { NODE_TABLES, REL_TABLE_NAME, @@ -63,128 +63,158 @@ export const initKuzu = async (dbPath: string) => { return { db, conn }; }; +export type KuzuProgressCallback = (message: string) => void; + export const loadGraphToKuzu = async ( graph: KnowledgeGraph, fileContents: Map, - storagePath: string + storagePath: string, + onProgress?: KuzuProgressCallback ) => { if (!conn) { throw new Error('KuzuDB not initialized. Call initKuzu first.'); } + const log = onProgress || (() => {}); + const csvData = generateAllCSVs(graph, fileContents); const csvDir = path.join(storagePath, 'csv'); await fs.mkdir(csvDir, { recursive: true }); - const nodeFiles: Array<{ table: NodeTableName; path: string }> = []; + log('Generating CSVs...'); + + const nodeFiles: Array<{ table: NodeTableName; path: string; rows: number }> = []; for (const [tableName, csv] of csvData.nodes.entries()) { - if (csv.split('\n').length <= 1) continue; + const rowCount = csv.split('\n').length - 1; + if (rowCount <= 0) continue; const filePath = path.join(csvDir, `${tableName.toLowerCase()}.csv`); await fs.writeFile(filePath, csv, 'utf-8'); - nodeFiles.push({ table: tableName, path: filePath }); + nodeFiles.push({ table: tableName, path: filePath, rows: rowCount }); } - const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim()); - - for (const { table, path: filePath } of nodeFiles) { - const normalizedPath = normalizeCopyPath(filePath); - const copyQuery = getCopyQuery(table, normalizedPath); - - if (isDev) { - const csvContent = await fs.readFile(filePath, 'utf-8'); - const csvLines = csvContent.split('\n').length; - console.log(` COPY ${table}: ${csvLines - 1} rows`); - } - - try { - await conn.query(copyQuery); - } catch (err) { - const errMsg = err instanceof Error ? err.message : String(err); - console.warn(`⚠️ COPY failed for ${table}: ${errMsg.slice(0, 100)}`); - - try { - const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)'); - await conn.query(retryQuery); - if (isDev) { - console.log(` ✅ ${table} loaded with IGNORE_ERRORS (some rows may have been skipped)`); - } - } catch (retryErr) { - const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); - console.error(`❌ COPY failed for ${table} even with IGNORE_ERRORS: ${retryMsg}`); - throw retryErr; - } - } - } - - if (isDev) { - console.log('✅ All COPY commands succeeded. Starting relationship insertion...'); - } - - // Build a set of valid table names for fast lookup + // Write relationship CSV to disk for bulk COPY + const relCsvPath = path.join(csvDir, 'relations.csv'); const validTables = new Set(NODE_TABLES as readonly string[]); - const getNodeLabel = (nodeId: string): string => { if (nodeId.startsWith('comm_')) return 'Community'; if (nodeId.startsWith('proc_')) return 'Process'; return nodeId.split(':')[0]; }; - // All multi-language tables are created with backticks - must always reference them with backticks - const escapeLabel = (label: string): string => { - return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label; - }; - - let insertedRels = 0; + const relLines = csvData.relCSV.split('\n'); + const relHeader = relLines[0]; + const validRelLines = [relHeader]; let skippedRels = 0; - for (const line of relLines) { + for (let i = 1; i < relLines.length; i++) { + const line = relLines[i]; + if (!line.trim()) continue; + const match = line.match(/"([^"]*)","([^"]*)"/); + if (!match) { skippedRels++; continue; } + const fromLabel = getNodeLabel(match[1]); + const toLabel = getNodeLabel(match[2]); + if (!validTables.has(fromLabel) || !validTables.has(toLabel)) { + skippedRels++; + continue; + } + validRelLines.push(line); + } + await fs.writeFile(relCsvPath, validRelLines.join('\n'), 'utf-8'); + + // Bulk COPY all node CSVs + const totalSteps = nodeFiles.length + 1; // +1 for relationships + let stepsDone = 0; + + for (const { table, path: filePath, rows } of nodeFiles) { + stepsDone++; + log(`Loading nodes ${stepsDone}/${totalSteps}: ${table} (${rows.toLocaleString()} rows)`); + + const normalizedPath = normalizeCopyPath(filePath); + const copyQuery = getCopyQuery(table, normalizedPath); + try { - const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/); + await conn.query(copyQuery); + } catch (err) { + try { + const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)'); + await conn.query(retryQuery); + } catch (retryErr) { + const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); + throw new Error(`COPY failed for ${table}: ${retryMsg.slice(0, 200)}`); + } + } + } + + // Bulk COPY relationships — split by FROM→TO label pair (KuzuDB requires it) + const insertedRels = validRelLines.length - 1; + const warnings: string[] = []; + if (insertedRels > 0) { + const relsByPair = new Map(); + for (let i = 1; i < validRelLines.length; i++) { + const line = validRelLines[i]; + const match = line.match(/"([^"]*)","([^"]*)"/); if (!match) continue; - const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match; + const fromLabel = getNodeLabel(match[1]); + const toLabel = getNodeLabel(match[2]); + const pairKey = `${fromLabel}|${toLabel}`; + let list = relsByPair.get(pairKey); + if (!list) { list = []; relsByPair.set(pairKey, list); } + list.push(line); + } - const fromLabel = getNodeLabel(fromId); - const toLabel = getNodeLabel(toId); + log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`); - // Skip relationships where either node's label doesn't have a table in KuzuDB - // (e.g. Variable, Import, Type nodes that aren't in the schema) - // Querying a non-existent table causes a fatal native crash - if (!validTables.has(fromLabel) || !validTables.has(toLabel)) { - skippedRels++; - continue; + let pairIdx = 0; + let failedPairEdges = 0; + const failedPairLines: string[] = []; + + for (const [pairKey, lines] of relsByPair) { + pairIdx++; + const [fromLabel, toLabel] = pairKey.split('|'); + const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`); + await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8'); + const normalizedPath = normalizeCopyPath(pairCsvPath); + const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`; + + if (pairIdx % 5 === 0 || lines.length > 1000) { + log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`); } - const confidence = parseFloat(confidenceStr) || 1.0; - const step = parseInt(stepStr) || 0; + try { + await conn.query(copyQuery); + } catch (err) { + try { + const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)'); + await conn.query(retryQuery); + } catch (retryErr) { + const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); + warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`); + failedPairEdges += lines.length; + failedPairLines.push(...lines); + } + } + try { await fs.unlink(pairCsvPath); } catch {} + } - const insertQuery = ` - MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }), - (b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' }) - CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b) - `; - await conn.query(insertQuery); - insertedRels++; - } catch { - skippedRels++; + if (failedPairLines.length > 0) { + log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`); + await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel); } } - // Cleanup CSVs + // Cleanup all CSVs + try { await fs.unlink(relCsvPath); } catch {} for (const { path: filePath } of nodeFiles) { - try { - await fs.unlink(filePath); - } catch { - // ignore - } + try { await fs.unlink(filePath); } catch {} } - - // Remove empty csv directory try { - await fs.rmdir(csvDir); - } catch { - // ignore if not empty or other error - } + const remaining = await fs.readdir(csvDir); + for (const f of remaining) { + try { await fs.unlink(path.join(csvDir, f)); } catch {} + } + } catch {} + try { await fs.rmdir(csvDir); } catch {} - return { success: true, insertedRels, skippedRels }; + return { success: true, insertedRels, skippedRels, warnings }; }; // KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes). @@ -205,6 +235,41 @@ const escapeTableName = (table: string): string => { return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table; }; +/** Fallback: insert relationships one-by-one if COPY fails */ +const fallbackRelationshipInserts = async ( + validRelLines: string[], + validTables: Set, + getNodeLabel: (id: string) => string +) => { + if (!conn) return; + const escapeLabel = (label: string): string => { + return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label; + }; + + for (let i = 1; i < validRelLines.length; i++) { + const line = validRelLines[i]; + try { + const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/); + if (!match) continue; + const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match; + const fromLabel = getNodeLabel(fromId); + const toLabel = getNodeLabel(toId); + if (!validTables.has(fromLabel) || !validTables.has(toLabel)) continue; + + const confidence = parseFloat(confidenceStr) || 1.0; + const step = parseInt(stepStr) || 0; + + await conn.query(` + MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }), + (b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' }) + CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b) + `); + } catch { + // skip + } + } +}; + const getCopyQuery = (table: NodeTableName, filePath: string): string => { const t = escapeTableName(table); if (table === 'File') { @@ -414,6 +479,147 @@ export const getKuzuStats = async (): Promise<{ nodes: number; edges: number }> return { nodes: totalNodes, edges: totalEdges }; }; +/** + * Load existing nodes and relationships from KuzuDB, excluding files in the changed set. + * Used for incremental updates: we keep unchanged data and only re-parse changed files. + */ +export const loadExistingGraph = async ( + changedFiles: Set, + deletedFiles: Set, +): Promise<{ + nodes: GraphNode[]; + relationships: GraphRelationship[]; + symbolEntries: Array<{ filePath: string; name: string; nodeId: string; type: string }>; + embeddingNodeIds: Set; + cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }>; +}> => { + if (!conn) { + return { nodes: [], relationships: [], symbolEntries: [], embeddingNodeIds: new Set(), cachedEmbeddings: [] }; + } + + const str = (v: any): string => String(v ?? ''); + const num = (v: any): number => Number(v) || 0; + + const excludedFiles = new Set([...changedFiles, ...deletedFiles]); + const nodes: GraphNode[] = []; + const symbolEntries: Array<{ filePath: string; name: string; nodeId: string; type: string }> = []; + + const codeElementTables: NodeTableName[] = NODE_TABLES.filter( + t => t !== 'File' && t !== 'Folder' && t !== 'Community' && t !== 'Process' + ) as NodeTableName[]; + + // Load File nodes + try { + const rows = await conn.query('MATCH (n:File) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.content AS content'); + const result = Array.isArray(rows) ? rows[0] : rows; + for (const row of await result.getAll()) { + const filePath = str(row.filePath ?? row[2]).replace(/\\/g, '/'); + if (excludedFiles.has(filePath)) continue; + nodes.push({ + id: str(row.id ?? row[0]), + label: 'File', + properties: { name: str(row.name ?? row[1]), filePath }, + }); + } + } catch { /* table may not exist */ } + + // Load Folder nodes + try { + const rows = await conn.query('MATCH (n:Folder) RETURN n.id AS id, n.name AS name, n.filePath AS filePath'); + const result = Array.isArray(rows) ? rows[0] : rows; + for (const row of await result.getAll()) { + nodes.push({ + id: str(row.id ?? row[0]), + label: 'Folder', + properties: { name: str(row.name ?? row[1]), filePath: str(row.filePath ?? row[2]).replace(/\\/g, '/') }, + }); + } + } catch { /* table may not exist */ } + + // Tables with isExported column (standard JS/TS code element tables) + const tablesWithExported = new Set(['Function', 'Class', 'Interface', 'Method', 'CodeElement']); + + // Load code element nodes (Function, Class, Method, Interface, etc.) + for (const table of codeElementTables) { + try { + const t = BACKTICK_TABLES.has(table) ? `\`${table}\`` : table; + const hasExported = tablesWithExported.has(table); + const query = hasExported + ? `MATCH (n:${t}) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine, n.isExported AS isExported` + : `MATCH (n:${t}) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine`; + const rows = await conn.query(query); + const result = Array.isArray(rows) ? rows[0] : rows; + for (const row of await result.getAll()) { + const filePath = str(row.filePath ?? row[2]).replace(/\\/g, '/'); + if (excludedFiles.has(filePath)) continue; + const nodeId = str(row.id ?? row[0]); + const name = str(row.name ?? row[1]); + nodes.push({ + id: nodeId, + label: table as any, + properties: { + name, + filePath, + startLine: num(row.startLine ?? row[3]), + endLine: num(row.endLine ?? row[4]), + ...(hasExported ? { isExported: !!(row.isExported ?? row[5]) } : {}), + }, + }); + symbolEntries.push({ filePath, name, nodeId, type: table }); + } + } catch { /* table may not exist or is empty */ } + } + + // Load relationships (exclude any involving changed/deleted files) + const relationships: GraphRelationship[] = []; + const nodeIdSet = new Set(nodes.map(n => n.id)); + try { + const rows = await conn.query( + `MATCH (a)-[r:${REL_TABLE_NAME}]->(b) RETURN a.id AS fromId, b.id AS toId, r.type AS type, r.confidence AS confidence, r.reason AS reason, r.step AS step` + ); + const result = Array.isArray(rows) ? rows[0] : rows; + for (const row of await result.getAll()) { + const fromId = str(row.fromId ?? row[0]); + const toId = str(row.toId ?? row[1]); + if (!nodeIdSet.has(fromId) || !nodeIdSet.has(toId)) continue; + const type = str(row.type ?? row[2]); + if (type === 'MEMBER_OF' || type === 'STEP_IN_PROCESS') continue; + relationships.push({ + id: `${fromId}_${type}_${toId}`, + sourceId: fromId, + targetId: toId, + type: type as any, + confidence: num(row.confidence ?? row[3]) || 1.0, + reason: str(row.reason ?? row[4]), + step: num(row.step ?? row[5]), + }); + } + } catch { /* relationship table may not exist */ } + + // Load existing embeddings (nodeId + vector) for unchanged nodes + const embeddingNodeIds = new Set(); + const cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }> = []; + try { + const rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding`); + const result = Array.isArray(rows) ? rows[0] : rows; + for (const row of await result.getAll()) { + const nodeId = str(row.nodeId ?? row[0]); + if (nodeIdSet.has(nodeId)) { + embeddingNodeIds.add(nodeId); + const embedding = row.embedding ?? row[1]; + if (embedding) { + cachedEmbeddings.push({ + nodeId, + embedding: Array.isArray(embedding) ? embedding.map(Number) : Array.from(embedding as any).map(Number), + }); + } + } + } + } catch { /* embedding table may not exist */ } + + return { nodes, relationships, symbolEntries, embeddingNodeIds, cachedEmbeddings }; +}; + export const closeKuzu = async (): Promise => { if (conn) { try { diff --git a/gitnexus/src/storage/git.ts b/gitnexus/src/storage/git.ts index 9a547b0c5..dd127f2f9 100644 --- a/gitnexus/src/storage/git.ts +++ b/gitnexus/src/storage/git.ts @@ -72,3 +72,20 @@ export const getDeletedFiles = (fromCommit: string, toCommit: string, repoPath: } }; +/** + * Get files with uncommitted changes (working tree vs HEAD). + * This catches staged + unstaged modifications that aren't in any commit yet. + * Returns relative paths (forward-slash normalized). + */ +export const getUncommittedChanges = (repoPath: string): string[] => { + try { + const output = execSync( + 'git diff HEAD --name-only --diff-filter=ACMR', + { cwd: repoPath, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 } + ).trim(); + return output ? output.split('\n').filter(Boolean).map(f => f.replace(/\\/g, '/')) : []; + } catch { + return []; + } +}; +