mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-11-13 08:51:28 +00:00
fix: include uncommitted working tree changes in incremental diff detection
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
abhigyanpatwari
parent
f8ac2a6a2e
commit
de98efec6a
3 changed files with 508 additions and 131 deletions
|
|
@ -1,17 +1,17 @@
|
|||
/**
|
||||
* Analyze Command
|
||||
*
|
||||
*
|
||||
* Indexes a repository and stores the knowledge graph in .gitnexus/
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import cliProgress from 'cli-progress';
|
||||
import { runPipelineFromRepo } from '../core/ingestion/pipeline.js';
|
||||
import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex } from '../core/kuzu/kuzu-adapter.js';
|
||||
import { runPipelineFromRepo, runIncrementalPipeline } from '../core/ingestion/pipeline.js';
|
||||
import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex, loadExistingGraph } from '../core/kuzu/kuzu-adapter.js';
|
||||
import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js';
|
||||
import { disposeEmbedder } from '../core/embeddings/embedder.js';
|
||||
import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, getGlobalRegistryPath, getGlobalDir } from '../storage/repo-manager.js';
|
||||
import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js';
|
||||
import { getCurrentCommit, isGitRepo, getGitRoot, getChangedFiles, getDeletedFiles, getUncommittedChanges } from '../storage/git.js';
|
||||
import { generateAIContextFiles } from './ai-context.js';
|
||||
import fs from 'fs/promises';
|
||||
import { registerClaudeHook } from './claude-hooks.js';
|
||||
|
|
@ -21,6 +21,9 @@ export interface AnalyzeOptions {
|
|||
skipEmbeddings?: boolean;
|
||||
}
|
||||
|
||||
/** Threshold: auto-skip embeddings for repos with more nodes than this */
|
||||
const EMBEDDING_NODE_LIMIT = 50_000;
|
||||
|
||||
const PHASE_LABELS: Record<string, string> = {
|
||||
extracting: 'Scanning files',
|
||||
structure: 'Building structure',
|
||||
|
|
@ -30,7 +33,11 @@ const PHASE_LABELS: Record<string, string> = {
|
|||
heritage: 'Extracting inheritance',
|
||||
communities: 'Detecting communities',
|
||||
processes: 'Detecting processes',
|
||||
complete: 'Complete',
|
||||
complete: 'Pipeline complete',
|
||||
kuzu: 'Loading into KuzuDB',
|
||||
fts: 'Creating search indexes',
|
||||
embeddings: 'Generating embeddings',
|
||||
done: 'Done',
|
||||
};
|
||||
|
||||
export const analyzeCommand = async (
|
||||
|
|
@ -45,7 +52,7 @@ export const analyzeCommand = async (
|
|||
} else {
|
||||
const gitRoot = getGitRoot(process.cwd());
|
||||
if (!gitRoot) {
|
||||
console.log(' ✗ Not inside a git repository\n');
|
||||
console.log(' Not inside a git repository\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
|
@ -53,7 +60,7 @@ export const analyzeCommand = async (
|
|||
}
|
||||
|
||||
if (!isGitRepo(repoPath)) {
|
||||
console.log(' ✗ Not a git repository\n');
|
||||
console.log(' Not a git repository\n');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
|
@ -63,41 +70,139 @@ export const analyzeCommand = async (
|
|||
const existingMeta = await loadMeta(storagePath);
|
||||
|
||||
if (existingMeta && !options?.force && existingMeta.lastCommit === currentCommit) {
|
||||
console.log(' ✓ Repository already up to date\n');
|
||||
console.log(' Already up to date\n');
|
||||
return;
|
||||
}
|
||||
|
||||
const multibar = new cliProgress.MultiBar({
|
||||
// ── Determine incremental vs full rebuild ──────────────────────────
|
||||
const INCREMENTAL_THRESHOLD = 0.6; // Fall back to full if >60% files changed
|
||||
let useIncremental = false;
|
||||
let changedFileList: string[] = [];
|
||||
let deletedFileList: string[] = [];
|
||||
let cachedEmbeddingNodeIds = new Set<string>();
|
||||
let cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }> = [];
|
||||
|
||||
if (existingMeta?.lastCommit && !options?.force) {
|
||||
changedFileList = getChangedFiles(existingMeta.lastCommit, currentCommit, repoPath);
|
||||
deletedFileList = getDeletedFiles(existingMeta.lastCommit, currentCommit, repoPath);
|
||||
|
||||
// Also include uncommitted working tree changes — the pipeline reads from disk,
|
||||
// so any file modified in the working tree needs re-parsing even if not committed.
|
||||
const uncommitted = getUncommittedChanges(repoPath);
|
||||
if (uncommitted.length > 0) {
|
||||
const changedSet = new Set(changedFileList);
|
||||
for (const f of uncommitted) changedSet.add(f);
|
||||
changedFileList = Array.from(changedSet);
|
||||
}
|
||||
|
||||
// We'll check the ratio after scanning total file count below
|
||||
if (changedFileList.length > 0 || deletedFileList.length > 0) {
|
||||
useIncremental = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Single progress bar for entire pipeline
|
||||
const bar = new cliProgress.SingleBar({
|
||||
format: ' {bar} {percentage}% | {phase}',
|
||||
barCompleteChar: '█',
|
||||
barIncompleteChar: '░',
|
||||
barCompleteChar: '\u2588',
|
||||
barIncompleteChar: '\u2591',
|
||||
hideCursor: true,
|
||||
barGlue: '',
|
||||
autopadding: true,
|
||||
clearOnComplete: false,
|
||||
stopOnComplete: false,
|
||||
}, cliProgress.Presets.shades_grey);
|
||||
|
||||
const progressBar = multibar.create(100, 0, { phase: 'Initializing...' });
|
||||
bar.start(100, 0, { phase: 'Initializing...' });
|
||||
|
||||
const pipelineResult = await runPipelineFromRepo(repoPath, (progress) => {
|
||||
const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase;
|
||||
progressBar.update(progress.percent, { phase: phaseLabel });
|
||||
});
|
||||
const t0Global = Date.now();
|
||||
let pipelineMode: 'full' | 'incremental' = 'full';
|
||||
|
||||
// ── Load existing graph for incremental mode ───────────────────────
|
||||
let preloadedData: Awaited<ReturnType<typeof loadExistingGraph>> | null = null;
|
||||
|
||||
if (useIncremental) {
|
||||
try {
|
||||
bar.update(0, { phase: 'Loading cached graph...' });
|
||||
await initKuzu(kuzuPath);
|
||||
preloadedData = await loadExistingGraph(
|
||||
new Set(changedFileList),
|
||||
new Set(deletedFileList),
|
||||
);
|
||||
await closeKuzu();
|
||||
|
||||
// Check threshold: if too many files changed, fall back to full rebuild
|
||||
const totalCachedFiles = preloadedData.nodes.filter(n => n.label === 'File').length;
|
||||
const totalFiles = totalCachedFiles + changedFileList.length;
|
||||
const changeRatio = totalFiles > 0 ? changedFileList.length / totalFiles : 1;
|
||||
|
||||
if (changeRatio > INCREMENTAL_THRESHOLD) {
|
||||
preloadedData = null;
|
||||
useIncremental = false;
|
||||
} else {
|
||||
pipelineMode = 'incremental';
|
||||
cachedEmbeddingNodeIds = preloadedData.embeddingNodeIds;
|
||||
cachedEmbeddings = preloadedData.cachedEmbeddings;
|
||||
}
|
||||
} catch {
|
||||
// KuzuDB load failed — fall back to full rebuild
|
||||
preloadedData = null;
|
||||
useIncremental = false;
|
||||
try { await closeKuzu(); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Phase 1: Pipeline (0–60%) ──────────────────────────────────────
|
||||
let pipelineResult;
|
||||
|
||||
if (pipelineMode === 'incremental' && preloadedData) {
|
||||
bar.update(0, { phase: `Incremental: ${changedFileList.length} files changed` });
|
||||
pipelineResult = await runIncrementalPipeline(
|
||||
repoPath,
|
||||
new Set(changedFileList),
|
||||
{
|
||||
nodes: preloadedData.nodes,
|
||||
relationships: preloadedData.relationships,
|
||||
symbolEntries: preloadedData.symbolEntries,
|
||||
},
|
||||
(progress) => {
|
||||
const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase;
|
||||
const scaled = Math.round(progress.percent * 0.6);
|
||||
bar.update(scaled, { phase: phaseLabel });
|
||||
}
|
||||
);
|
||||
} else {
|
||||
pipelineResult = await runPipelineFromRepo(repoPath, (progress) => {
|
||||
const phaseLabel = PHASE_LABELS[progress.phase] || progress.phase;
|
||||
const scaled = Math.round(progress.percent * 0.6);
|
||||
bar.update(scaled, { phase: phaseLabel });
|
||||
});
|
||||
}
|
||||
|
||||
// ── Phase 2: KuzuDB (60–85%) ──────────────────────────────────────
|
||||
bar.update(60, { phase: 'Loading into KuzuDB...' });
|
||||
|
||||
progressBar.update(100, { phase: 'Loading graph into KuzuDB...' });
|
||||
|
||||
await closeKuzu();
|
||||
|
||||
const fsClean = await import('fs/promises');
|
||||
const kuzuFiles = [kuzuPath, `${kuzuPath}.wal`, `${kuzuPath}.lock`];
|
||||
for (const f of kuzuFiles) {
|
||||
try { await fsClean.rm(f, { recursive: true, force: true }); } catch { /* may not exist */ }
|
||||
try { await fs.rm(f, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
|
||||
|
||||
const t0Kuzu = Date.now();
|
||||
await initKuzu(kuzuPath);
|
||||
await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath);
|
||||
let kuzuMsgCount = 0;
|
||||
const kuzuResult = await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath, (msg) => {
|
||||
kuzuMsgCount++;
|
||||
const progress = Math.min(84, 60 + Math.round((kuzuMsgCount / (kuzuMsgCount + 10)) * 24));
|
||||
bar.update(progress, { phase: msg });
|
||||
});
|
||||
const kuzuTime = ((Date.now() - t0Kuzu) / 1000).toFixed(1);
|
||||
const kuzuWarnings = kuzuResult.warnings;
|
||||
|
||||
progressBar.update(100, { phase: 'Creating search indexes...' });
|
||||
// ── Phase 3: FTS (85–90%) ─────────────────────────────────────────
|
||||
bar.update(85, { phase: 'Creating search indexes...' });
|
||||
|
||||
const t0Fts = Date.now();
|
||||
try {
|
||||
await createFTSIndex('File', 'file_fts', ['name', 'content']);
|
||||
await createFTSIndex('Function', 'function_fts', ['name', 'content']);
|
||||
|
|
@ -105,21 +210,60 @@ export const analyzeCommand = async (
|
|||
await createFTSIndex('Method', 'method_fts', ['name', 'content']);
|
||||
await createFTSIndex('Interface', 'interface_fts', ['name', 'content']);
|
||||
} catch (e: any) {
|
||||
console.error(' Note: Some FTS indexes may not have been created:', e.message);
|
||||
// Non-fatal — FTS is best-effort
|
||||
}
|
||||
const ftsTime = ((Date.now() - t0Fts) / 1000).toFixed(1);
|
||||
|
||||
// ── Phase 3.5: Re-insert cached embeddings (incremental) ───────────
|
||||
if (cachedEmbeddings.length > 0) {
|
||||
bar.update(88, { phase: `Restoring ${cachedEmbeddings.length} cached embeddings...` });
|
||||
const EMBED_BATCH = 200;
|
||||
for (let i = 0; i < cachedEmbeddings.length; i += EMBED_BATCH) {
|
||||
const batch = cachedEmbeddings.slice(i, i + EMBED_BATCH);
|
||||
const paramsList = batch.map(e => ({ nodeId: e.nodeId, embedding: e.embedding }));
|
||||
try {
|
||||
await executeWithReusedStatement(
|
||||
`CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`,
|
||||
paramsList,
|
||||
);
|
||||
} catch { /* some may fail if node was removed, that's fine */ }
|
||||
}
|
||||
}
|
||||
|
||||
if (!options?.skipEmbeddings) {
|
||||
progressBar.update(100, { phase: 'Generating embeddings...' });
|
||||
// ── Phase 4: Embeddings (90–98%) ──────────────────────────────────
|
||||
const stats = await getKuzuStats();
|
||||
let embeddingTime = '0.0';
|
||||
let embeddingSkipped = false;
|
||||
let embeddingSkipReason = '';
|
||||
|
||||
if (options?.skipEmbeddings) {
|
||||
embeddingSkipped = true;
|
||||
embeddingSkipReason = 'skipped (--skip-embeddings)';
|
||||
} else if (stats.nodes > EMBEDDING_NODE_LIMIT) {
|
||||
embeddingSkipped = true;
|
||||
embeddingSkipReason = `skipped (${stats.nodes.toLocaleString()} nodes > ${EMBEDDING_NODE_LIMIT.toLocaleString()} limit)`;
|
||||
}
|
||||
|
||||
if (!embeddingSkipped) {
|
||||
bar.update(90, { phase: 'Loading embedding model...' });
|
||||
const t0Emb = Date.now();
|
||||
await runEmbeddingPipeline(
|
||||
executeQuery,
|
||||
executeWithReusedStatement,
|
||||
(progress) => {
|
||||
progressBar.update(progress.percent, { phase: `Embeddings ${progress.percent}%` });
|
||||
}
|
||||
const scaled = 90 + Math.round((progress.percent / 100) * 8);
|
||||
const label = progress.phase === 'loading-model' ? 'Loading embedding model...' : `Embedding ${progress.nodesProcessed || 0}/${progress.totalNodes || '?'}`;
|
||||
bar.update(scaled, { phase: label });
|
||||
},
|
||||
{},
|
||||
cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined,
|
||||
);
|
||||
embeddingTime = ((Date.now() - t0Emb) / 1000).toFixed(1);
|
||||
}
|
||||
|
||||
const stats = await getKuzuStats();
|
||||
// ── Phase 5: Finalize (98–100%) ───────────────────────────────────
|
||||
bar.update(98, { phase: 'Saving metadata...' });
|
||||
|
||||
const meta = {
|
||||
repoPath,
|
||||
lastCommit: currentCommit,
|
||||
|
|
@ -133,14 +277,11 @@ export const analyzeCommand = async (
|
|||
},
|
||||
};
|
||||
await saveMeta(storagePath, meta);
|
||||
|
||||
await registerRepo(repoPath, meta);
|
||||
|
||||
await addToGitignore(repoPath);
|
||||
|
||||
// Auto-register Claude Code hook (idempotent)
|
||||
|
||||
const hookResult = await registerClaudeHook();
|
||||
|
||||
|
||||
const projectName = path.basename(repoPath);
|
||||
let aggregatedClusterCount = 0;
|
||||
if (pipelineResult.communityResult?.communities) {
|
||||
|
|
@ -160,25 +301,38 @@ export const analyzeCommand = async (
|
|||
clusters: aggregatedClusterCount,
|
||||
processes: pipelineResult.processResult?.stats.totalProcesses,
|
||||
});
|
||||
|
||||
await closeKuzu();
|
||||
|
||||
await closeKuzu();
|
||||
await disposeEmbedder();
|
||||
|
||||
multibar.stop();
|
||||
const totalTime = ((Date.now() - t0Global) / 1000).toFixed(1);
|
||||
|
||||
bar.update(100, { phase: 'Done' });
|
||||
bar.stop();
|
||||
|
||||
// ── Summary ───────────────────────────────────────────────────────
|
||||
const modeLabel = pipelineMode === 'incremental'
|
||||
? `incremental — ${changedFileList.length} files re-parsed`
|
||||
: 'full rebuild';
|
||||
console.log(`\n Repository indexed successfully (${totalTime}s) [${modeLabel}]\n`);
|
||||
console.log(` ${stats.nodes.toLocaleString()} nodes | ${stats.edges.toLocaleString()} edges | ${pipelineResult.communityResult?.stats.totalCommunities || 0} clusters | ${pipelineResult.processResult?.stats.totalProcesses || 0} flows`);
|
||||
console.log(` KuzuDB ${kuzuTime}s | FTS ${ftsTime}s | Embeddings ${embeddingSkipped ? embeddingSkipReason : embeddingTime + 's'}`);
|
||||
console.log(` ${repoPath}`);
|
||||
|
||||
console.log('\n ✓ Repository indexed successfully\n');
|
||||
console.log(` Path: ${repoPath}`);
|
||||
console.log(` Storage: ${storagePath}`);
|
||||
console.log(` Registry: ${getGlobalDir()}`);
|
||||
console.log(` Stats: ${stats.nodes} nodes, ${stats.edges} edges, ${pipelineResult.communityResult?.stats.totalCommunities || 0} clusters, ${pipelineResult.processResult?.stats.totalProcesses || 0} processes`);
|
||||
|
||||
if (aiContext.files.length > 0) {
|
||||
console.log(` Context: ${aiContext.files.join(', ')}`);
|
||||
console.log(` Context: ${aiContext.files.join(', ')}`);
|
||||
}
|
||||
|
||||
|
||||
if (hookResult.registered) {
|
||||
console.log(` Hooks: ${hookResult.message}`);
|
||||
console.log(` Hooks: ${hookResult.message}`);
|
||||
}
|
||||
|
||||
// Show warnings (missing schema pairs, etc.) after the clean output
|
||||
if (kuzuWarnings.length > 0) {
|
||||
console.log(`\n Warnings (${kuzuWarnings.length}):`);
|
||||
for (const w of kuzuWarnings) {
|
||||
console.log(` ${w}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import kuzu from 'kuzu';
|
||||
import { KnowledgeGraph } from '../graph/types.js';
|
||||
import { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types.js';
|
||||
import {
|
||||
NODE_TABLES,
|
||||
REL_TABLE_NAME,
|
||||
|
|
@ -63,128 +63,158 @@ export const initKuzu = async (dbPath: string) => {
|
|||
return { db, conn };
|
||||
};
|
||||
|
||||
export type KuzuProgressCallback = (message: string) => void;
|
||||
|
||||
export const loadGraphToKuzu = async (
|
||||
graph: KnowledgeGraph,
|
||||
fileContents: Map<string, string>,
|
||||
storagePath: string
|
||||
storagePath: string,
|
||||
onProgress?: KuzuProgressCallback
|
||||
) => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
}
|
||||
|
||||
const log = onProgress || (() => {});
|
||||
|
||||
const csvData = generateAllCSVs(graph, fileContents);
|
||||
const csvDir = path.join(storagePath, 'csv');
|
||||
await fs.mkdir(csvDir, { recursive: true });
|
||||
|
||||
const nodeFiles: Array<{ table: NodeTableName; path: string }> = [];
|
||||
log('Generating CSVs...');
|
||||
|
||||
const nodeFiles: Array<{ table: NodeTableName; path: string; rows: number }> = [];
|
||||
for (const [tableName, csv] of csvData.nodes.entries()) {
|
||||
if (csv.split('\n').length <= 1) continue;
|
||||
const rowCount = csv.split('\n').length - 1;
|
||||
if (rowCount <= 0) continue;
|
||||
const filePath = path.join(csvDir, `${tableName.toLowerCase()}.csv`);
|
||||
await fs.writeFile(filePath, csv, 'utf-8');
|
||||
nodeFiles.push({ table: tableName, path: filePath });
|
||||
nodeFiles.push({ table: tableName, path: filePath, rows: rowCount });
|
||||
}
|
||||
|
||||
const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim());
|
||||
|
||||
for (const { table, path: filePath } of nodeFiles) {
|
||||
const normalizedPath = normalizeCopyPath(filePath);
|
||||
const copyQuery = getCopyQuery(table, normalizedPath);
|
||||
|
||||
if (isDev) {
|
||||
const csvContent = await fs.readFile(filePath, 'utf-8');
|
||||
const csvLines = csvContent.split('\n').length;
|
||||
console.log(` COPY ${table}: ${csvLines - 1} rows`);
|
||||
}
|
||||
|
||||
try {
|
||||
await conn.query(copyQuery);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`⚠️ COPY failed for ${table}: ${errMsg.slice(0, 100)}`);
|
||||
|
||||
try {
|
||||
const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)');
|
||||
await conn.query(retryQuery);
|
||||
if (isDev) {
|
||||
console.log(` ✅ ${table} loaded with IGNORE_ERRORS (some rows may have been skipped)`);
|
||||
}
|
||||
} catch (retryErr) {
|
||||
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
|
||||
console.error(`❌ COPY failed for ${table} even with IGNORE_ERRORS: ${retryMsg}`);
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log('✅ All COPY commands succeeded. Starting relationship insertion...');
|
||||
}
|
||||
|
||||
// Build a set of valid table names for fast lookup
|
||||
// Write relationship CSV to disk for bulk COPY
|
||||
const relCsvPath = path.join(csvDir, 'relations.csv');
|
||||
const validTables = new Set<string>(NODE_TABLES as readonly string[]);
|
||||
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) return 'Community';
|
||||
if (nodeId.startsWith('proc_')) return 'Process';
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
|
||||
// All multi-language tables are created with backticks - must always reference them with backticks
|
||||
const escapeLabel = (label: string): string => {
|
||||
return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
let insertedRels = 0;
|
||||
const relLines = csvData.relCSV.split('\n');
|
||||
const relHeader = relLines[0];
|
||||
const validRelLines = [relHeader];
|
||||
let skippedRels = 0;
|
||||
for (const line of relLines) {
|
||||
for (let i = 1; i < relLines.length; i++) {
|
||||
const line = relLines[i];
|
||||
if (!line.trim()) continue;
|
||||
const match = line.match(/"([^"]*)","([^"]*)"/);
|
||||
if (!match) { skippedRels++; continue; }
|
||||
const fromLabel = getNodeLabel(match[1]);
|
||||
const toLabel = getNodeLabel(match[2]);
|
||||
if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
|
||||
skippedRels++;
|
||||
continue;
|
||||
}
|
||||
validRelLines.push(line);
|
||||
}
|
||||
await fs.writeFile(relCsvPath, validRelLines.join('\n'), 'utf-8');
|
||||
|
||||
// Bulk COPY all node CSVs
|
||||
const totalSteps = nodeFiles.length + 1; // +1 for relationships
|
||||
let stepsDone = 0;
|
||||
|
||||
for (const { table, path: filePath, rows } of nodeFiles) {
|
||||
stepsDone++;
|
||||
log(`Loading nodes ${stepsDone}/${totalSteps}: ${table} (${rows.toLocaleString()} rows)`);
|
||||
|
||||
const normalizedPath = normalizeCopyPath(filePath);
|
||||
const copyQuery = getCopyQuery(table, normalizedPath);
|
||||
|
||||
try {
|
||||
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
|
||||
await conn.query(copyQuery);
|
||||
} catch (err) {
|
||||
try {
|
||||
const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)');
|
||||
await conn.query(retryQuery);
|
||||
} catch (retryErr) {
|
||||
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
|
||||
throw new Error(`COPY failed for ${table}: ${retryMsg.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bulk COPY relationships — split by FROM→TO label pair (KuzuDB requires it)
|
||||
const insertedRels = validRelLines.length - 1;
|
||||
const warnings: string[] = [];
|
||||
if (insertedRels > 0) {
|
||||
const relsByPair = new Map<string, string[]>();
|
||||
for (let i = 1; i < validRelLines.length; i++) {
|
||||
const line = validRelLines[i];
|
||||
const match = line.match(/"([^"]*)","([^"]*)"/);
|
||||
if (!match) continue;
|
||||
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
|
||||
const fromLabel = getNodeLabel(match[1]);
|
||||
const toLabel = getNodeLabel(match[2]);
|
||||
const pairKey = `${fromLabel}|${toLabel}`;
|
||||
let list = relsByPair.get(pairKey);
|
||||
if (!list) { list = []; relsByPair.set(pairKey, list); }
|
||||
list.push(line);
|
||||
}
|
||||
|
||||
const fromLabel = getNodeLabel(fromId);
|
||||
const toLabel = getNodeLabel(toId);
|
||||
log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`);
|
||||
|
||||
// Skip relationships where either node's label doesn't have a table in KuzuDB
|
||||
// (e.g. Variable, Import, Type nodes that aren't in the schema)
|
||||
// Querying a non-existent table causes a fatal native crash
|
||||
if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
|
||||
skippedRels++;
|
||||
continue;
|
||||
let pairIdx = 0;
|
||||
let failedPairEdges = 0;
|
||||
const failedPairLines: string[] = [];
|
||||
|
||||
for (const [pairKey, lines] of relsByPair) {
|
||||
pairIdx++;
|
||||
const [fromLabel, toLabel] = pairKey.split('|');
|
||||
const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
|
||||
await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
|
||||
const normalizedPath = normalizeCopyPath(pairCsvPath);
|
||||
const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
|
||||
|
||||
if (pairIdx % 5 === 0 || lines.length > 1000) {
|
||||
log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
|
||||
}
|
||||
|
||||
const confidence = parseFloat(confidenceStr) || 1.0;
|
||||
const step = parseInt(stepStr) || 0;
|
||||
try {
|
||||
await conn.query(copyQuery);
|
||||
} catch (err) {
|
||||
try {
|
||||
const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)');
|
||||
await conn.query(retryQuery);
|
||||
} catch (retryErr) {
|
||||
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
|
||||
warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`);
|
||||
failedPairEdges += lines.length;
|
||||
failedPairLines.push(...lines);
|
||||
}
|
||||
}
|
||||
try { await fs.unlink(pairCsvPath); } catch {}
|
||||
}
|
||||
|
||||
const insertQuery = `
|
||||
MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }),
|
||||
(b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' })
|
||||
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
|
||||
`;
|
||||
await conn.query(insertQuery);
|
||||
insertedRels++;
|
||||
} catch {
|
||||
skippedRels++;
|
||||
if (failedPairLines.length > 0) {
|
||||
log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
|
||||
await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel);
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup CSVs
|
||||
// Cleanup all CSVs
|
||||
try { await fs.unlink(relCsvPath); } catch {}
|
||||
for (const { path: filePath } of nodeFiles) {
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try { await fs.unlink(filePath); } catch {}
|
||||
}
|
||||
|
||||
// Remove empty csv directory
|
||||
try {
|
||||
await fs.rmdir(csvDir);
|
||||
} catch {
|
||||
// ignore if not empty or other error
|
||||
}
|
||||
const remaining = await fs.readdir(csvDir);
|
||||
for (const f of remaining) {
|
||||
try { await fs.unlink(path.join(csvDir, f)); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
try { await fs.rmdir(csvDir); } catch {}
|
||||
|
||||
return { success: true, insertedRels, skippedRels };
|
||||
return { success: true, insertedRels, skippedRels, warnings };
|
||||
};
|
||||
|
||||
// KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes).
|
||||
|
|
@ -205,6 +235,41 @@ const escapeTableName = (table: string): string => {
|
|||
return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table;
|
||||
};
|
||||
|
||||
/** Fallback: insert relationships one-by-one if COPY fails */
|
||||
const fallbackRelationshipInserts = async (
|
||||
validRelLines: string[],
|
||||
validTables: Set<string>,
|
||||
getNodeLabel: (id: string) => string
|
||||
) => {
|
||||
if (!conn) return;
|
||||
const escapeLabel = (label: string): string => {
|
||||
return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
for (let i = 1; i < validRelLines.length; i++) {
|
||||
const line = validRelLines[i];
|
||||
try {
|
||||
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
|
||||
if (!match) continue;
|
||||
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
|
||||
const fromLabel = getNodeLabel(fromId);
|
||||
const toLabel = getNodeLabel(toId);
|
||||
if (!validTables.has(fromLabel) || !validTables.has(toLabel)) continue;
|
||||
|
||||
const confidence = parseFloat(confidenceStr) || 1.0;
|
||||
const step = parseInt(stepStr) || 0;
|
||||
|
||||
await conn.query(`
|
||||
MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }),
|
||||
(b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' })
|
||||
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
|
||||
`);
|
||||
} catch {
|
||||
// skip
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const getCopyQuery = (table: NodeTableName, filePath: string): string => {
|
||||
const t = escapeTableName(table);
|
||||
if (table === 'File') {
|
||||
|
|
@ -414,6 +479,147 @@ export const getKuzuStats = async (): Promise<{ nodes: number; edges: number }>
|
|||
return { nodes: totalNodes, edges: totalEdges };
|
||||
};
|
||||
|
||||
/**
|
||||
* Load existing nodes and relationships from KuzuDB, excluding files in the changed set.
|
||||
* Used for incremental updates: we keep unchanged data and only re-parse changed files.
|
||||
*/
|
||||
export const loadExistingGraph = async (
|
||||
changedFiles: Set<string>,
|
||||
deletedFiles: Set<string>,
|
||||
): Promise<{
|
||||
nodes: GraphNode[];
|
||||
relationships: GraphRelationship[];
|
||||
symbolEntries: Array<{ filePath: string; name: string; nodeId: string; type: string }>;
|
||||
embeddingNodeIds: Set<string>;
|
||||
cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }>;
|
||||
}> => {
|
||||
if (!conn) {
|
||||
return { nodes: [], relationships: [], symbolEntries: [], embeddingNodeIds: new Set(), cachedEmbeddings: [] };
|
||||
}
|
||||
|
||||
const str = (v: any): string => String(v ?? '');
|
||||
const num = (v: any): number => Number(v) || 0;
|
||||
|
||||
const excludedFiles = new Set([...changedFiles, ...deletedFiles]);
|
||||
const nodes: GraphNode[] = [];
|
||||
const symbolEntries: Array<{ filePath: string; name: string; nodeId: string; type: string }> = [];
|
||||
|
||||
const codeElementTables: NodeTableName[] = NODE_TABLES.filter(
|
||||
t => t !== 'File' && t !== 'Folder' && t !== 'Community' && t !== 'Process'
|
||||
) as NodeTableName[];
|
||||
|
||||
// Load File nodes
|
||||
try {
|
||||
const rows = await conn.query('MATCH (n:File) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.content AS content');
|
||||
const result = Array.isArray(rows) ? rows[0] : rows;
|
||||
for (const row of await result.getAll()) {
|
||||
const filePath = str(row.filePath ?? row[2]).replace(/\\/g, '/');
|
||||
if (excludedFiles.has(filePath)) continue;
|
||||
nodes.push({
|
||||
id: str(row.id ?? row[0]),
|
||||
label: 'File',
|
||||
properties: { name: str(row.name ?? row[1]), filePath },
|
||||
});
|
||||
}
|
||||
} catch { /* table may not exist */ }
|
||||
|
||||
// Load Folder nodes
|
||||
try {
|
||||
const rows = await conn.query('MATCH (n:Folder) RETURN n.id AS id, n.name AS name, n.filePath AS filePath');
|
||||
const result = Array.isArray(rows) ? rows[0] : rows;
|
||||
for (const row of await result.getAll()) {
|
||||
nodes.push({
|
||||
id: str(row.id ?? row[0]),
|
||||
label: 'Folder',
|
||||
properties: { name: str(row.name ?? row[1]), filePath: str(row.filePath ?? row[2]).replace(/\\/g, '/') },
|
||||
});
|
||||
}
|
||||
} catch { /* table may not exist */ }
|
||||
|
||||
// Tables with isExported column (standard JS/TS code element tables)
|
||||
const tablesWithExported = new Set(['Function', 'Class', 'Interface', 'Method', 'CodeElement']);
|
||||
|
||||
// Load code element nodes (Function, Class, Method, Interface, etc.)
|
||||
for (const table of codeElementTables) {
|
||||
try {
|
||||
const t = BACKTICK_TABLES.has(table) ? `\`${table}\`` : table;
|
||||
const hasExported = tablesWithExported.has(table);
|
||||
const query = hasExported
|
||||
? `MATCH (n:${t}) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine, n.isExported AS isExported`
|
||||
: `MATCH (n:${t}) RETURN n.id AS id, n.name AS name, n.filePath AS filePath, n.startLine AS startLine, n.endLine AS endLine`;
|
||||
const rows = await conn.query(query);
|
||||
const result = Array.isArray(rows) ? rows[0] : rows;
|
||||
for (const row of await result.getAll()) {
|
||||
const filePath = str(row.filePath ?? row[2]).replace(/\\/g, '/');
|
||||
if (excludedFiles.has(filePath)) continue;
|
||||
const nodeId = str(row.id ?? row[0]);
|
||||
const name = str(row.name ?? row[1]);
|
||||
nodes.push({
|
||||
id: nodeId,
|
||||
label: table as any,
|
||||
properties: {
|
||||
name,
|
||||
filePath,
|
||||
startLine: num(row.startLine ?? row[3]),
|
||||
endLine: num(row.endLine ?? row[4]),
|
||||
...(hasExported ? { isExported: !!(row.isExported ?? row[5]) } : {}),
|
||||
},
|
||||
});
|
||||
symbolEntries.push({ filePath, name, nodeId, type: table });
|
||||
}
|
||||
} catch { /* table may not exist or is empty */ }
|
||||
}
|
||||
|
||||
// Load relationships (exclude any involving changed/deleted files)
|
||||
const relationships: GraphRelationship[] = [];
|
||||
const nodeIdSet = new Set(nodes.map(n => n.id));
|
||||
try {
|
||||
const rows = await conn.query(
|
||||
`MATCH (a)-[r:${REL_TABLE_NAME}]->(b) RETURN a.id AS fromId, b.id AS toId, r.type AS type, r.confidence AS confidence, r.reason AS reason, r.step AS step`
|
||||
);
|
||||
const result = Array.isArray(rows) ? rows[0] : rows;
|
||||
for (const row of await result.getAll()) {
|
||||
const fromId = str(row.fromId ?? row[0]);
|
||||
const toId = str(row.toId ?? row[1]);
|
||||
if (!nodeIdSet.has(fromId) || !nodeIdSet.has(toId)) continue;
|
||||
const type = str(row.type ?? row[2]);
|
||||
if (type === 'MEMBER_OF' || type === 'STEP_IN_PROCESS') continue;
|
||||
relationships.push({
|
||||
id: `${fromId}_${type}_${toId}`,
|
||||
sourceId: fromId,
|
||||
targetId: toId,
|
||||
type: type as any,
|
||||
confidence: num(row.confidence ?? row[3]) || 1.0,
|
||||
reason: str(row.reason ?? row[4]),
|
||||
step: num(row.step ?? row[5]),
|
||||
});
|
||||
}
|
||||
} catch { /* relationship table may not exist */ }
|
||||
|
||||
// Load existing embeddings (nodeId + vector) for unchanged nodes
|
||||
const embeddingNodeIds = new Set<string>();
|
||||
const cachedEmbeddings: Array<{ nodeId: string; embedding: number[] }> = [];
|
||||
try {
|
||||
const rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding`);
|
||||
const result = Array.isArray(rows) ? rows[0] : rows;
|
||||
for (const row of await result.getAll()) {
|
||||
const nodeId = str(row.nodeId ?? row[0]);
|
||||
if (nodeIdSet.has(nodeId)) {
|
||||
embeddingNodeIds.add(nodeId);
|
||||
const embedding = row.embedding ?? row[1];
|
||||
if (embedding) {
|
||||
cachedEmbeddings.push({
|
||||
nodeId,
|
||||
embedding: Array.isArray(embedding) ? embedding.map(Number) : Array.from(embedding as any).map(Number),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch { /* embedding table may not exist */ }
|
||||
|
||||
return { nodes, relationships, symbolEntries, embeddingNodeIds, cachedEmbeddings };
|
||||
};
|
||||
|
||||
export const closeKuzu = async (): Promise<void> => {
|
||||
if (conn) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -72,3 +72,20 @@ export const getDeletedFiles = (fromCommit: string, toCommit: string, repoPath:
|
|||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Get files with uncommitted changes (working tree vs HEAD).
|
||||
* This catches staged + unstaged modifications that aren't in any commit yet.
|
||||
* Returns relative paths (forward-slash normalized).
|
||||
*/
|
||||
export const getUncommittedChanges = (repoPath: string): string[] => {
|
||||
try {
|
||||
const output = execSync(
|
||||
'git diff HEAD --name-only --diff-filter=ACMR',
|
||||
{ cwd: repoPath, encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 }
|
||||
).trim();
|
||||
return output ? output.split('\n').filter(Boolean).map(f => f.replace(/\\/g, '/')) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue