scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-08-28 05:25:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

feat: merge gitnexus-mcp into gitnexus package - unified CLI+MCP

Browse source
This commit is contained in:
abhigyanpatwari 2026-02-04 01:12:41 +05:30
parent 789e7809be
commit c90576442e
154 changed files with 16143 additions and 14866 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5012
gitnexus-cli/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

54
gitnexus-cli/package.json
View file

@ -1,54 +0,0 @@
{
"name": "gitnexus",
"version": "0.1.0",
"description": "GitNexus local CLI and MCP server",
"author": "Abhigyan Patwari",
"license": "MIT",
"type": "module",
"bin": {
"gitnexus": "./dist/cli/index.js"
},
"files": [
"dist"
],
"scripts": {
"build": "tsc",
"dev": "tsx watch src/cli/index.ts"
},
"dependencies": {
"@huggingface/transformers": "^3.0.0",
"@modelcontextprotocol/sdk": "^1.0.0",
"commander": "^12.0.0",
"cors": "^2.8.5",
"express": "^4.19.2",
"glob": "^11.0.0",
"graphology": "^0.25.4",
"graphology-communities-louvain": "^2.0.1",
"kuzu": "^0.11.3",
"lru-cache": "^11.0.0",
"minisearch": "^7.2.0",
"ora": "^8.0.0",
"tree-sitter": "^0.21.0",
"tree-sitter-c": "^0.21.0",
"tree-sitter-c-sharp": "^0.21.0",
"tree-sitter-cpp": "^0.22.0",
"tree-sitter-go": "^0.21.0",
"tree-sitter-java": "^0.20.0",
"tree-sitter-javascript": "^0.21.0",
"tree-sitter-python": "^0.21.0",
"tree-sitter-rust": "^0.21.0",
"tree-sitter-typescript": "^0.21.0",
"uuid": "^13.0.0"
},
"devDependencies": {
"@types/cors": "^2.8.17",
"@types/express": "^4.17.21",
"@types/node": "^20.0.0",
"@types/uuid": "^10.0.0",
"tsx": "^4.0.0",
"typescript": "^5.4.5"
},
"engines": {
"node": ">=18.0.0"
}
}

8
gitnexus-cli/src/cli/mcp.ts
View file

@ -1,8 +0,0 @@
import { startMCPServer } from '../mcp/server.js';
export const mcpCommand = async () => {
await startMCPServer();
};

11
gitnexus-cli/src/core/embeddings/index.ts
View file

@ -1,11 +0,0 @@
/**
* Embeddings Module
*
* Re-exports for the embedding pipeline system.
*/
export * from './types.js';
export * from './embedder.js';
export * from './text-generator.js';
export * from './embedding-pipeline.js';

267
gitnexus-cli/src/core/ingestion/pipeline.ts
View file

@ -1,267 +0,0 @@
import { createKnowledgeGraph } from '../graph/graph.js';
import { processStructure } from './structure-processor.js';
import { processParsing } from './parsing-processor.js';
import { processImports, createImportMap } from './import-processor.js';
import { processCalls } from './call-processor.js';
import { processHeritage } from './heritage-processor.js';
import { processCommunities } from './community-processor.js';
import { processProcesses } from './process-processor.js';
import { createSymbolTable } from './symbol-table.js';
import { createASTCache } from './ast-cache.js';
import { PipelineProgress, PipelineResult } from '../../types/pipeline.js';
import { walkRepository } from './filesystem-walker.js';
const isDev = process.env.NODE_ENV !== 'production';
export const runPipelineFromRepo = async (
repoPath: string,
onProgress: (progress: PipelineProgress) => void
): Promise<PipelineResult> => {
const graph = createKnowledgeGraph();
const fileContents = new Map<string, string>();
const symbolTable = createSymbolTable();
const astCache = createASTCache(50);
const importMap = createImportMap();
const cleanup = () => {
astCache.clear();
symbolTable.clear();
};
try {
onProgress({
phase: 'extracting',
percent: 0,
message: 'Scanning repository...',
});
const files = await walkRepository(repoPath, (current, total, filePath) => {
const scanProgress = Math.round((current / total) * 15);
onProgress({
phase: 'extracting',
percent: scanProgress,
message: 'Scanning repository...',
detail: filePath,
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
files.forEach(f => fileContents.set(f.path, f.content));
onProgress({
phase: 'extracting',
percent: 15,
message: 'Repository scanned successfully',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
onProgress({
phase: 'structure',
percent: 15,
message: 'Analyzing project structure...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
const filePaths = files.map(f => f.path);
processStructure(graph, filePaths);
onProgress({
phase: 'structure',
percent: 30,
message: 'Project structure analyzed',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
onProgress({
phase: 'parsing',
percent: 30,
message: 'Parsing code definitions...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processParsing(graph, files, symbolTable, astCache, (current, total, filePath) => {
const parsingProgress = 30 + ((current / total) * 40);
onProgress({
phase: 'parsing',
percent: Math.round(parsingProgress),
message: 'Parsing code definitions...',
detail: filePath,
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
onProgress({
phase: 'imports',
percent: 70,
message: 'Resolving imports...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processImports(graph, files, astCache, importMap, (current, total) => {
const importProgress = 70 + ((current / total) * 12);
onProgress({
phase: 'imports',
percent: Math.round(importProgress),
message: 'Resolving imports...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
if (isDev) {
const importsCount = graph.relationships.filter(r => r.type === 'IMPORTS').length;
console.log(`📊 Pipeline: After import phase, graph has ${importsCount} IMPORTS relationships (total: ${graph.relationshipCount})`);
}
onProgress({
phase: 'calls',
percent: 82,
message: 'Tracing function calls...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processCalls(graph, files, astCache, symbolTable, importMap, (current, total) => {
const callProgress = 82 + ((current / total) * 10);
onProgress({
phase: 'calls',
percent: Math.round(callProgress),
message: 'Tracing function calls...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
onProgress({
phase: 'heritage',
percent: 92,
message: 'Extracting class inheritance...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processHeritage(graph, files, astCache, symbolTable, (current, total) => {
const heritageProgress = 88 + ((current / total) * 4);
onProgress({
phase: 'heritage',
percent: Math.round(heritageProgress),
message: 'Extracting class inheritance...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
onProgress({
phase: 'communities',
percent: 92,
message: 'Detecting code communities...',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
const communityResult = await processCommunities(graph, (message, progress) => {
const communityProgress = 92 + (progress * 0.06);
onProgress({
phase: 'communities',
percent: Math.round(communityProgress),
message,
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
});
if (isDev) {
console.log(`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`);
}
communityResult.communities.forEach(comm => {
graph.addNode({
id: comm.id,
label: 'Community' as const,
properties: {
name: comm.label,
filePath: '',
heuristicLabel: comm.heuristicLabel,
cohesion: comm.cohesion,
symbolCount: comm.symbolCount,
}
});
});
communityResult.memberships.forEach(membership => {
graph.addRelationship({
id: `${membership.nodeId}_member_of_${membership.communityId}`,
type: 'MEMBER_OF',
sourceId: membership.nodeId,
targetId: membership.communityId,
confidence: 1.0,
reason: 'leiden-algorithm',
});
});
onProgress({
phase: 'processes',
percent: 98,
message: 'Detecting execution flows...',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
const processResult = await processProcesses(
graph,
communityResult.memberships,
(message, progress) => {
const processProgress = 98 + (progress * 0.01);
onProgress({
phase: 'processes',
percent: Math.round(processProgress),
message,
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
}
);
if (isDev) {
console.log(`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`);
}
processResult.processes.forEach(proc => {
graph.addNode({
id: proc.id,
label: 'Process' as const,
properties: {
name: proc.label,
filePath: '',
heuristicLabel: proc.heuristicLabel,
processType: proc.processType,
stepCount: proc.stepCount,
communities: proc.communities,
entryPointId: proc.entryPointId,
terminalId: proc.terminalId,
}
});
});
processResult.steps.forEach(step => {
graph.addRelationship({
id: `${step.nodeId}_step_${step.step}_${step.processId}`,
type: 'STEP_IN_PROCESS',
sourceId: step.nodeId,
targetId: step.processId,
confidence: 1.0,
reason: 'trace-detection',
step: step.step,
});
});
onProgress({
phase: 'complete',
percent: 100,
message: `Graph complete! ${communityResult.stats.totalCommunities} communities, ${processResult.stats.totalProcesses} processes detected.`,
stats: {
filesProcessed: files.length,
totalFiles: files.length,
nodesCreated: graph.nodeCount
},
});
astCache.clear();
return { graph, fileContents, communityResult, processResult };
} catch (error) {
cleanup();
throw error;
}
};

243
gitnexus-cli/src/core/kuzu/kuzu-adapter.ts
View file

@ -1,243 +0,0 @@
import fs from 'fs/promises';
import path from 'path';
import kuzu from 'kuzu';
import { KnowledgeGraph } from '../graph/types.js';
import {
NODE_TABLES,
REL_TABLE_NAME,
SCHEMA_QUERIES,
EMBEDDING_TABLE_NAME,
NodeTableName,
} from './schema.js';
import { generateAllCSVs } from './csv-generator.js';
let db: kuzu.Database | null = null;
let conn: kuzu.Connection | null = null;
const normalizeCopyPath = (filePath: string): string => filePath.replace(/\\/g, '/');
export const initKuzu = async (dbPath: string) => {
if (conn) return { db, conn };
// kuzu v0.11 expects the database path to NOT exist (it will create it)
// or to be an existing valid kuzu database
// If an empty directory exists from a previous clean, remove it
try {
const stat = await fs.stat(dbPath);
if (stat.isDirectory()) {
// Check if it's an empty directory
const files = await fs.readdir(dbPath);
if (files.length === 0) {
// Empty directory - remove it so kuzu can create fresh
await fs.rmdir(dbPath);
}
}
} catch {
// Path doesn't exist, which is what kuzu v0.11 wants for a new database
}
// Ensure parent directory exists
const parentDir = path.dirname(dbPath);
await fs.mkdir(parentDir, { recursive: true });
db = new kuzu.Database(dbPath);
conn = new kuzu.Connection(db);
for (const schemaQuery of SCHEMA_QUERIES) {
try {
await conn.query(schemaQuery);
} catch {
// Schema may already exist
}
}
return { db, conn };
};
export const loadGraphToKuzu = async (
graph: KnowledgeGraph,
fileContents: Map<string, string>,
storagePath: string
) => {
if (!conn) {
throw new Error('KuzuDB not initialized. Call initKuzu first.');
}
const csvData = generateAllCSVs(graph, fileContents);
const csvDir = path.join(storagePath, 'csv');
await fs.mkdir(csvDir, { recursive: true });
const nodeFiles: Array<{ table: NodeTableName; path: string }> = [];
for (const [tableName, csv] of csvData.nodes.entries()) {
if (csv.split('\n').length <= 1) continue;
const filePath = path.join(csvDir, `${tableName.toLowerCase()}.csv`);
await fs.writeFile(filePath, csv, 'utf-8');
nodeFiles.push({ table: tableName, path: filePath });
}
const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim());
for (const { table, path: filePath } of nodeFiles) {
const copyQuery = getCopyQuery(table, normalizeCopyPath(filePath));
await conn.query(copyQuery);
}
let insertedRels = 0;
let skippedRels = 0;
for (const line of relLines) {
try {
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
if (!match) continue;
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
const confidence = parseFloat(confidenceStr) || 1.0;
const step = parseInt(stepStr) || 0;
const getNodeLabel = (nodeId: string): string => {
if (nodeId.startsWith('comm_')) return 'Community';
if (nodeId.startsWith('proc_')) return 'Process';
return nodeId.split(':')[0];
};
const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct'];
const escapeLabel = (label: string): string => {
return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label;
};
const fromLabel = escapeLabel(getNodeLabel(fromId));
const toLabel = escapeLabel(getNodeLabel(toId));
const insertQuery = `
MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}' }),
(b:${toLabel} {id: '${toId.replace(/'/g, "''")}' })
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
`;
await conn.query(insertQuery);
insertedRels++;
} catch {
skippedRels++;
}
}
// Cleanup CSVs
for (const { path: filePath } of nodeFiles) {
try {
await fs.unlink(filePath);
} catch {
// ignore
}
}
return { success: true, insertedRels, skippedRels };
};
const getCopyQuery = (table: NodeTableName, filePath: string): string => {
if (table === 'File') {
return `COPY File(id, name, filePath, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Folder') {
return `COPY Folder(id, name, filePath) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Community') {
return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Process') {
return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
}
return `COPY ${table}(id, name, filePath, startLine, endLine, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
};
export const executeQuery = async (cypher: string): Promise<any[]> => {
if (!conn) {
throw new Error('KuzuDB not initialized. Call initKuzu first.');
}
const queryResult = await conn.query(cypher);
// kuzu v0.11 uses getAll() instead of hasNext()/getNext()
// Query returns QueryResult for single queries, QueryResult[] for multi-statement
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
const rows = await result.getAll();
return rows;
};
export const executeWithReusedStatement = async (
cypher: string,
paramsList: Array<Record<string, any>>
): Promise<void> => {
if (!conn) {
throw new Error('KuzuDB not initialized. Call initKuzu first.');
}
if (paramsList.length === 0) return;
const SUB_BATCH_SIZE = 4;
for (let i = 0; i < paramsList.length; i += SUB_BATCH_SIZE) {
const subBatch = paramsList.slice(i, i + SUB_BATCH_SIZE);
const stmt = await conn.prepare(cypher);
if (!stmt.isSuccess()) {
const errMsg = await stmt.getErrorMessage();
throw new Error(`Prepare failed: ${errMsg}`);
}
try {
for (const params of subBatch) {
await conn.execute(stmt, params);
}
} catch (e) {
// Log the error and continue with next batch
console.warn('Batch execution error:', e);
}
// Note: kuzu 0.8.2 PreparedStatement doesn't require explicit close()
}
};
export const getKuzuStats = async (): Promise<{ nodes: number; edges: number }> => {
if (!conn) return { nodes: 0, edges: 0 };
let totalNodes = 0;
for (const tableName of NODE_TABLES) {
try {
const queryResult = await conn.query(`MATCH (n:${tableName}) RETURN count(n) AS cnt`);
const nodeResult = Array.isArray(queryResult) ? queryResult[0] : queryResult;
const nodeRows = await nodeResult.getAll();
if (nodeRows.length > 0) {
totalNodes += Number(nodeRows[0]?.cnt ?? nodeRows[0]?.[0] ?? 0);
}
} catch {
// ignore
}
}
let totalEdges = 0;
try {
const queryResult = await conn.query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS cnt`);
const edgeResult = Array.isArray(queryResult) ? queryResult[0] : queryResult;
const edgeRows = await edgeResult.getAll();
if (edgeRows.length > 0) {
totalEdges = Number(edgeRows[0]?.cnt ?? edgeRows[0]?.[0] ?? 0);
}
} catch {
// ignore
}
return { nodes: totalNodes, edges: totalEdges };
};
export const closeKuzu = async (): Promise<void> => {
if (conn) {
try {
await conn.close();
} catch {}
conn = null;
}
if (db) {
try {
await db.close();
} catch {}
db = null;
}
};
export const isKuzuReady = (): boolean => conn !== null && db !== null;
export const getEmbeddingTableName = (): string => EMBEDDING_TABLE_NAME;

45
gitnexus-cli/src/core/tree-sitter/parser-loader.ts
View file

@ -1,45 +0,0 @@
import Parser from 'tree-sitter';
import JavaScript from 'tree-sitter-javascript';
import TypeScript from 'tree-sitter-typescript';
import Python from 'tree-sitter-python';
import Java from 'tree-sitter-java';
import C from 'tree-sitter-c';
import CPP from 'tree-sitter-cpp';
import CSharp from 'tree-sitter-c-sharp';
import Go from 'tree-sitter-go';
import Rust from 'tree-sitter-rust';
import { SupportedLanguages } from '../../config/supported-languages.js';
let parser: Parser | null = null;
const languageMap: Record<string, any> = {
[SupportedLanguages.JavaScript]: JavaScript,
[SupportedLanguages.TypeScript]: TypeScript.typescript,
[`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx,
[SupportedLanguages.Python]: Python,
[SupportedLanguages.Java]: Java,
[SupportedLanguages.C]: C,
[SupportedLanguages.CPlusPlus]: CPP,
[SupportedLanguages.CSharp]: CSharp,
[SupportedLanguages.Go]: Go,
[SupportedLanguages.Rust]: Rust,
};
export const loadParser = async (): Promise<Parser> => {
if (parser) return parser;
parser = new Parser();
return parser;
};
export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<void> => {
if (!parser) await loadParser();
const key = language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
? `${language}:tsx`
: language;
const lang = languageMap[key];
if (!lang) {
throw new Error(`Unsupported language: ${language}`);
}
parser!.setLanguage(lang);
};

175
gitnexus-cli/src/mcp/server.ts
View file

@ -1,175 +0,0 @@
/**
* CLI MCP Server
*
* Standalone MCP server that uses local .gitnexus/ index.
*/
import path from 'path';
import fs from 'fs/promises';
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
CallToolRequestSchema,
ListToolsRequestSchema,
ListResourcesRequestSchema,
ReadResourceRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { GITNEXUS_TOOLS } from './tools.js';
import { findRepo } from '../storage/repo-manager.js';
import { initKuzu, executeQuery } from '../core/kuzu/kuzu-adapter.js';
import { loadBM25Index, isBM25Ready, searchBM25 } from '../core/search/bm25-index.js';
import { hybridSearch } from '../core/search/hybrid-search.js';
import { semanticSearch } from '../core/embeddings/embedding-pipeline.js';
import { isEmbedderReady } from '../core/embeddings/embedder.js';
const notIndexedMessage = (cwd: string) => `
Repository not indexed.
Run:
cd ${cwd}
gitnexus analyze
`;
const formatContext = (meta: { repoPath: string; indexedAt: string; lastCommit: string; stats?: any }) => {
const stats = meta.stats || {};
return [
`# GitNexus: ${meta.repoPath}`,
'',
'## Stats',
`- Files: ${stats.files ?? 0}`,
`- Nodes: ${stats.nodes ?? 0}`,
`- Edges: ${stats.edges ?? 0}`,
`- Communities: ${stats.communities ?? 0}`,
`- Processes: ${stats.processes ?? 0}`,
'',
`Indexed at: ${meta.indexedAt}`,
`Last commit: ${meta.lastCommit}`,
'',
'## Available Tools',
'- search, cypher, read, overview',
].join('\n');
};
export const startMCPServer = async () => {
const server = new Server(
{ name: 'gitnexus', version: '0.1.0' },
{ capabilities: { tools: {}, resources: {} } }
);
server.setRequestHandler(ListResourcesRequestSchema, async () => {
const repo = await findRepo(process.cwd());
if (!repo) return { resources: [] };
return {
resources: [
{
uri: 'gitnexus://context',
name: `GitNexus: ${repo.meta.repoPath}`,
description: 'Indexed repository context',
mimeType: 'text/markdown',
},
],
};
});
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
if (request.params.uri !== 'gitnexus://context') {
throw new Error(`Unknown resource: ${request.params.uri}`);
}
const repo = await findRepo(process.cwd());
if (!repo) {
return {
contents: [
{
uri: 'gitnexus://context',
mimeType: 'text/plain',
text: notIndexedMessage(process.cwd()),
},
],
};
}
return {
contents: [
{
uri: 'gitnexus://context',
mimeType: 'text/markdown',
text: formatContext(repo.meta),
},
],
};
});
server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: GITNEXUS_TOOLS.map((tool) => ({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema,
})),
}));
server.setRequestHandler(CallToolRequestSchema, async (request) => {
const repo = await findRepo(process.cwd());
if (!repo) {
return {
content: [{ type: 'text', text: notIndexedMessage(process.cwd()) }],
isError: true,
};
}
await initKuzu(repo.kuzuPath);
await loadBM25Index(repo.bm25Path);
const name = request.params.name;
const args = request.params.arguments || {};
if (name === 'search') {
const query = String(args.query || '');
const limit = Number(args.limit ?? 10);
let results: any[] = [];
if (isBM25Ready() && isEmbedderReady()) {
results = await hybridSearch(query, limit, executeQuery, semanticSearch);
} else if (isBM25Ready()) {
results = searchBM25(query, limit);
} else if (isEmbedderReady()) {
results = await semanticSearch(executeQuery, query, limit);
}
return {
content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
};
}
if (name === 'cypher') {
const query = String(args.query || '');
const result = await executeQuery(query);
return {
content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
};
}
if (name === 'read') {
const filePath = args.path;
if (!filePath) {
return {
content: [{ type: 'text', text: 'Missing path.' }],
isError: true,
};
}
const fullPath = path.join(repo.repoPath, String(filePath));
const content = await fs.readFile(fullPath, 'utf-8');
return { content: [{ type: 'text', text: content }] };
}
if (name === 'overview') {
return {
content: [{ type: 'text', text: JSON.stringify(repo.meta, null, 2) }],
};
}
return {
content: [{ type: 'text', text: `Unknown tool: ${name}` }],
isError: true,
};
});
const transport = new StdioServerTransport();
await server.connect(transport);
};

47
gitnexus-cli/src/mcp/tools.ts
View file

@ -1,47 +0,0 @@
export const GITNEXUS_TOOLS = [
{
name: 'search',
description: 'Hybrid search across the indexed repository (BM25 + semantic if available).',
inputSchema: {
type: 'object',
properties: {
query: { type: 'string', description: 'Search query' },
limit: { type: 'number', description: 'Max results', default: 10 },
},
required: ['query'],
},
},
{
name: 'cypher',
description: 'Execute a Cypher query on the knowledge graph.',
inputSchema: {
type: 'object',
properties: {
query: { type: 'string', description: 'Cypher query string' },
},
required: ['query'],
},
},
{
name: 'read',
description: 'Read a file from the repository.',
inputSchema: {
type: 'object',
properties: {
path: { type: 'string', description: 'File path relative to repo root' },
},
required: ['path'],
},
},
{
name: 'overview',
description: 'Return basic stats for the indexed repository.',
inputSchema: {
type: 'object',
properties: {},
},
},
];

24
gitnexus-cli/tsconfig.json
View file

@ -1,24 +0,0 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": [
"ES2022"
],
"module": "NodeNext",
"moduleResolution": "NodeNext",
"outDir": "dist",
"rootDir": "src",
"strict": false,
"esModuleInterop": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"forceConsistentCasingInFileNames": true,
"declaration": true,
"types": [
"node"
]
},
"include": [
"src/**/*"
]
}

7
gitnexus-test-setup/.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
# GitNexus AI Context
.gitnexus-rules.md
.cursorrules
.windsurfrules
CLAUDE.md
.github/copilot-instructions.md

0
gitnexus/TODO.md → gitnexus-web/TODO.md
View file

0
gitnexus/api/proxy.ts → gitnexus-web/api/proxy.ts
View file

0
gitnexus/docs/FRAMEWORK_SUPPORT.md → gitnexus-web/docs/FRAMEWORK_SUPPORT.md
View file

0
gitnexus/index.html → gitnexus-web/index.html
View file

10215
gitnexus-web/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

67
gitnexus-web/package.json Normal file
View file

@ -0,0 +1,67 @@
{
"name": "gitnexus",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc -b && vite build",
"preview": "vite preview"
},
"dependencies": {
"@huggingface/transformers": "^3.0.0",
"@isomorphic-git/lightning-fs": "^4.6.2",
"@langchain/anthropic": "^1.3.10",
"@langchain/core": "^1.1.15",
"@langchain/google-genai": "^2.1.10",
"@langchain/langgraph": "^1.1.0",
"@langchain/ollama": "^1.2.0",
"@langchain/openai": "^1.2.2",
"@sigma/edge-curve": "^3.1.0",
"@tailwindcss/vite": "^4.1.18",
"axios": "^1.13.2",
"buffer": "^6.0.3",
"comlink": "^4.4.2",
"d3": "^7.9.0",
"graphology": "^0.26.0",
"graphology-communities-louvain": "^2.0.2",
"graphology-layout-force": "^0.2.4",
"graphology-layout-forceatlas2": "^0.10.1",
"graphology-layout-noverlap": "^0.4.2",
"isomorphic-git": "^1.36.1",
"jszip": "^3.10.1",
"kuzu-wasm": "^0.11.1",
"langchain": "^1.2.10",
"lru-cache": "^11.2.4",
"lucide-react": "^0.562.0",
"mermaid": "^11.12.2",
"minisearch": "^7.2.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-markdown": "^10.1.0",
"react-syntax-highlighter": "^16.1.0",
"react-zoom-pan-pinch": "^3.7.0",
"remark-gfm": "^4.0.1",
"sigma": "^3.0.2",
"tailwindcss": "^4.1.18",
"uuid": "^13.0.0",
"vite-plugin-top-level-await": "^1.6.0",
"vite-plugin-wasm": "^3.5.0",
"web-tree-sitter": "^0.20.8",
"zod": "^3.25.76"
},
"devDependencies": {
"@babel/types": "^7.28.5",
"@types/jszip": "^3.4.0",
"@types/node": "^24.10.1",
"@types/react": "^18.3.5",
"@types/react-dom": "^18.3.0",
"@types/react-syntax-highlighter": "^15.5.13",
"@vercel/node": "^5.5.16",
"@vitejs/plugin-react": "^5.1.0",
"tree-sitter-wasms": "^0.1.13",
"typescript": "^5.4.5",
"vite": "^5.2.0",
"vite-plugin-static-copy": "^3.1.4"
}
}

0
gitnexus/public/wasm/c/tree-sitter-c.wasm → gitnexus-web/public/wasm/c/tree-sitter-c.wasm
View file

0
gitnexus/public/wasm/cpp/tree-sitter-cpp.wasm → gitnexus-web/public/wasm/cpp/tree-sitter-cpp.wasm
View file

0
gitnexus/public/wasm/csharp/tree-sitter-csharp.wasm → gitnexus-web/public/wasm/csharp/tree-sitter-csharp.wasm
View file

0
gitnexus/public/wasm/go/tree-sitter-go.wasm → gitnexus-web/public/wasm/go/tree-sitter-go.wasm
View file

0
gitnexus/public/wasm/java/tree-sitter-java.wasm → gitnexus-web/public/wasm/java/tree-sitter-java.wasm
View file

0
gitnexus/public/wasm/javascript/tree-sitter-javascript.wasm → gitnexus-web/public/wasm/javascript/tree-sitter-javascript.wasm
View file

0
gitnexus/public/wasm/kuzu-wasm.wasm → gitnexus-web/public/wasm/kuzu-wasm.wasm
View file

0
gitnexus/public/wasm/python/tree-sitter-python.wasm → gitnexus-web/public/wasm/python/tree-sitter-python.wasm
View file

0
gitnexus/public/wasm/rust/tree-sitter-rust.wasm → gitnexus-web/public/wasm/rust/tree-sitter-rust.wasm
View file

0
gitnexus/public/wasm/tree-sitter.wasm → gitnexus-web/public/wasm/tree-sitter.wasm
View file

0
gitnexus/public/wasm/typescript/tree-sitter-tsx.wasm → gitnexus-web/public/wasm/typescript/tree-sitter-tsx.wasm
View file

0
gitnexus/public/wasm/typescript/tree-sitter-typescript.wasm → gitnexus-web/public/wasm/typescript/tree-sitter-typescript.wasm
View file

0
gitnexus/src/App.tsx → gitnexus-web/src/App.tsx
View file

0
gitnexus/src/components/ActivityFeed.tsx → gitnexus-web/src/components/ActivityFeed.tsx
View file

0
gitnexus/src/components/CodeReferencesPanel.tsx → gitnexus-web/src/components/CodeReferencesPanel.tsx
View file

0
gitnexus/src/components/DropZone.tsx → gitnexus-web/src/components/DropZone.tsx
View file

0
gitnexus/src/components/EmbeddingStatus.tsx → gitnexus-web/src/components/EmbeddingStatus.tsx
View file

0
gitnexus/src/components/FileTreePanel.tsx → gitnexus-web/src/components/FileTreePanel.tsx
View file

0
gitnexus/src/components/GraphCanvas.tsx → gitnexus-web/src/components/GraphCanvas.tsx
View file

0
gitnexus/src/components/Header.tsx → gitnexus-web/src/components/Header.tsx
View file

0
gitnexus/src/components/IntelligentClusteringModal.tsx → gitnexus-web/src/components/IntelligentClusteringModal.tsx
View file

0
gitnexus/src/components/LoadingOverlay.tsx → gitnexus-web/src/components/LoadingOverlay.tsx
View file

0
gitnexus/src/components/MCPToggle.tsx → gitnexus-web/src/components/MCPToggle.tsx
View file

0
gitnexus/src/components/MarkdownRenderer.tsx → gitnexus-web/src/components/MarkdownRenderer.tsx
View file

0
gitnexus/src/components/MermaidDiagram.tsx → gitnexus-web/src/components/MermaidDiagram.tsx
View file

0
gitnexus/src/components/ProcessFlowModal.tsx → gitnexus-web/src/components/ProcessFlowModal.tsx
View file

0
gitnexus/src/components/ProcessesPanel.tsx → gitnexus-web/src/components/ProcessesPanel.tsx
View file

0
gitnexus/src/components/QueryFAB.tsx → gitnexus-web/src/components/QueryFAB.tsx
View file

0
gitnexus/src/components/RightPanel.tsx → gitnexus-web/src/components/RightPanel.tsx
View file

0
gitnexus/src/components/SettingsPanel.tsx → gitnexus-web/src/components/SettingsPanel.tsx
View file

0
gitnexus/src/components/StatusBar.tsx → gitnexus-web/src/components/StatusBar.tsx
View file

0
gitnexus/src/components/ToolCallCard.tsx → gitnexus-web/src/components/ToolCallCard.tsx
View file

0
gitnexus/src/components/WebGPUFallbackDialog.tsx → gitnexus-web/src/components/WebGPUFallbackDialog.tsx
View file

0
gitnexus-cli/src/config/ignore-service.ts → gitnexus-web/src/config/ignore-service.ts
View file

0
gitnexus-cli/src/config/supported-languages.ts → gitnexus-web/src/config/supported-languages.ts
View file

157
gitnexus-cli/src/core/embeddings/embedder.ts → gitnexus-web/src/core/embeddings/embedder.ts
View file

@ -8,23 +8,59 @@
*/
import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types';
// Module-level state for singleton pattern
let embedderInstance: FeatureExtractionPipeline | null = null;
let isInitializing = false;
let initPromise: Promise<FeatureExtractionPipeline> | null = null;
let currentDevice: 'webgpu' | 'cuda' | 'cpu' | 'wasm' | null = null;
let currentDevice: 'webgpu' | 'wasm' | null = null;
/**
* Progress callback type for model loading
*/
export type ModelProgressCallback = (progress: ModelProgress) => void;
/**
* Custom error thrown when WebGPU is not available
* Allows UI to prompt user for fallback choice
*/
export class WebGPUNotAvailableError extends Error {
constructor(originalError?: Error) {
super('WebGPU not available in this browser');
this.name = 'WebGPUNotAvailableError';
this.cause = originalError;
}
}
/**
* Check if WebGPU is available in this browser
* Quick check without loading the model
*/
export const checkWebGPUAvailability = async (): Promise<boolean> => {
try {
// Cast to any to avoid WebGPU types not being available in all TS configs
const nav = navigator as any;
if (!nav.gpu) {
return false;
}
const adapter = await nav.gpu.requestAdapter();
if (!adapter) {
return false;
}
// Try to get a device - this is where it usually fails
const device = await adapter.requestDevice();
device.destroy(); // Clean up
return true;
} catch {
return false;
}
};
/**
* Get the current device being used for inference
*/
export const getCurrentDevice = (): 'webgpu' | 'cuda' | 'cpu' | 'wasm' | null => currentDevice;
export const getCurrentDevice = (): 'webgpu' | 'wasm' | null => currentDevice;
/**
* Initialize the embedding model
@ -32,13 +68,14 @@ export const getCurrentDevice = (): 'webgpu' | 'cuda' | 'cpu' | 'wasm' | null =>
*
* @param onProgress - Optional callback for model download progress
* @param config - Optional configuration override
* @param forceDevice - Force a specific device
* @param forceDevice - Force a specific device (bypasses WebGPU check)
* @returns Promise resolving to the embedder pipeline
* @throws WebGPUNotAvailableError if WebGPU is requested but unavailable
*/
export const initEmbedder = async (
onProgress?: ModelProgressCallback,
config: Partial<EmbeddingConfig> = {},
forceDevice?: 'webgpu' | 'cuda' | 'cpu' | 'wasm'
forceDevice?: 'webgpu' | 'wasm'
): Promise<FeatureExtractionPipeline> => {
// Return existing instance if available
if (embedderInstance) {
@ -53,19 +90,14 @@ export const initEmbedder = async (
isInitializing = true;
const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
// On Windows, use webgpu for GPU acceleration (via DirectX12/DirectML)
// CUDA is only available on Linux with onnxruntime-node
const isWindows = process.platform === 'win32';
const gpuDevice = isWindows ? 'webgpu' : 'cuda';
let requestedDevice = forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device);
const requestedDevice = forceDevice || finalConfig.device;
initPromise = (async () => {
try {
// Configure transformers.js environment
env.allowLocalModels = false;
const isDev = process.env.NODE_ENV !== 'production';
if (isDev) {
if (import.meta.env.DEV) {
console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);
}
@ -80,59 +112,86 @@ export const initEmbedder = async (
onProgress(progress);
} : undefined;
// Try GPU first if auto, fall back to CPU
// Windows: webgpu (DirectX12/DirectML), Linux: cuda
const devicesToTry: Array<'webgpu' | 'cuda' | 'cpu' | 'wasm'> =
(requestedDevice === 'webgpu' || requestedDevice === 'cuda')
? [requestedDevice, 'cpu']
: [requestedDevice as 'cpu' | 'wasm'];
for (const device of devicesToTry) {
try {
if (isDev && device === 'webgpu') {
console.log('🔧 Trying WebGPU (DirectX12) backend...');
} else if (isDev && device === 'cuda') {
console.log('🔧 Trying CUDA GPU backend...');
} else if (isDev && device === 'cpu') {
console.log('🔧 Using CPU backend...');
} else if (isDev && device === 'wasm') {
console.log('🔧 Using WASM backend (slower)...');
// If WebGPU is requested (default), check availability first
if (requestedDevice === 'webgpu') {
if (import.meta.env.DEV) {
console.log('🔧 Checking WebGPU availability...');
}
const webgpuAvailable = await checkWebGPUAvailability();
if (!webgpuAvailable) {
if (import.meta.env.DEV) {
console.warn('⚠️ WebGPU not available');
}
isInitializing = false;
initPromise = null;
throw new WebGPUNotAvailableError();
}
// Try WebGPU
try {
if (import.meta.env.DEV) {
console.log('🔧 Initializing WebGPU backend...');
}
// Type assertion needed due to complex union types in transformers.js
embedderInstance = await (pipeline as any)(
'feature-extraction',
finalConfig.modelId,
{
device: device,
device: 'webgpu',
dtype: 'fp32',
progress_callback: progressCallback,
}
);
currentDevice = device;
if (isDev) {
const label = device === 'webgpu' ? 'GPU (WebGPU/DirectX12)'
: device === 'cuda' ? 'GPU (CUDA)'
: device.toUpperCase();
console.log(`✅ Using ${label} backend`);
console.log('✅ Embedding model loaded successfully');
currentDevice = 'webgpu';
if (import.meta.env.DEV) {
console.log('✅ Using WebGPU backend');
}
return embedderInstance!;
} catch (deviceError) {
if (isDev && (device === 'cuda' || device === 'webgpu')) {
const gpuType = device === 'webgpu' ? 'WebGPU' : 'CUDA';
console.log(`⚠️ ${gpuType} not available, falling back to CPU...`);
} catch (err) {
if (import.meta.env.DEV) {
console.warn('⚠️ WebGPU initialization failed:', err);
}
// Continue to next device in list
if (device === devicesToTry[devicesToTry.length - 1]) {
throw deviceError; // Last device failed, propagate error
isInitializing = false;
initPromise = null;
embedderInstance = null;
throw new WebGPUNotAvailableError(err as Error);
}
} else {
// WASM mode requested (user chose fallback)
if (import.meta.env.DEV) {
console.log('🔧 Initializing WASM backend (this will be slower)...');
}
// Type assertion needed due to complex union types in transformers.js
embedderInstance = await (pipeline as any)(
'feature-extraction',
finalConfig.modelId,
{
device: 'wasm', // WASM-based CPU execution
dtype: 'fp32',
progress_callback: progressCallback,
}
);
currentDevice = 'wasm';
if (import.meta.env.DEV) {
console.log('✅ Using WASM backend');
}
}
throw new Error('No suitable device found for embedding model');
if (import.meta.env.DEV) {
console.log('✅ Embedding model loaded successfully');
}
return embedderInstance!;
} catch (error) {
// Re-throw WebGPUNotAvailableError as-is
if (error instanceof WebGPUNotAvailableError) {
throw error;
}
isInitializing = false;
initPromise = null;
embedderInstance = null;

22
gitnexus-cli/src/core/embeddings/embedding-pipeline.ts → gitnexus-web/src/core/embeddings/embedding-pipeline.ts
View file

@ -9,8 +9,8 @@
* 5. Create vector index for semantic search
*/
import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady } from './embedder.js';
import { generateBatchEmbeddingTexts, generateEmbeddingText } from './text-generator.js';
import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady } from './embedder';
import { generateBatchEmbeddingTexts, generateEmbeddingText } from './text-generator';
import {
type EmbeddingProgress,
type EmbeddingConfig,
@ -19,9 +19,7 @@ import {
type ModelProgress,
DEFAULT_EMBEDDING_CONFIG,
EMBEDDABLE_LABELS,
} from './types.js';
const isDev = process.env.NODE_ENV !== 'production';
} from './types';
/**
* Progress callback type
@ -73,7 +71,7 @@ const queryEmbeddableNodes = async (
}
} catch (error) {
// Table might not exist or be empty, continue
if (isDev) {
if (import.meta.env.DEV) {
console.warn(`Query for ${label} nodes failed:`, error);
}
}
@ -115,7 +113,7 @@ const createVectorIndex = async (
await executeQuery(cypher);
} catch (error) {
// Index might already exist
if (isDev) {
if (import.meta.env.DEV) {
console.warn('Vector index creation warning:', error);
}
}
@ -161,7 +159,7 @@ export const runEmbeddingPipeline = async (
modelDownloadPercent: 100,
});
if (isDev) {
if (import.meta.env.DEV) {
console.log('🔍 Querying embeddable nodes...');
}
@ -169,7 +167,7 @@ export const runEmbeddingPipeline = async (
const nodes = await queryEmbeddableNodes(executeQuery);
const totalNodes = nodes.length;
if (isDev) {
if (import.meta.env.DEV) {
console.log(`📊 Found ${totalNodes} embeddable nodes`);
}
@ -238,7 +236,7 @@ export const runEmbeddingPipeline = async (
totalNodes,
});
if (isDev) {
if (import.meta.env.DEV) {
console.log('📇 Creating vector index...');
}
@ -252,13 +250,13 @@ export const runEmbeddingPipeline = async (
totalNodes,
});
if (isDev) {
if (import.meta.env.DEV) {
console.log('✅ Embedding pipeline complete!');
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
if (isDev) {
if (import.meta.env.DEV) {
console.error('❌ Embedding pipeline error:', error);
}

11
gitnexus-web/src/core/embeddings/index.ts Normal file
View file

@ -0,0 +1,11 @@
/**
* Embeddings Module
*
* Re-exports for the embedding pipeline system.
*/
export * from './types';
export * from './embedder';
export * from './text-generator';
export * from './embedding-pipeline';

4
gitnexus-cli/src/core/embeddings/text-generator.ts → gitnexus-web/src/core/embeddings/text-generator.ts
View file

@ -5,8 +5,8 @@
* Combines node metadata with code snippets for semantic matching.
*/
import type { EmbeddableNode, EmbeddingConfig } from './types.js';
import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
import type { EmbeddableNode, EmbeddingConfig } from './types';
import { DEFAULT_EMBEDDING_CONFIG } from './types';
/**
* Extract the filename from a file path

6
gitnexus-cli/src/core/embeddings/types.ts → gitnexus-web/src/core/embeddings/types.ts
View file

@ -59,8 +59,8 @@ export interface EmbeddingConfig {
batchSize: number;
/** Embedding vector dimensions */
dimensions: number;
/** Device to use for inference: 'auto' tries GPU first, falls back to CPU */
device: 'auto' | 'webgpu' | 'cuda' | 'cpu' | 'wasm';
/** Device to use for inference: 'webgpu' for GPU acceleration, 'wasm' for WASM-based CPU */
device: 'webgpu' | 'wasm';
/** Maximum characters of code snippet to include */
maxSnippetLength: number;
}
@ -74,7 +74,7 @@ export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = {
modelId: 'Snowflake/snowflake-arctic-embed-xs',
batchSize: 16,
dimensions: 384,
device: 'auto',
device: 'webgpu', // WebGPU preferred, WASM fallback available if user chooses
maxSnippetLength: 500,
};

2
gitnexus-cli/src/core/graph/graph.ts → gitnexus-web/src/core/graph/graph.ts
View file

@ -1,4 +1,4 @@
import { GraphNode, GraphRelationship, KnowledgeGraph } from './types.js'
import { GraphNode, GraphRelationship, KnowledgeGraph } from './types'
export const createKnowledgeGraph = (): KnowledgeGraph => {
const nodeMap = new Map<string, GraphNode>();

0
gitnexus-cli/src/core/graph/types.ts → gitnexus-web/src/core/graph/types.ts
View file

7
gitnexus-cli/src/core/ingestion/ast-cache.ts → gitnexus-web/src/core/ingestion/ast-cache.ts
View file

@ -1,5 +1,5 @@
import { LRUCache } from 'lru-cache';
import Parser from 'tree-sitter';
import Parser from 'web-tree-sitter';
// Define the interface for the Cache
export interface ASTCache {
@ -16,9 +16,8 @@ export const createASTCache = (maxSize: number = 50): ASTCache => {
max: maxSize,
dispose: (tree) => {
try {
// NOTE: web-tree-sitter has tree.delete(); native tree-sitter trees are GC-managed.
// Keep this try/catch so we don't crash on either runtime.
(tree as any).delete?.();
// CRITICAL: Free the WASM memory when the tree leaves the cache
tree.delete();
} catch (e) {
console.warn('Failed to delete tree from WASM memory', e);
}

32
gitnexus-cli/src/core/ingestion/call-processor.ts → gitnexus-web/src/core/ingestion/call-processor.ts
View file

@ -1,12 +1,11 @@
import { KnowledgeGraph } from '../graph/types.js';
import { ASTCache } from './ast-cache.js';
import { SymbolTable } from './symbol-table.js';
import { ImportMap } from './import-processor.js';
import Parser from 'tree-sitter';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
import { generateId } from '../../lib/utils.js';
import { getLanguageFromFilename } from './utils.js';
import { KnowledgeGraph } from '../graph/types';
import { ASTCache } from './ast-cache';
import { SymbolTable } from './symbol-table';
import { ImportMap } from './import-processor';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader';
import { LANGUAGE_QUERIES } from './tree-sitter-queries';
import { generateId } from '../../lib/utils';
import { getLanguageFromFilename } from './utils';
/**
* Node types that represent function/method definitions across languages.
@ -157,25 +156,18 @@ export const processCalls = async (
if (!tree) {
// Cache Miss: Re-parse
// Use larger bufferSize for files > 32KB
try {
tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
} catch (parseError) {
// Skip files that can't be parsed
continue;
}
tree = parser.parse(file.content);
wasReparsed = true;
}
let query;
let matches;
try {
const language = parser.getLanguage();
query = new Parser.Query(language, queryStr);
query = parser.getLanguage().query(queryStr);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Query error for ${file.path}:`, queryError);
if (wasReparsed) (tree as any).delete?.();
if (wasReparsed) tree.delete();
continue;
}
@ -226,7 +218,7 @@ export const processCalls = async (
// Cleanup if re-parsed
if (wasReparsed) {
(tree as any).delete?.();
tree.delete();
}
}
};

2
gitnexus-cli/src/core/ingestion/cluster-enricher.ts → gitnexus-web/src/core/ingestion/cluster-enricher.ts
View file

@ -5,7 +5,7 @@
* Generates semantic names, keywords, and descriptions using an LLM.
*/
import { CommunityNode } from './community-processor.js';
import { CommunityNode } from './community-processor';
// ============================================================================
// TYPES

16
gitnexus-cli/src/core/ingestion/community-processor.ts → gitnexus-web/src/core/ingestion/community-processor.ts
View file

@ -8,11 +8,9 @@
* helping agents navigate the codebase by functional area rather than file structure.
*/
// NOTE: graphology + louvain typings are a bit inconsistent across versions.
// Keep these as `any` to avoid blocking the CLI build.
import Graph from 'graphology';
import louvain from 'graphology-communities-louvain';
import { KnowledgeGraph, NodeLabel } from '../graph/types.js';
import { KnowledgeGraph, NodeLabel } from '../graph/types';
// ============================================================================
// TYPES
@ -96,7 +94,7 @@ export const processCommunities = async (
onProgress?.(`Running Leiden algorithm on ${graph.order} nodes...`, 30);
// Step 2: Run Leiden (via Louvain implementation with refinement)
const details = (louvain as any).detailed(graph, {
const details = louvain.detailed(graph, {
resolution: 1.0, // Default resolution, can be tuned
randomWalk: true,
});
@ -143,9 +141,9 @@ export const processCommunities = async (
* Build a graphology graph containing only symbol nodes and CALLS edges
* This is what the Leiden algorithm will cluster
*/
const buildGraphologyGraph = (knowledgeGraph: KnowledgeGraph): any => {
const buildGraphologyGraph = (knowledgeGraph: KnowledgeGraph): Graph => {
// Use undirected graph for Leiden - it looks at edge density, not direction
const graph = new (Graph as any)({ type: 'undirected', allowSelfLoops: false });
const graph = new Graph({ type: 'undirected', allowSelfLoops: false });
// Symbol types that should be clustered
const symbolTypes = new Set<NodeLabel>(['Function', 'Class', 'Method', 'Interface']);
@ -191,7 +189,7 @@ const buildGraphologyGraph = (knowledgeGraph: KnowledgeGraph): any => {
const createCommunityNodes = (
communities: Record<string, number>,
communityCount: number,
graph: any,
graph: Graph,
knowledgeGraph: KnowledgeGraph
): CommunityNode[] => {
// Group node IDs by community
@ -246,7 +244,7 @@ const createCommunityNodes = (
const generateHeuristicLabel = (
memberIds: string[],
nodePathMap: Map<string, string>,
graph: any,
graph: Graph,
commNum: number
): string => {
// Collect folder names from file paths
@ -327,7 +325,7 @@ const findCommonPrefix = (strings: string[]): string => {
* Calculate cohesion score (0-1) based on internal edge density
* Higher cohesion = more internal connections relative to size
*/
const calculateCohesion = (memberIds: string[], graph: any): number => {
const calculateCohesion = (memberIds: string[], graph: Graph): number => {
if (memberIds.length <= 1) return 1.0;
const memberSet = new Set(memberIds);

2
gitnexus-cli/src/core/ingestion/entry-point-scoring.ts → gitnexus-web/src/core/ingestion/entry-point-scoring.ts
View file

@ -10,7 +10,7 @@
* This module is language-agnostic - language-specific patterns are defined per language.
*/
import { detectFrameworkFromPath } from './framework-detection.js';
import { detectFrameworkFromPath } from './framework-detection';
// ============================================================================
// NAME PATTERNS - All 9 supported languages

0
gitnexus-cli/src/core/ingestion/framework-detection.ts → gitnexus-web/src/core/ingestion/framework-detection.ts
View file

30
gitnexus-cli/src/core/ingestion/heritage-processor.ts → gitnexus-web/src/core/ingestion/heritage-processor.ts
View file

@ -6,14 +6,13 @@
* - IMPLEMENTS: Class implements an Interface (TS only)
*/
import { KnowledgeGraph } from '../graph/types.js';
import { ASTCache } from './ast-cache.js';
import { SymbolTable } from './symbol-table.js';
import Parser from 'tree-sitter';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
import { generateId } from '../../lib/utils.js';
import { getLanguageFromFilename } from './utils.js';
import { KnowledgeGraph } from '../graph/types';
import { ASTCache } from './ast-cache';
import { SymbolTable } from './symbol-table';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader';
import { LANGUAGE_QUERIES } from './tree-sitter-queries';
import { generateId } from '../../lib/utils';
import { getLanguageFromFilename } from './utils';
export const processHeritage = async (
graph: KnowledgeGraph,
@ -43,25 +42,18 @@ export const processHeritage = async (
let wasReparsed = false;
if (!tree) {
// Use larger bufferSize for files > 32KB
try {
tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
} catch (parseError) {
// Skip files that can't be parsed
continue;
}
tree = parser.parse(file.content);
wasReparsed = true;
}
let query;
let matches;
try {
const language = parser.getLanguage();
query = new Parser.Query(language, queryStr);
query = parser.getLanguage().query(queryStr);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Heritage query error for ${file.path}:`, queryError);
if (wasReparsed) (tree as any).delete?.();
if (wasReparsed) tree.delete();
continue;
}
@ -156,7 +148,7 @@ export const processHeritage = async (
// Cleanup
if (wasReparsed) {
(tree as any).delete?.();
tree.delete();
}
}
};

34
gitnexus-cli/src/core/ingestion/import-processor.ts → gitnexus-web/src/core/ingestion/import-processor.ts
View file

@ -1,12 +1,9 @@
import { KnowledgeGraph } from '../graph/types.js';
import { ASTCache } from './ast-cache.js';
import Parser from 'tree-sitter';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
import { generateId } from '../../lib/utils.js';
import { getLanguageFromFilename } from './utils.js';
const isDev = process.env.NODE_ENV !== 'production';
import { KnowledgeGraph } from '../graph/types';
import { ASTCache } from './ast-cache';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader';
import { LANGUAGE_QUERIES } from './tree-sitter-queries';
import { generateId } from '../../lib/utils';
import { getLanguageFromFilename } from './utils';
// Type: Map<FilePath, Set<ResolvedFilePath>>
// Stores all files that a given file imports from
@ -144,21 +141,14 @@ export const processImports = async (
if (!tree) {
// Cache Miss: Re-parse (slower, but necessary if evicted)
// Use larger bufferSize for files > 32KB
try {
tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
} catch (parseError) {
// Skip files that can't be parsed
continue;
}
tree = parser.parse(file.content);
wasReparsed = true;
}
let query;
let matches;
try {
const language = parser.getLanguage();
query = new Parser.Query(language, queryStr);
query = parser.getLanguage().query(queryStr);
matches = query.matches(tree.rootNode);
// Removed verbose Java import logging
@ -173,7 +163,7 @@ export const processImports = async (
console.log('AST has errors:', tree.rootNode?.hasError);
console.groupEnd();
if (wasReparsed) (tree as any).delete?.();
if (wasReparsed) tree.delete();
continue;
}
@ -184,7 +174,7 @@ export const processImports = async (
if (captureMap['import']) {
const sourceNode = captureMap['import.source'];
if (!sourceNode) {
if (isDev) {
if (import.meta.env.DEV) {
console.log(`⚠️ Import captured but no source node in ${file.path}`);
}
return;
@ -234,11 +224,11 @@ export const processImports = async (
// If re-parsed just for this, delete the tree to save memory
if (wasReparsed) {
(tree as any).delete?.();
tree.delete();
}
}
if (isDev) {
if (import.meta.env.DEV) {
console.log(`📊 Import processing complete: ${totalImportsResolved}/${totalImportsFound} imports resolved to graph edges`);
}
};

28
gitnexus-cli/src/core/ingestion/parsing-processor.ts → gitnexus-web/src/core/ingestion/parsing-processor.ts
View file

@ -1,11 +1,10 @@
import { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types.js';
import Parser from 'tree-sitter';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
import { generateId } from '../../lib/utils.js';
import { SymbolTable } from './symbol-table.js';
import { ASTCache } from './ast-cache.js';
import { getLanguageFromFilename } from './utils.js';
import { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types';
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader';
import { LANGUAGE_QUERIES } from './tree-sitter-queries';
import { generateId } from '../../lib/utils';
import { SymbolTable } from './symbol-table';
import { ASTCache } from './ast-cache';
import { getLanguageFromFilename } from './utils';
export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
@ -135,15 +134,7 @@ export const processParsing = async (
await loadLanguage(language, file.path);
// 3. Parse the text content into an AST
// Use larger bufferSize for files > 32KB (default limit)
let tree;
try {
tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 });
} catch (parseError) {
// Skip files that can't be parsed (binary, encoding issues, etc.)
console.warn(`Skipping unparseable file: ${file.path}`);
continue;
}
const tree = parser.parse(file.content);
// Store in cache immediately (this might evict an old one)
astCache.set(file.path, tree);
@ -159,8 +150,7 @@ export const processParsing = async (
let query;
let matches;
try {
const language = parser.getLanguage();
query = new Parser.Query(language, queryString);
query = parser.getLanguage().query(queryString);
matches = query.matches(tree.rootNode);
} catch (queryError) {
console.warn(`Query error for ${file.path}:`, queryError);

304
gitnexus-web/src/core/ingestion/pipeline.ts Normal file
View file

@ -0,0 +1,304 @@
import { createKnowledgeGraph } from '../graph/graph';
import { extractZip, FileEntry } from '../../services/zip';
import { processStructure } from './structure-processor';
import { processParsing } from './parsing-processor';
import { processImports, createImportMap } from './import-processor';
import { processCalls } from './call-processor';
import { processHeritage } from './heritage-processor';
import { processCommunities, CommunityDetectionResult } from './community-processor';
import { processProcesses, ProcessDetectionResult } from './process-processor';
import { createSymbolTable } from './symbol-table';
import { createASTCache } from './ast-cache';
import { PipelineProgress, PipelineResult } from '../../types/pipeline';
/**
* Run the ingestion pipeline from a ZIP file
*/
export const runIngestionPipeline = async ( file: File, onProgress: (progress: PipelineProgress) => void): Promise<PipelineResult> => {
// Phase 1: Extracting (0-15%)
onProgress({
phase: 'extracting',
percent: 0,
message: 'Extracting ZIP file...',
});
// Fake progress for extraction (JSZip doesn't expose progress)
const fakeExtractionProgress = setInterval(() => {
onProgress({
phase: 'extracting',
percent: Math.min(14, Math.random() * 10 + 5),
message: 'Extracting ZIP file...',
});
}, 200);
const files = await extractZip(file);
clearInterval(fakeExtractionProgress);
// Continue with common pipeline
return runPipelineFromFiles(files, onProgress);
};
/**
* Run the ingestion pipeline from pre-extracted files (e.g., from git clone)
*/
export const runPipelineFromFiles = async (
files: FileEntry[],
onProgress: (progress: PipelineProgress) => void
): Promise<PipelineResult> => {
const graph = createKnowledgeGraph();
const fileContents = new Map<string, string>();
const symbolTable = createSymbolTable();
const astCache = createASTCache(50); // Keep last 50 files hot
const importMap = createImportMap();
// Cleanup function for error handling
const cleanup = () => {
astCache.clear();
symbolTable.clear();
};
try {
// Store file contents for code panel
files.forEach(f => fileContents.set(f.path, f.content));
onProgress({
phase: 'extracting',
percent: 15,
message: 'ZIP extracted successfully',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: 0 },
});
// Phase 2: Structure (15-30%)
onProgress({
phase: 'structure',
percent: 15,
message: 'Analyzing project structure...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: 0 },
});
const filePaths = files.map(f => f.path);
processStructure(graph, filePaths);
onProgress({
phase: 'structure',
percent: 30,
message: 'Project structure analyzed',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
// Phase 3: Parsing (30-70%)
onProgress({
phase: 'parsing',
percent: 30,
message: 'Parsing code definitions...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processParsing(graph, files, symbolTable, astCache, (current, total, filePath) => {
const parsingProgress = 30 + ((current / total) * 40);
onProgress({
phase: 'parsing',
percent: Math.round(parsingProgress),
message: 'Parsing code definitions...',
detail: filePath,
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
// Phase 4: Imports (70-82%)
onProgress({
phase: 'imports',
percent: 70,
message: 'Resolving imports...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processImports(graph, files, astCache, importMap, (current, total) => {
const importProgress = 70 + ((current / total) * 12);
onProgress({
phase: 'imports',
percent: Math.round(importProgress),
message: 'Resolving imports...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
// Debug: Count IMPORTS relationships
if (import.meta.env.DEV) {
const importsCount = graph.relationships.filter(r => r.type === 'IMPORTS').length;
console.log(`📊 Pipeline: After import phase, graph has ${importsCount} IMPORTS relationships (total: ${graph.relationshipCount})`);
if (importsCount > 0) {
const sample = graph.relationships.filter(r => r.type === 'IMPORTS').slice(0, 3);
sample.forEach(r => console.log(` Sample IMPORTS: ${r.sourceId} → ${r.targetId}`));
}
}
// Phase 5: Calls (82-98%)
onProgress({
phase: 'calls',
percent: 82,
message: 'Tracing function calls...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processCalls(graph, files, astCache, symbolTable, importMap, (current, total) => {
const callProgress = 82 + ((current / total) * 10);
onProgress({
phase: 'calls',
percent: Math.round(callProgress),
message: 'Tracing function calls...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
// Phase 6: Heritage - Class inheritance (92-98%)
onProgress({
phase: 'heritage',
percent: 92,
message: 'Extracting class inheritance...',
stats: { filesProcessed: 0, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
await processHeritage(graph, files, astCache, symbolTable, (current, total) => {
const heritageProgress = 88 + ((current / total) * 4);
onProgress({
phase: 'heritage',
percent: Math.round(heritageProgress),
message: 'Extracting class inheritance...',
stats: { filesProcessed: current, totalFiles: total, nodesCreated: graph.nodeCount },
});
});
// Phase 7: Community Detection (92-98%)
onProgress({
phase: 'communities',
percent: 92,
message: 'Detecting code communities...',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
const communityResult = await processCommunities(graph, (message, progress) => {
const communityProgress = 92 + (progress * 0.06);
onProgress({
phase: 'communities',
percent: Math.round(communityProgress),
message,
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
});
// Log community detection results
if (import.meta.env.DEV) {
console.log(`🏘️ Community detection: ${communityResult.stats.totalCommunities} communities found (modularity: ${communityResult.stats.modularity.toFixed(3)})`);
}
// Add community nodes to the graph
communityResult.communities.forEach(comm => {
graph.addNode({
id: comm.id,
label: 'Community' as const,
properties: {
name: comm.label,
filePath: '',
heuristicLabel: comm.heuristicLabel,
cohesion: comm.cohesion,
symbolCount: comm.symbolCount,
}
});
});
// Add MEMBER_OF relationships
communityResult.memberships.forEach(membership => {
graph.addRelationship({
id: `${membership.nodeId}_member_of_${membership.communityId}`,
type: 'MEMBER_OF',
sourceId: membership.nodeId,
targetId: membership.communityId,
confidence: 1.0,
reason: 'leiden-algorithm',
});
});
// Phase 8: Process Detection (98-99%)
onProgress({
phase: 'processes',
percent: 98,
message: 'Detecting execution flows...',
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
const processResult = await processProcesses(
graph,
communityResult.memberships,
(message, progress) => {
const processProgress = 98 + (progress * 0.01);
onProgress({
phase: 'processes',
percent: Math.round(processProgress),
message,
stats: { filesProcessed: files.length, totalFiles: files.length, nodesCreated: graph.nodeCount },
});
}
);
// Log process detection results
if (import.meta.env.DEV) {
console.log(`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`);
}
// Add Process nodes to the graph
processResult.processes.forEach(proc => {
graph.addNode({
id: proc.id,
label: 'Process' as const,
properties: {
name: proc.label,
filePath: '',
heuristicLabel: proc.heuristicLabel,
processType: proc.processType,
stepCount: proc.stepCount,
communities: proc.communities,
entryPointId: proc.entryPointId,
terminalId: proc.terminalId,
}
});
});
// Add STEP_IN_PROCESS relationships
processResult.steps.forEach(step => {
graph.addRelationship({
id: `${step.nodeId}_step_${step.step}_${step.processId}`,
type: 'STEP_IN_PROCESS',
sourceId: step.nodeId,
targetId: step.processId,
confidence: 1.0,
reason: 'trace-detection',
step: step.step,
});
});
// Phase 9: Complete (100%)
onProgress({
phase: 'complete',
percent: 100,
message: `Graph complete! ${communityResult.stats.totalCommunities} communities, ${processResult.stats.totalProcesses} processes detected.`,
stats: {
filesProcessed: files.length,
totalFiles: files.length,
nodesCreated: graph.nodeCount
},
});
// Cleanup WASM memory before returning
astCache.clear();
return { graph, fileContents, communityResult, processResult };
} catch (error) {
cleanup();
throw error;
}
};

10
gitnexus-cli/src/core/ingestion/process-processor.ts → gitnexus-web/src/core/ingestion/process-processor.ts
View file

@ -10,11 +10,9 @@
* Processes help agents understand how features work through the codebase.
*/
import { KnowledgeGraph, GraphNode, GraphRelationship, NodeLabel } from '../graph/types.js';
import { CommunityMembership } from './community-processor.js';
import { calculateEntryPointScore, isTestFile } from './entry-point-scoring.js';
const isDev = process.env.NODE_ENV !== 'production';
import { KnowledgeGraph, GraphNode, GraphRelationship, NodeLabel } from '../graph/types';
import { CommunityMembership } from './community-processor';
import { calculateEntryPointScore, isTestFile } from './entry-point-scoring';
// ============================================================================
// CONFIGURATION
@ -291,7 +289,7 @@ const findEntryPoints = (
const sorted = entryPointCandidates.sort((a, b) => b.score - a.score);
// DEBUG: Log top candidates with new scoring details
if (sorted.length > 0 && isDev) {
if (sorted.length > 0 && typeof import.meta !== 'undefined' && import.meta.env?.DEV) {
console.log(`[Process] Top 10 entry point candidates (new scoring):`);
sorted.slice(0, 10).forEach((c, i) => {
const node = graph.nodes.find(n => n.id === c.id);

4
gitnexus-cli/src/core/ingestion/structure-processor.ts → gitnexus-web/src/core/ingestion/structure-processor.ts
View file

@ -1,5 +1,5 @@
import { generateId } from "../../lib/utils.js";
import { KnowledgeGraph, GraphNode, GraphRelationship } from "../graph/types.js";
import { generateId } from "@/lib/utils";
import { KnowledgeGraph, GraphNode, GraphRelationship } from "../graph/types";
export const processStructure = ( graph: KnowledgeGraph, paths: string[])=>{
paths.forEach( path => {

0
gitnexus-cli/src/core/ingestion/symbol-table.ts → gitnexus-web/src/core/ingestion/symbol-table.ts
View file

2
gitnexus-cli/src/core/ingestion/tree-sitter-queries.ts → gitnexus-web/src/core/ingestion/tree-sitter-queries.ts
View file

@ -1,4 +1,4 @@
import { SupportedLanguages } from '../../config/supported-languages.js';
import { SupportedLanguages } from '../../config/supported-languages';
/*
* Tree-sitter queries for extracting code definitions.

2
gitnexus-cli/src/core/ingestion/utils.ts → gitnexus-web/src/core/ingestion/utils.ts
View file

@ -1,4 +1,4 @@
import { SupportedLanguages } from '../../config/supported-languages.js';
import { SupportedLanguages } from '../../config/supported-languages';
/**
* Map file extension to SupportedLanguage enum

9
gitnexus-cli/src/core/kuzu/csv-generator.ts → gitnexus-web/src/core/kuzu/csv-generator.ts
View file

@ -10,8 +10,8 @@
* - All fields are consistently quoted for safety with code content
*/
import { KnowledgeGraph, GraphNode, NodeLabel } from '../graph/types.js';
import { NODE_TABLES, NodeTableName } from './schema.js';
import { KnowledgeGraph, GraphNode, NodeLabel } from '../graph/types';
import { NODE_TABLES, NodeTableName } from './schema';
// ============================================================================
// CSV ESCAPE UTILITIES
@ -133,14 +133,9 @@ export interface CSVData {
const generateFileCSV = (nodes: GraphNode[], fileContents: Map<string, string>): string => {
const headers = ['id', 'name', 'filePath', 'content'];
const rows: string[] = [headers.join(',')];
const seenIds = new Set<string>();
for (const node of nodes) {
if (node.label !== 'File') continue;
// Skip duplicates
if (seenIds.has(node.id)) continue;
seenIds.add(node.id);
const content = extractContent(node, fileContents);
rows.push([
escapeCSVField(node.id),

520
gitnexus-web/src/core/kuzu/kuzu-adapter.ts Normal file
View file

@ -0,0 +1,520 @@
/**
* KuzuDB Adapter
*
* Manages the KuzuDB WASM instance for client-side graph database operations.
* Uses the "Snapshot / Bulk Load" pattern with COPY FROM for performance.
*
* Multi-table schema: separate tables for File, Function, Class, etc.
*/
import { KnowledgeGraph } from '../graph/types';
import {
NODE_TABLES,
REL_TABLE_NAME,
SCHEMA_QUERIES,
EMBEDDING_TABLE_NAME,
NodeTableName,
} from './schema';
import { generateAllCSVs } from './csv-generator';
// Holds the reference to the dynamically loaded module
let kuzu: any = null;
let db: any = null;
let conn: any = null;
/**
* Initialize KuzuDB WASM module and create in-memory database
*/
export const initKuzu = async () => {
if (conn) return { db, conn, kuzu };
try {
if (import.meta.env.DEV) console.log('🚀 Initializing KuzuDB...');
// 1. Dynamic Import (Fixes the "not a function" bundler issue)
const kuzuModule = await import('kuzu-wasm');
// 2. Handle Vite/Webpack "default" wrapping
kuzu = kuzuModule.default || kuzuModule;
// 3. Initialize WASM
await kuzu.init();
// 4. Create Database with 512MB buffer pool
const BUFFER_POOL_SIZE = 512 * 1024 * 1024; // 512MB
db = new kuzu.Database(':memory:', BUFFER_POOL_SIZE);
conn = new kuzu.Connection(db);
if (import.meta.env.DEV) console.log('✅ KuzuDB WASM Initialized');
// 5. Initialize Schema (all node tables, then rel tables, then embedding table)
for (const schemaQuery of SCHEMA_QUERIES) {
try {
await conn.query(schemaQuery);
} catch (e) {
// Schema might already exist, skip
if (import.meta.env.DEV) {
console.warn('Schema creation skipped (may already exist):', e);
}
}
}
if (import.meta.env.DEV) console.log('✅ KuzuDB Multi-Table Schema Created');
return { db, conn, kuzu };
} catch (error) {
if (import.meta.env.DEV) console.error('❌ KuzuDB Initialization Failed:', error);
throw error;
}
};
/**
* Load a KnowledgeGraph into KuzuDB using COPY FROM (bulk load)
* Uses batched CSV writes and COPY statements for optimal performance
*/
export const loadGraphToKuzu = async (
graph: KnowledgeGraph,
fileContents: Map<string, string>
) => {
const { conn, kuzu } = await initKuzu();
try {
if (import.meta.env.DEV) console.log(`KuzuDB: Generating CSVs for ${graph.nodeCount} nodes...`);
// 1. Generate all CSVs (per-table)
const csvData = generateAllCSVs(graph, fileContents);
const fs = kuzu.FS;
// 2. Write all node CSVs to virtual filesystem
const nodeFiles: Array<{ table: NodeTableName; path: string }> = [];
for (const [tableName, csv] of csvData.nodes.entries()) {
// Skip empty CSVs (only header row)
if (csv.split('\n').length <= 1) continue;
const path = `/${tableName.toLowerCase()}.csv`;
try { await fs.unlink(path); } catch {}
await fs.writeFile(path, csv);
nodeFiles.push({ table: tableName, path });
}
// 3. Parse relation CSV and prepare for INSERT (COPY FROM doesn't work with multi-pair tables)
const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim());
const relCount = relLines.length;
if (import.meta.env.DEV) {
console.log(`KuzuDB: Wrote ${nodeFiles.length} node CSVs, ${relCount} relations to insert`);
}
// 4. COPY all node tables (must complete before rels due to FK constraints)
for (const { table, path } of nodeFiles) {
const copyQuery = getCopyQuery(table, path);
await conn.query(copyQuery);
}
// 5. INSERT relations one by one (COPY doesn't work with multi-pair REL tables)
// Parse CSV format: "from","to","type",confidence,"reason"
let insertedRels = 0;
let skippedRels = 0;
const skippedRelStats = new Map<string, number>();
for (const line of relLines) {
try {
// Parse CSV - handle quoted fields and numeric confidence
// Parse CSV - handle quoted fields and numeric confidence
// Format: "from","to","type",confidence,"reason",step
// Note: step is unquoted numeric
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
if (!match) continue;
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
const confidence = parseFloat(confidenceStr) || 1.0;
const step = parseInt(stepStr) || 0;
// Extract labels from node IDs
// Community nodes have IDs like "comm_14" (no colon)
// Other nodes have IDs like "Label:path:name"
const getNodeLabel = (nodeId: string): string => {
if (nodeId.startsWith('comm_')) {
return 'Community';
}
if (nodeId.startsWith('proc_')) {
return 'Process';
}
return nodeId.split(':')[0];
};
// Reserved Cypher keywords need backtick escaping
const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct'];
const escapeLabel = (label: string): string => {
return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label;
};
const fromLabel = escapeLabel(getNodeLabel(fromId));
const toLabel = escapeLabel(getNodeLabel(toId));
// INSERT with explicit node matching (including confidence and reason)
const insertQuery = `
MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}'}),
(b:${toLabel} {id: '${toId.replace(/'/g, "''")}'})
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
`;
await conn.query(insertQuery);
insertedRels++;
} catch (err) {
// Skip failed insertions (nodes might not exist, or relation pair not allowed by schema)
skippedRels++;
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)"/);
if (match) {
const [, fromId, toId, relType] = match;
const getNodeLabel = (nodeId: string): string => {
if (nodeId.startsWith('comm_')) return 'Community';
if (nodeId.startsWith('proc_')) return 'Process';
return nodeId.split(':')[0];
};
const fromLabel = getNodeLabel(fromId);
const toLabel = getNodeLabel(toId);
const key = `${relType}:${fromLabel}->` + toLabel;
skippedRelStats.set(key, (skippedRelStats.get(key) || 0) + 1);
// Log each skipped relation
if (import.meta.env.DEV) {
console.warn(`⚠️ Skipped: ${key} | "${fromId}" → "${toId}" | ${err instanceof Error ? err.message : String(err)}`);
}
}
}
}
if (import.meta.env.DEV) {
console.log(`KuzuDB: Inserted ${insertedRels}/${relCount} relations`);
if (skippedRels > 0) {
const topSkipped = Array.from(skippedRelStats.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 10);
console.warn(`KuzuDB: Skipped ${skippedRels}/${relCount} relations (top by kind/pair):`, topSkipped);
}
}
// 6. Verify results
let totalNodes = 0;
for (const tableName of NODE_TABLES) {
try {
const countRes = await conn.query(`MATCH (n:${tableName}) RETURN count(n) AS cnt`);
const countRow = await countRes.getNext();
const count = countRow ? (countRow.cnt ?? countRow[0] ?? 0) : 0;
totalNodes += Number(count);
} catch {
// Table might be empty, skip
}
}
if (import.meta.env.DEV) console.log(`✅ KuzuDB Bulk Load Complete. Total nodes: ${totalNodes}, edges: ${insertedRels}`);
// 7. Cleanup CSV files
for (const { path } of nodeFiles) {
try { await fs.unlink(path); } catch {}
}
return { success: true, count: totalNodes };
} catch (error) {
if (import.meta.env.DEV) console.error('❌ KuzuDB Bulk Load Failed:', error);
return { success: false, count: 0 };
}
};
/**
* Get the COPY query for a node table with correct column mapping
*/
const getCopyQuery = (table: NodeTableName, path: string): string => {
// File and Folder have different columns than code elements
if (table === 'File') {
return `COPY File(id, name, filePath, content) FROM "${path}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Folder') {
return `COPY Folder(id, name, filePath) FROM "${path}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Community') {
return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${path}" (HEADER=true, PARALLEL=false)`;
}
if (table === 'Process') {
return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${path}" (HEADER=true, PARALLEL=false)`;
}
// All code element tables: Function, Class, Interface, Method, CodeElement
return `COPY ${table}(id, name, filePath, startLine, endLine, content) FROM "${path}" (HEADER=true, PARALLEL=false)`;
};
/**
* Execute a Cypher query against the database
* Returns results as named objects (not tuples) for better usability
*/
export const executeQuery = async (cypher: string): Promise<any[]> => {
if (!conn) {
await initKuzu();
}
try {
const result = await conn.query(cypher);
// Extract column names from RETURN clause
const returnMatch = cypher.match(/RETURN\s+(.+?)(?:\s+ORDER|\s+LIMIT|\s+SKIP|\s*$)/is);
let columnNames: string[] = [];
if (returnMatch) {
// Parse RETURN clause to get column names/aliases
// Handles: "a.name, b.filePath AS path, count(x) AS cnt"
const returnClause = returnMatch[1];
columnNames = returnClause.split(',').map(col => {
col = col.trim();
// Check for AS alias
const asMatch = col.match(/\s+AS\s+(\w+)\s*$/i);
if (asMatch) return asMatch[1];
// Check for property access like n.name
const propMatch = col.match(/\.(\w+)\s*$/);
if (propMatch) return propMatch[1];
// Check for function call like count(x)
const funcMatch = col.match(/^(\w+)\s*\(/);
if (funcMatch) return funcMatch[1];
// Just use as-is if simple identifier
return col.replace(/[^a-zA-Z0-9_]/g, '_');
});
}
// Collect all rows
const rows: any[] = [];
while (await result.hasNext()) {
const row = await result.getNext();
// Convert tuple to named object if we have column names and row is array
if (Array.isArray(row) && columnNames.length === row.length) {
const namedRow: Record<string, any> = {};
for (let i = 0; i < row.length; i++) {
namedRow[columnNames[i]] = row[i];
}
rows.push(namedRow);
} else {
// Already an object or column count doesn't match
rows.push(row);
}
}
return rows;
} catch (error) {
if (import.meta.env.DEV) console.error('Query execution failed:', error);
throw error;
}
};
/**
* Get database statistics
*/
export const getKuzuStats = async (): Promise<{ nodes: number; edges: number }> => {
if (!conn) {
return { nodes: 0, edges: 0 };
}
try {
// Count nodes across all tables
let totalNodes = 0;
for (const tableName of NODE_TABLES) {
try {
const nodeResult = await conn.query(`MATCH (n:${tableName}) RETURN count(n) AS cnt`);
const nodeRow = await nodeResult.getNext();
totalNodes += Number(nodeRow?.cnt ?? nodeRow?.[0] ?? 0);
} catch {
// Table might not exist or be empty
}
}
// Count edges from single relation table
let totalEdges = 0;
try {
const edgeResult = await conn.query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS cnt`);
const edgeRow = await edgeResult.getNext();
totalEdges = Number(edgeRow?.cnt ?? edgeRow?.[0] ?? 0);
} catch {
// Table might not exist or be empty
}
return { nodes: totalNodes, edges: totalEdges };
} catch (error) {
if (import.meta.env.DEV) {
console.warn('Failed to get Kuzu stats:', error);
}
return { nodes: 0, edges: 0 };
}
};
/**
* Check if KuzuDB is initialized and has data
*/
export const isKuzuReady = (): boolean => {
return conn !== null && db !== null;
};
/**
* Close the database connection (cleanup)
*/
export const closeKuzu = async (): Promise<void> => {
if (conn) {
try {
await conn.close();
} catch {}
conn = null;
}
if (db) {
try {
await db.close();
} catch {}
db = null;
}
kuzu = null;
};
/**
* Execute a prepared statement with parameters
* @param cypher - Cypher query with $param placeholders
* @param params - Object mapping param names to values
* @returns Query results
*/
export const executePrepared = async (
cypher: string,
params: Record<string, any>
): Promise<any[]> => {
if (!conn) {
await initKuzu();
}
try {
const stmt = await conn.prepare(cypher);
if (!stmt.isSuccess()) {
const errMsg = await stmt.getErrorMessage();
throw new Error(`Prepare failed: ${errMsg}`);
}
const result = await conn.execute(stmt, params);
const rows: any[] = [];
while (await result.hasNext()) {
const row = await result.getNext();
rows.push(row);
}
await stmt.close();
return rows;
} catch (error) {
if (import.meta.env.DEV) console.error('Prepared query failed:', error);
throw error;
}
};
/**
* Execute a prepared statement with multiple parameter sets in small sub-batches
*/
export const executeWithReusedStatement = async (
cypher: string,
paramsList: Array<Record<string, any>>
): Promise<void> => {
if (!conn) {
await initKuzu();
}
if (paramsList.length === 0) return;
const SUB_BATCH_SIZE = 4;
for (let i = 0; i < paramsList.length; i += SUB_BATCH_SIZE) {
const subBatch = paramsList.slice(i, i + SUB_BATCH_SIZE);
const stmt = await conn.prepare(cypher);
if (!stmt.isSuccess()) {
const errMsg = await stmt.getErrorMessage();
throw new Error(`Prepare failed: ${errMsg}`);
}
try {
for (const params of subBatch) {
await conn.execute(stmt, params);
}
} finally {
await stmt.close();
}
if (i + SUB_BATCH_SIZE < paramsList.length) {
await new Promise(r => setTimeout(r, 0));
}
}
};
/**
* Test if array parameters work with prepared statements
*/
export const testArrayParams = async (): Promise<{ success: boolean; error?: string }> => {
if (!conn) {
await initKuzu();
}
try {
const testEmbedding = new Array(384).fill(0).map((_, i) => i / 384);
// Get any node ID to test with (try File first, then others)
let testNodeId: string | null = null;
for (const tableName of NODE_TABLES) {
try {
const nodeResult = await conn.query(`MATCH (n:${tableName}) RETURN n.id AS id LIMIT 1`);
const nodeRow = await nodeResult.getNext();
if (nodeRow) {
testNodeId = nodeRow.id ?? nodeRow[0];
break;
}
} catch {}
}
if (!testNodeId) {
return { success: false, error: 'No nodes found to test with' };
}
if (import.meta.env.DEV) {
console.log('🧪 Testing array params with node:', testNodeId);
}
// First create an embedding entry
const createQuery = `CREATE (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId, embedding: $embedding})`;
const stmt = await conn.prepare(createQuery);
if (!stmt.isSuccess()) {
const errMsg = await stmt.getErrorMessage();
return { success: false, error: `Prepare failed: ${errMsg}` };
}
await conn.execute(stmt, {
nodeId: testNodeId,
embedding: testEmbedding,
});
await stmt.close();
// Verify it was stored
const verifyResult = await conn.query(
`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: '${testNodeId}'}) RETURN e.embedding AS emb`
);
const verifyRow = await verifyResult.getNext();
const storedEmb = verifyRow?.emb ?? verifyRow?.[0];
if (storedEmb && Array.isArray(storedEmb) && storedEmb.length === 384) {
if (import.meta.env.DEV) {
console.log('✅ Array params WORK! Stored embedding length:', storedEmb.length);
}
return { success: true };
} else {
return {
success: false,
error: `Embedding not stored correctly. Got: ${typeof storedEmb}, length: ${storedEmb?.length}`
};
}
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
if (import.meta.env.DEV) {
console.error('❌ Array params test failed:', errorMsg);
}
return { success: false, error: errorMsg };
}
};

0
gitnexus-cli/src/core/kuzu/schema.ts → gitnexus-web/src/core/kuzu/schema.ts
View file

0
gitnexus/src/core/llm/agent.ts → gitnexus-web/src/core/llm/agent.ts
View file

0
gitnexus/src/core/llm/context-builder.ts → gitnexus-web/src/core/llm/context-builder.ts
View file

0
gitnexus/src/core/llm/index.ts → gitnexus-web/src/core/llm/index.ts
View file

0
gitnexus/src/core/llm/settings-service.ts → gitnexus-web/src/core/llm/settings-service.ts
View file

0
gitnexus/src/core/llm/tools.ts → gitnexus-web/src/core/llm/tools.ts
View file

0
gitnexus/src/core/llm/types.ts → gitnexus-web/src/core/llm/types.ts
View file

0
gitnexus/src/core/mcp/mcp-client.ts → gitnexus-web/src/core/mcp/mcp-client.ts
View file

44
gitnexus-cli/src/core/search/bm25-index.ts → gitnexus-web/src/core/search/bm25-index.ts
View file

@ -6,7 +6,6 @@
*/
import MiniSearch from 'minisearch';
import fs from 'fs/promises';
export interface BM25Document {
id: string; // File path
@ -83,8 +82,7 @@ export const buildBM25Index = (fileContents: Map<string, string>): number => {
searchIndex.addAll(documents);
indexedDocCount = documents.length;
const isDev = process.env.NODE_ENV !== 'production';
if (isDev) {
if (import.meta.env.DEV) {
console.log(`📚 BM25 index built: ${indexedDocCount} documents`);
}
@ -147,46 +145,6 @@ export const clearBM25Index = (): void => {
indexedDocCount = 0;
};
/**
* Export the BM25 index to disk
*/
export const exportBM25Index = async (filePath: string): Promise<void> => {
if (!searchIndex) return;
const json = JSON.stringify(searchIndex.toJSON());
await fs.writeFile(filePath, json, 'utf-8');
};
/**
* Load a BM25 index from disk
*/
export const loadBM25Index = async (filePath: string): Promise<boolean> => {
try {
const json = await fs.readFile(filePath, 'utf-8');
const data = JSON.parse(json);
searchIndex = MiniSearch.loadJSON(data, {
fields: ['content', 'name'],
storeFields: ['id'],
tokenize: (text: string) => {
const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/);
const expanded: string[] = [];
for (const token of tokens) {
if (token.length === 0) continue;
const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' ');
expanded.push(...camelParts);
if (camelParts.length > 1) {
expanded.push(token);
}
}
return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t));
},
});
indexedDocCount = searchIndex.documentCount;
return true;
} catch {
return false;
}
};
/**
* Common stop words to filter out (too common to be useful)
*/

19
gitnexus-cli/src/core/search/hybrid-search.ts → gitnexus-web/src/core/search/hybrid-search.ts
View file

@ -8,8 +8,8 @@
* production search systems.
*/
import { searchBM25, isBM25Ready, type BM25SearchResult } from './bm25-index.js';
import type { SemanticSearchResult } from '../embeddings/types.js';
import { searchBM25, isBM25Ready, type BM25SearchResult } from './bm25-index';
import type { SemanticSearchResult } from '../embeddings/types';
/**
* RRF constant - standard value used in the literature
@ -144,21 +144,6 @@ export const formatHybridResults = (results: HybridSearchResult[]): string => {
return `Found ${results.length} results:\n\n${formatted.join('\n\n')}`;
};
/**
* Execute BM25 + semantic search and merge with RRF.
* The semanticSearch function is injected to keep this module environment-agnostic.
*/
export const hybridSearch = async (
query: string,
limit: number,
executeQuery: (cypher: string) => Promise<any[]>,
semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]>
): Promise<HybridSearchResult[]> => {
const bm25Results = isBM25Ready() ? searchBM25(query, limit) : [];
const semanticResults = await semanticSearch(executeQuery, query, limit);
return mergeWithRRF(bm25Results, semanticResults, limit);
};

0
gitnexus/src/core/search/index.ts → gitnexus-web/src/core/search/index.ts
View file

72
gitnexus-web/src/core/tree-sitter/parser-loader.ts Normal file
View file

@ -0,0 +1,72 @@
import Parser from 'web-tree-sitter';
import { SupportedLanguages } from '../../config/supported-languages';
let parser: Parser | null = null;
// Cache the compiled Language objects to avoid fetching/compiling twice
const languageCache = new Map<string, Parser.Language>();
export const loadParser = async (): Promise<Parser> => {
if (parser) return parser;
await Parser.init({
locateFile: (scriptName: string) => {
return `/wasm/${scriptName}`;
}
})
parser = new Parser();
return parser;
}
// Get the appropriate WASM file based on language and file extension
const getWasmPath = (language: SupportedLanguages, filePath?: string): string => {
// For TypeScript, check if it's a TSX file
if (language === SupportedLanguages.TypeScript) {
if (filePath?.endsWith('.tsx')) {
return '/wasm/typescript/tree-sitter-tsx.wasm';
}
return '/wasm/typescript/tree-sitter-typescript.wasm';
}
const languageFileMap: Record<SupportedLanguages, string> = {
[SupportedLanguages.JavaScript]: '/wasm/javascript/tree-sitter-javascript.wasm',
[SupportedLanguages.TypeScript]: '/wasm/typescript/tree-sitter-typescript.wasm',
[SupportedLanguages.Python]: '/wasm/python/tree-sitter-python.wasm',
[SupportedLanguages.Java]: '/wasm/java/tree-sitter-java.wasm',
[SupportedLanguages.C]: '/wasm/c/tree-sitter-c.wasm',
[SupportedLanguages.CPlusPlus]: '/wasm/cpp/tree-sitter-cpp.wasm',
[SupportedLanguages.CSharp]: '/wasm/csharp/tree-sitter-csharp.wasm',
[SupportedLanguages.Go]: '/wasm/go/tree-sitter-go.wasm',
[SupportedLanguages.Rust]: '/wasm/rust/tree-sitter-rust.wasm',
};
return languageFileMap[language];
};
export const loadLanguage = async (language: SupportedLanguages, filePath?: string): Promise<void> => {
if (!parser) await loadParser();
const wasmPath = getWasmPath(language, filePath);
if (languageCache.has(wasmPath)) {
parser!.setLanguage(languageCache.get(wasmPath)!);
return;
}
if (!wasmPath) {
console.error(`❌ [Parser] No WASM path configured for language: ${language}`);
throw new Error(`Unsupported language: ${language}`);
}
try {
const loadedLanguage = await Parser.Language.load(wasmPath);
languageCache.set(wasmPath, loadedLanguage);
parser!.setLanguage(loadedLanguage);
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(`❌ [Parser] Failed to load WASM grammar for ${language}`);
console.error(` WASM Path: ${wasmPath}`);
console.error(` Error: ${errorMessage}`);
throw new Error(`Failed to load grammar for ${language}: ${errorMessage}`);
}
}

0
gitnexus/src/hooks/useAppState.tsx → gitnexus-web/src/hooks/useAppState.tsx
View file

0
gitnexus/src/hooks/useSettings.ts → gitnexus-web/src/hooks/useSettings.ts
View file

0
gitnexus/src/hooks/useSigma.ts → gitnexus-web/src/hooks/useSigma.ts
View file

0
gitnexus/src/index.css → gitnexus-web/src/index.css
View file

0
gitnexus/src/lib/constants.ts → gitnexus-web/src/lib/constants.ts
View file

0
gitnexus/src/lib/graph-adapter.ts → gitnexus-web/src/lib/graph-adapter.ts
View file

0
gitnexus/src/lib/mermaid-generator.ts → gitnexus-web/src/lib/mermaid-generator.ts
View file

0
gitnexus-cli/src/lib/utils.ts → gitnexus-web/src/lib/utils.ts
View file

0
gitnexus/src/main.tsx → gitnexus-web/src/main.tsx
View file

0
gitnexus/src/repomix-output.md → gitnexus-web/src/repomix-output.md
View file

0
gitnexus/src/services/git-clone.ts → gitnexus-web/src/services/git-clone.ts
View file

0
gitnexus/src/services/zip.ts → gitnexus-web/src/services/zip.ts
View file

Some files were not shown because too many files have changed in this diff Show more