mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-10 22:43:40 +00:00
feat: git namespace isolation, markdown/pseudocode ingestion, doc-resolver, Section/CodeElement embedding
This commit is contained in:
Vreyka
parent
87b4d34d54
commit
efcc78a50e
7 changed files with 7 additions and 8085 deletions
|
|
@ -1,279 +0,0 @@
|
|||
/**
|
||||
* Markdown Processor (AST-Based)
|
||||
*
|
||||
* V3 Architecture compliant parser. Extracts markdown structures using mdast.
|
||||
* Identifies headings as documentation Sections, code blocks as Pseudocode CodeElements.
|
||||
* Maintains chronological process flow via stepCounter in CALLS edges.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import type { GraphNode, GraphRelationship } from 'gitnexus-shared';
|
||||
import { KnowledgeGraph } from '../graph/types.js';
|
||||
import { resolveGitNamespace, type GitNamespaceMap } from './git-namespace-detector.js';
|
||||
import { unified } from 'unified';
|
||||
import remarkParse from 'remark-parse';
|
||||
import { visit } from 'unist-util-visit';
|
||||
import type { Root, Heading, Code, Link } from 'mdast';
|
||||
|
||||
const MD_EXTENSIONS = new Set(['.md', '.mdx']);
|
||||
|
||||
interface MdFile {
|
||||
path: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface PendingResolution {
|
||||
source: string;
|
||||
name: string;
|
||||
step: number;
|
||||
sourceContext: string;
|
||||
}
|
||||
|
||||
export const processMarkdown = (
|
||||
graph: KnowledgeGraph,
|
||||
files: MdFile[],
|
||||
allPathSet: Set<string>,
|
||||
namespaceMap?: GitNamespaceMap,
|
||||
): { sections: number; links: number; pendingResolutions: PendingResolution[] } => {
|
||||
let totalSections = 0;
|
||||
let totalLinks = 0;
|
||||
const pendingResolutions: PendingResolution[] = [];
|
||||
|
||||
const processor = unified().use(remarkParse);
|
||||
|
||||
// Cross-file Symbol Table: Mapping funcName -> CodeElement ID
|
||||
const docSymbolTable = new Map<string, string>();
|
||||
|
||||
// Store code blocks globally for Step 3 pass
|
||||
const allDesignCodeBlocks: { id: string, calledSymbols: string[], filePath: string }[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const ext = path.extname(file.path).toLowerCase();
|
||||
if (!MD_EXTENSIONS.has(ext)) continue;
|
||||
|
||||
const fileNodeId = generateId('File', file.path);
|
||||
if (!graph.getNode(fileNodeId)) continue;
|
||||
|
||||
// Update File node metadata for documentation recognition
|
||||
const fileNode = graph.getNode(fileNodeId);
|
||||
if (fileNode) {
|
||||
fileNode.properties.nodeCategory = 'documentation';
|
||||
}
|
||||
|
||||
const ast = processor.parse(file.content) as Root;
|
||||
|
||||
// Extract Headings
|
||||
const headings: { id: string, level: number, lineNum: number, endLine: number, slug: string }[] = [];
|
||||
visit(ast, 'heading', (node: Heading) => {
|
||||
if (!node.position) return;
|
||||
const textNode = node.children.find(c => c.type === 'text');
|
||||
const text = textNode && 'value' in textNode ? textNode.value : `Heading ${node.depth}`;
|
||||
const slug = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
|
||||
headings.push({
|
||||
id: generateId('Section', `${file.path}:L${node.position.start.line}:${text}`),
|
||||
level: node.depth,
|
||||
lineNum: node.position.start.line,
|
||||
endLine: file.content.split('\n').length, // Will be refined below
|
||||
slug: slug
|
||||
});
|
||||
});
|
||||
|
||||
// Refine heading endlines based on hierarchy
|
||||
for (let h = 0; h < headings.length; h++) {
|
||||
for (let j = h + 1; j < headings.length; j++) {
|
||||
if (headings[j].level <= headings[h].level) {
|
||||
headings[h].endLine = headings[j].lineNum - 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Register Heading Nodes and CONTAINS hierarchy
|
||||
const sectionStack: { level: number; id: string }[] = [];
|
||||
for (const h of headings) {
|
||||
const sectionNode: GraphNode = {
|
||||
id: h.id,
|
||||
label: 'Section',
|
||||
properties: {
|
||||
name: h.slug,
|
||||
filePath: file.path,
|
||||
startLine: h.lineNum,
|
||||
endLine: h.endLine,
|
||||
level: h.level,
|
||||
description: `h${h.level}`,
|
||||
nodeCategory: 'documentation',
|
||||
isPseudocode: false,
|
||||
docType: 'design',
|
||||
...(namespaceMap ? { git_namespace: resolveGitNamespace(file.path, namespaceMap) } : {}),
|
||||
}
|
||||
};
|
||||
graph.addNode(sectionNode);
|
||||
totalSections++;
|
||||
|
||||
while (sectionStack.length > 0 && sectionStack[sectionStack.length - 1].level >= h.level) {
|
||||
sectionStack.pop();
|
||||
}
|
||||
const parentId = sectionStack.length > 0 ? sectionStack[sectionStack.length - 1].id : fileNodeId;
|
||||
|
||||
graph.addRelationship({
|
||||
id: generateId('CONTAINS', `${parentId}->${h.id}`),
|
||||
type: 'CONTAINS',
|
||||
sourceId: parentId,
|
||||
targetId: h.id,
|
||||
confidence: 1.0,
|
||||
reason: 'markdown-heading'
|
||||
});
|
||||
sectionStack.push({ level: h.level, id: h.id });
|
||||
}
|
||||
|
||||
// Helper: Find enclosing heading
|
||||
const findEnclosingHeading = (line: number) => {
|
||||
let closest = null;
|
||||
for (const h of headings) {
|
||||
if (line >= h.lineNum && line <= h.endLine) {
|
||||
if (!closest || h.level > closest.level) closest = h;
|
||||
}
|
||||
}
|
||||
return closest;
|
||||
};
|
||||
|
||||
// Extract Code Blocks
|
||||
visit(ast, 'code', (node: Code) => {
|
||||
if (!node.position) return;
|
||||
const startLine = node.position.start.line;
|
||||
const endLine = node.position.end.line;
|
||||
const id = generateId('CodeElement', `${file.path}:${startLine}-${endLine}`);
|
||||
|
||||
const defPattern = /(?:async\s+)?(?:function|procedure|def|method)\s+(\w+)\s*\(/g;
|
||||
const callPattern = /\b(\w+)\s*\(/g;
|
||||
|
||||
const extractAll = (text: string, regex: RegExp) => {
|
||||
const results = [];
|
||||
let match;
|
||||
while ((match = regex.exec(text)) !== null) {
|
||||
if (match[1]) results.push(match[1]);
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
||||
const definedSymbols = extractAll(node.value, defPattern);
|
||||
let calledSymbols = extractAll(node.value, callPattern);
|
||||
|
||||
const EXCLUDE_CALLS = new Set([
|
||||
"console", "log", "warn", "error", "parseInt", "parseFloat",
|
||||
"setTimeout", "setInterval", "clearTimeout", "clearInterval",
|
||||
"Array", "Object", "Map", "Set", "Promise", "JSON",
|
||||
"Math", "Date", "String", "Number", "Boolean",
|
||||
"require", "import", "export", "typeof", "instanceof",
|
||||
"if", "else", "for", "while", "switch", "case", "return", "throw"
|
||||
]);
|
||||
calledSymbols = calledSymbols.filter(s => !EXCLUDE_CALLS.has(s) && !definedSymbols.includes(s));
|
||||
|
||||
const codeNode: GraphNode = {
|
||||
id,
|
||||
label: 'CodeElement',
|
||||
properties: {
|
||||
name: definedSymbols.length > 0 ? definedSymbols[0] : 'anonymous_block',
|
||||
filePath: file.path,
|
||||
startLine,
|
||||
endLine,
|
||||
isExported: false,
|
||||
content: '',
|
||||
description: node.lang ? `lang:${node.lang}` : 'pseudocode block',
|
||||
nodeCategory: 'documentation',
|
||||
isPseudocode: true,
|
||||
rawContent: node.value,
|
||||
definedSymbols,
|
||||
calledSymbols,
|
||||
docType: 'design',
|
||||
...(namespaceMap ? { git_namespace: resolveGitNamespace(file.path, namespaceMap) } : {}),
|
||||
}
|
||||
};
|
||||
graph.addNode(codeNode);
|
||||
|
||||
// §1.2.1 CONTAINS Rule (Design -> Pseudocode)
|
||||
const parentHeading = findEnclosingHeading(startLine);
|
||||
const parentId = parentHeading ? parentHeading.id : fileNodeId;
|
||||
|
||||
graph.addRelationship({
|
||||
id: generateId('CONTAINS', `${parentId}->${id}`),
|
||||
type: 'CONTAINS',
|
||||
sourceId: parentId,
|
||||
targetId: id,
|
||||
confidence: 0.98,
|
||||
reason: 'structural-containment'
|
||||
});
|
||||
|
||||
// Build Document Symbol Table Map
|
||||
for (const funcName of definedSymbols) {
|
||||
docSymbolTable.set(funcName, id);
|
||||
}
|
||||
|
||||
allDesignCodeBlocks.push({ id, calledSymbols, filePath: file.path });
|
||||
});
|
||||
|
||||
// §1.2.2 IMPORTS Rule (Design -> Design)
|
||||
visit(ast, 'link', (node: Link) => {
|
||||
// Find what section contains this link
|
||||
if (!node.position) return;
|
||||
const enclosingHeading = findEnclosingHeading(node.position.start.line);
|
||||
const sourceId = enclosingHeading ? enclosingHeading.id : fileNodeId;
|
||||
|
||||
if (node.url.endsWith('.md') || node.url.includes('.md#')) {
|
||||
const cleanHref = node.url.split('#')[0];
|
||||
const targetAnchor = node.url.split('#')[1];
|
||||
|
||||
if (cleanHref) {
|
||||
const fileDir = path.dirname(file.path);
|
||||
const resolved = path.posix.normalize(path.posix.join(fileDir, cleanHref));
|
||||
if (allPathSet.has(resolved)) {
|
||||
const targetFileId = generateId('File', resolved);
|
||||
// Cannot resolve perfect sibling section now because other file may not be parsed yet.
|
||||
// Standard implementation points IMPORTS to target file ID as a baseline.
|
||||
graph.addRelationship({
|
||||
id: generateId('IMPORTS', `${sourceId}->${targetFileId}`),
|
||||
type: 'IMPORTS',
|
||||
sourceId,
|
||||
targetId: targetFileId,
|
||||
confidence: targetAnchor ? 0.95 : 0.85,
|
||||
reason: 'markdown-link'
|
||||
});
|
||||
totalLinks++;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// §1.2.3 CALLS Edge Rule (Pseudocode -> Pseudocode Chronology Tracking)
|
||||
for (const B of allDesignCodeBlocks) {
|
||||
let stepCounter = 1; // Required for GitNexus Native Process Tracing
|
||||
for (const callName of B.calledSymbols) {
|
||||
if (docSymbolTable.has(callName)) {
|
||||
const target = docSymbolTable.get(callName)!;
|
||||
if (target !== B.id) { // No self-loops
|
||||
graph.addRelationship({
|
||||
id: generateId('CALLS', `${B.id}->${target}`),
|
||||
type: 'CALLS',
|
||||
sourceId: B.id,
|
||||
targetId: target,
|
||||
confidence: 0.90,
|
||||
reason: 'pseudocode-call',
|
||||
step: stepCounter++
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Unresolved -> Schedule for §1.2.4 IMPLEMENTS mapping
|
||||
pendingResolutions.push({
|
||||
source: B.id,
|
||||
name: callName,
|
||||
step: stepCounter++,
|
||||
sourceContext: B.filePath
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { sections: totalSections, links: totalLinks, pendingResolutions };
|
||||
};
|
||||
|
|
@ -1,596 +0,0 @@
|
|||
/**
|
||||
* CSV Generator for LadybugDB Hybrid Schema
|
||||
*
|
||||
* Streams CSV rows directly to disk files in a single pass over graph nodes.
|
||||
* File contents are lazy-read from disk per-node to avoid holding the entire
|
||||
* repo in RAM. Rows are buffered (FLUSH_EVERY) before writing to minimize
|
||||
* per-row Promise overhead.
|
||||
*
|
||||
* RFC 4180 Compliant:
|
||||
* - Fields containing commas, double quotes, or newlines are enclosed in double quotes
|
||||
* - Double quotes within fields are escaped by doubling them ("")
|
||||
* - All fields are consistently quoted for safety with code content
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import { createWriteStream, WriteStream } from 'fs';
|
||||
import path from 'path';
|
||||
import type { GraphNode } from 'gitnexus-shared';
|
||||
import { KnowledgeGraph } from '../graph/types.js';
|
||||
import { NodeTableName } from './schema.js';
|
||||
|
||||
/** Flush buffered rows to disk every N rows */
|
||||
const FLUSH_EVERY = 500;
|
||||
|
||||
// ============================================================================
|
||||
// CSV ESCAPE UTILITIES
|
||||
// ============================================================================
|
||||
|
||||
export const sanitizeUTF8 = (str: string): string => {
|
||||
return str
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\r/g, '\n')
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
|
||||
.replace(/[\uD800-\uDFFF]/g, '')
|
||||
.replace(/[\uFFFE\uFFFF]/g, '');
|
||||
};
|
||||
|
||||
export const escapeCSVField = (value: string | number | undefined | null): string => {
|
||||
if (value === undefined || value === null) return '""';
|
||||
let str = String(value);
|
||||
str = sanitizeUTF8(str);
|
||||
return `"${str.replace(/"/g, '""')}"`;
|
||||
};
|
||||
|
||||
export const escapeCSVNumber = (
|
||||
value: number | undefined | null,
|
||||
defaultValue: number = -1,
|
||||
): string => {
|
||||
if (value === undefined || value === null) return String(defaultValue);
|
||||
return String(value);
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// CONTENT EXTRACTION (lazy — reads from disk on demand)
|
||||
// ============================================================================
|
||||
|
||||
export const isBinaryContent = (content: string): boolean => {
|
||||
if (!content || content.length === 0) return false;
|
||||
const sample = content.slice(0, 1000);
|
||||
let nonPrintable = 0;
|
||||
for (let i = 0; i < sample.length; i++) {
|
||||
const code = sample.charCodeAt(i);
|
||||
if (code < 9 || (code > 13 && code < 32) || code === 127) nonPrintable++;
|
||||
}
|
||||
return nonPrintable / sample.length > 0.1;
|
||||
};
|
||||
|
||||
/**
|
||||
* LRU content cache — avoids re-reading the same source file for every
|
||||
* symbol defined in it. Sized generously so most files stay cached during
|
||||
* the single-pass node iteration.
|
||||
*/
|
||||
class FileContentCache {
|
||||
private cache = new Map<string, string>();
|
||||
private accessOrder: string[] = [];
|
||||
private maxSize: number;
|
||||
private repoPath: string;
|
||||
|
||||
constructor(repoPath: string, maxSize: number = 3000) {
|
||||
this.repoPath = repoPath;
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
async get(relativePath: string): Promise<string> {
|
||||
if (!relativePath) return '';
|
||||
const cached = this.cache.get(relativePath);
|
||||
if (cached !== undefined) {
|
||||
// Move to end of accessOrder (LRU promotion)
|
||||
const idx = this.accessOrder.indexOf(relativePath);
|
||||
if (idx !== -1) {
|
||||
this.accessOrder.splice(idx, 1);
|
||||
this.accessOrder.push(relativePath);
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
try {
|
||||
const fullPath = path.join(this.repoPath, relativePath);
|
||||
const content = await fs.readFile(fullPath, 'utf-8');
|
||||
this.set(relativePath, content);
|
||||
return content;
|
||||
} catch {
|
||||
this.set(relativePath, '');
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
private set(key: string, value: string) {
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
const oldest = this.accessOrder.shift();
|
||||
if (oldest) this.cache.delete(oldest);
|
||||
}
|
||||
this.cache.set(key, value);
|
||||
this.accessOrder.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const extractContent = async (node: GraphNode, contentCache: FileContentCache): Promise<string> => {
|
||||
const filePath = node.properties.filePath;
|
||||
const content = await contentCache.get(filePath);
|
||||
if (!content) return '';
|
||||
if (node.label === 'Folder') return '';
|
||||
if (isBinaryContent(content)) return '[Binary file - content not stored]';
|
||||
|
||||
if (node.label === 'File') {
|
||||
const MAX_FILE_CONTENT = 10000;
|
||||
return content.length > MAX_FILE_CONTENT
|
||||
? content.slice(0, MAX_FILE_CONTENT) + '\n... [truncated]'
|
||||
: content;
|
||||
}
|
||||
|
||||
const startLine = node.properties.startLine;
|
||||
const endLine = node.properties.endLine;
|
||||
if (startLine === undefined || endLine === undefined) return '';
|
||||
|
||||
const lines = content.split('\n');
|
||||
const start = Math.max(0, startLine - 2);
|
||||
const end = Math.min(lines.length - 1, endLine + 2);
|
||||
const snippet = lines.slice(start, end + 1).join('\n');
|
||||
const MAX_SNIPPET = 5000;
|
||||
return snippet.length > MAX_SNIPPET
|
||||
? snippet.slice(0, MAX_SNIPPET) + '\n... [truncated]'
|
||||
: snippet;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// BUFFERED CSV WRITER
|
||||
// ============================================================================
|
||||
|
||||
class BufferedCSVWriter {
|
||||
private ws: WriteStream;
|
||||
private buffer: string[] = [];
|
||||
rows = 0;
|
||||
|
||||
constructor(filePath: string, header: string) {
|
||||
this.ws = createWriteStream(filePath, 'utf-8');
|
||||
// Large repos flush many times — raise listener cap to avoid MaxListenersExceededWarning
|
||||
this.ws.setMaxListeners(50);
|
||||
this.buffer.push(header);
|
||||
}
|
||||
|
||||
addRow(row: string) {
|
||||
this.buffer.push(row);
|
||||
this.rows++;
|
||||
if (this.buffer.length >= FLUSH_EVERY) {
|
||||
return this.flush();
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
flush(): Promise<void> {
|
||||
if (this.buffer.length === 0) return Promise.resolve();
|
||||
const chunk = this.buffer.join('\n') + '\n';
|
||||
this.buffer.length = 0;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.ws.once('error', reject);
|
||||
const ok = this.ws.write(chunk);
|
||||
if (ok) {
|
||||
this.ws.removeListener('error', reject);
|
||||
resolve();
|
||||
} else {
|
||||
this.ws.once('drain', () => {
|
||||
this.ws.removeListener('error', reject);
|
||||
resolve();
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async finish(): Promise<void> {
|
||||
await this.flush();
|
||||
return new Promise((resolve, reject) => {
|
||||
this.ws.end(() => resolve());
|
||||
this.ws.on('error', reject);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// STREAMING CSV GENERATION — SINGLE PASS
|
||||
// ============================================================================
|
||||
|
||||
export interface StreamedCSVResult {
|
||||
nodeFiles: Map<NodeTableName, { csvPath: string; rows: number }>;
|
||||
relCsvPath: string;
|
||||
relRows: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream all CSV data directly to disk files.
|
||||
* Iterates graph nodes exactly ONCE — routes each node to the right writer.
|
||||
* File contents are lazy-read from disk with a generous LRU cache.
|
||||
*/
|
||||
export const streamAllCSVsToDisk = async (
|
||||
graph: KnowledgeGraph,
|
||||
repoPath: string,
|
||||
csvDir: string,
|
||||
): Promise<StreamedCSVResult> => {
|
||||
// Remove stale CSVs from previous crashed runs, then recreate
|
||||
try {
|
||||
await fs.rm(csvDir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
await fs.mkdir(csvDir, { recursive: true });
|
||||
|
||||
// We open ~30 concurrent write-streams; raise process limit to suppress
|
||||
// MaxListenersExceededWarning (restored after all streams finish).
|
||||
const prevMax = process.getMaxListeners();
|
||||
process.setMaxListeners(prevMax + 40);
|
||||
|
||||
const contentCache = new FileContentCache(repoPath);
|
||||
|
||||
// Create writers for every node type up-front
|
||||
const fileWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'file.csv'),
|
||||
'id,name,filePath,content,nodeCategory,isPseudocode,rawContent,definedSymbols,calledSymbols,docType,domain,git_namespace',
|
||||
);
|
||||
const folderWriter = new BufferedCSVWriter(path.join(csvDir, 'folder.csv'), 'id,name,filePath,git_namespace');
|
||||
const codeElementHeader = 'id,name,filePath,startLine,endLine,isExported,content,description,git_namespace';
|
||||
const functionWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'function.csv'),
|
||||
codeElementHeader,
|
||||
);
|
||||
const classWriter = new BufferedCSVWriter(path.join(csvDir, 'class.csv'), codeElementHeader);
|
||||
const interfaceWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'interface.csv'),
|
||||
codeElementHeader,
|
||||
);
|
||||
const methodHeader =
|
||||
'id,name,filePath,startLine,endLine,isExported,content,description,parameterCount,returnType,git_namespace';
|
||||
const methodWriter = new BufferedCSVWriter(path.join(csvDir, 'method.csv'), methodHeader);
|
||||
const codeElemSpecificHeader =
|
||||
'id,name,filePath,startLine,endLine,isExported,content,description,nodeCategory,isPseudocode,rawContent,definedSymbols,calledSymbols,docType,domain,git_namespace';
|
||||
const codeElemWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'codeelement.csv'),
|
||||
codeElemSpecificHeader,
|
||||
);
|
||||
const communityWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'community.csv'),
|
||||
'id,label,heuristicLabel,keywords,description,enrichedBy,cohesion,symbolCount',
|
||||
);
|
||||
const processWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'process.csv'),
|
||||
'id,label,heuristicLabel,processType,stepCount,communities,entryPointId,terminalId',
|
||||
);
|
||||
|
||||
// Section nodes have an extra 'level' column
|
||||
const sectionWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'section.csv'),
|
||||
'id,name,filePath,startLine,endLine,level,content,description,nodeCategory,isPseudocode,rawContent,definedSymbols,calledSymbols,docType,domain,git_namespace',
|
||||
);
|
||||
|
||||
// Route nodes for API endpoint mapping
|
||||
const routeWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'route.csv'),
|
||||
'id,name,filePath,responseKeys,errorKeys,middleware,git_namespace',
|
||||
);
|
||||
|
||||
// Tool nodes for MCP tool definitions
|
||||
const toolWriter = new BufferedCSVWriter(
|
||||
path.join(csvDir, 'tool.csv'),
|
||||
'id,name,filePath,description,git_namespace',
|
||||
);
|
||||
|
||||
// Multi-language node types share the same CSV shape (no isExported column)
|
||||
const multiLangHeader = 'id,name,filePath,startLine,endLine,content,description,git_namespace';
|
||||
const MULTI_LANG_TYPES = [
|
||||
'Struct',
|
||||
'Enum',
|
||||
'Macro',
|
||||
'Typedef',
|
||||
'Union',
|
||||
'Namespace',
|
||||
'Trait',
|
||||
'Impl',
|
||||
'TypeAlias',
|
||||
'Const',
|
||||
'Static',
|
||||
'Property',
|
||||
'Record',
|
||||
'Delegate',
|
||||
'Annotation',
|
||||
'Constructor',
|
||||
'Template',
|
||||
'Module',
|
||||
] as const;
|
||||
const multiLangWriters = new Map<string, BufferedCSVWriter>();
|
||||
for (const t of MULTI_LANG_TYPES) {
|
||||
multiLangWriters.set(
|
||||
t,
|
||||
new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), multiLangHeader),
|
||||
);
|
||||
}
|
||||
|
||||
const codeWriterMap: Record<string, BufferedCSVWriter> = {
|
||||
Function: functionWriter,
|
||||
Class: classWriter,
|
||||
Interface: interfaceWriter,
|
||||
CodeElement: codeElemWriter,
|
||||
};
|
||||
|
||||
const seenFileIds = new Set<string>();
|
||||
|
||||
const formatArray = (arr: any[] | undefined) => {
|
||||
if (!arr) return '[]';
|
||||
return `[${arr.map((k: string) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
|
||||
};
|
||||
|
||||
// --- SINGLE PASS over all nodes ---
|
||||
for (const node of graph.iterNodes()) {
|
||||
switch (node.label) {
|
||||
case 'File': {
|
||||
if (seenFileIds.has(node.id)) break;
|
||||
seenFileIds.add(node.id);
|
||||
const content = await extractContent(node, contentCache);
|
||||
await fileWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVField(content),
|
||||
escapeCSVField((node.properties.nodeCategory as string) || ''),
|
||||
node.properties.isPseudocode ? 'true' : 'false',
|
||||
escapeCSVField((node.properties.rawContent as string) || ''),
|
||||
escapeCSVField(formatArray(node.properties.definedSymbols as string[])),
|
||||
escapeCSVField(formatArray(node.properties.calledSymbols as string[])),
|
||||
escapeCSVField((node.properties.docType as string) || ''),
|
||||
escapeCSVField((node.properties.domain as string) || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Folder':
|
||||
await folderWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
case 'Community': {
|
||||
const keywords = node.properties.keywords || [];
|
||||
const keywordsStr = `[${keywords.map((k: string) => `'${k.replace(/\\/g, '\\\\').replace(/'/g, "''").replace(/,/g, '\\,')}'`).join(',')}]`;
|
||||
await communityWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.heuristicLabel || ''),
|
||||
keywordsStr,
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
escapeCSVField(node.properties.enrichedBy || 'heuristic'),
|
||||
escapeCSVNumber(node.properties.cohesion, 0),
|
||||
escapeCSVNumber(node.properties.symbolCount, 0),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Process': {
|
||||
const communities = node.properties.communities || [];
|
||||
const communitiesStr = `[${communities.map((c: string) => `'${c.replace(/'/g, "''")}'`).join(',')}]`;
|
||||
await processWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.heuristicLabel || ''),
|
||||
escapeCSVField(node.properties.processType || ''),
|
||||
escapeCSVNumber(node.properties.stepCount, 0),
|
||||
escapeCSVField(communitiesStr),
|
||||
escapeCSVField(node.properties.entryPointId || ''),
|
||||
escapeCSVField(node.properties.terminalId || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Method': {
|
||||
const content = await extractContent(node, contentCache);
|
||||
await methodWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVNumber(node.properties.startLine, -1),
|
||||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
node.properties.isExported ? 'true' : 'false',
|
||||
escapeCSVField(content),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
escapeCSVNumber(node.properties.parameterCount, 0),
|
||||
escapeCSVField(node.properties.returnType || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Section': {
|
||||
const content = await extractContent(node, contentCache);
|
||||
await sectionWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVNumber(node.properties.startLine, -1),
|
||||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
escapeCSVNumber(node.properties.level, 1),
|
||||
escapeCSVField(content),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
escapeCSVField((node.properties.nodeCategory as string) || ''),
|
||||
node.properties.isPseudocode ? 'true' : 'false',
|
||||
escapeCSVField((node.properties.rawContent as string) || ''),
|
||||
escapeCSVField(formatArray(node.properties.definedSymbols as string[])),
|
||||
escapeCSVField(formatArray(node.properties.calledSymbols as string[])),
|
||||
escapeCSVField((node.properties.docType as string) || ''),
|
||||
escapeCSVField((node.properties.domain as string) || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Route': {
|
||||
const responseKeys = node.properties.responseKeys || [];
|
||||
// LadybugDB array literal inside a quoted CSV field: escapeCSVField wraps in "..."
|
||||
// and the array uses single-quoted elements
|
||||
const keysStr = `[${responseKeys.map((k: string) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
|
||||
const errorKeys = node.properties.errorKeys || [];
|
||||
const errorKeysStr = `[${errorKeys.map((k: string) => `'${k.replace(/'/g, "''")}'`).join(',')}]`;
|
||||
const middleware = node.properties.middleware || [];
|
||||
const middlewareStr = `[${middleware.map((m: string) => `'${m.replace(/'/g, "''")}'`).join(',')}]`;
|
||||
await routeWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVField(keysStr),
|
||||
escapeCSVField(errorKeysStr),
|
||||
escapeCSVField(middlewareStr),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'Tool':
|
||||
await toolWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
break;
|
||||
default: {
|
||||
// Code element nodes (Function, Class, Interface, CodeElement)
|
||||
const writer = codeWriterMap[node.label];
|
||||
if (writer) {
|
||||
const content = await extractContent(node, contentCache);
|
||||
const baseRow = [
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVNumber(node.properties.startLine, -1),
|
||||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
node.properties.isExported ? 'true' : 'false',
|
||||
escapeCSVField(content),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
];
|
||||
|
||||
if (node.label === 'CodeElement') {
|
||||
baseRow.push(
|
||||
escapeCSVField((node.properties.nodeCategory as string) || ''),
|
||||
node.properties.isPseudocode ? 'true' : 'false',
|
||||
escapeCSVField((node.properties.rawContent as string) || ''),
|
||||
escapeCSVField(formatArray(node.properties.definedSymbols as string[])),
|
||||
escapeCSVField(formatArray(node.properties.calledSymbols as string[])),
|
||||
escapeCSVField((node.properties.docType as string) || ''),
|
||||
escapeCSVField((node.properties.domain as string) || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
);
|
||||
} else {
|
||||
// Function, Class, Interface — git_namespace is the last column
|
||||
baseRow.push(escapeCSVField((node.properties.git_namespace as string) || ''));
|
||||
}
|
||||
|
||||
await writer.addRow(baseRow.join(','));
|
||||
} else {
|
||||
// Multi-language node types (Struct, Impl, Trait, Macro, etc.)
|
||||
const mlWriter = multiLangWriters.get(node.label);
|
||||
if (mlWriter) {
|
||||
const content = await extractContent(node, contentCache);
|
||||
await mlWriter.addRow(
|
||||
[
|
||||
escapeCSVField(node.id),
|
||||
escapeCSVField(node.properties.name || ''),
|
||||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVNumber(node.properties.startLine, -1),
|
||||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
escapeCSVField(content),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
escapeCSVField((node.properties.git_namespace as string) || ''),
|
||||
].join(','),
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finish all node writers
|
||||
const allWriters = [
|
||||
fileWriter,
|
||||
folderWriter,
|
||||
functionWriter,
|
||||
classWriter,
|
||||
interfaceWriter,
|
||||
methodWriter,
|
||||
codeElemWriter,
|
||||
communityWriter,
|
||||
processWriter,
|
||||
sectionWriter,
|
||||
routeWriter,
|
||||
toolWriter,
|
||||
...multiLangWriters.values(),
|
||||
];
|
||||
await Promise.all(allWriters.map((w) => w.finish()));
|
||||
|
||||
// --- Stream relationship CSV ---
|
||||
const relCsvPath = path.join(csvDir, 'relations.csv');
|
||||
const relWriter = new BufferedCSVWriter(relCsvPath, 'from,to,type,confidence,reason,step');
|
||||
for (const rel of graph.iterRelationships()) {
|
||||
await relWriter.addRow(
|
||||
[
|
||||
escapeCSVField(rel.sourceId),
|
||||
escapeCSVField(rel.targetId),
|
||||
escapeCSVField(rel.type),
|
||||
escapeCSVNumber(rel.confidence, 1.0),
|
||||
escapeCSVField(rel.reason),
|
||||
escapeCSVNumber((rel as any).step, 0),
|
||||
].join(','),
|
||||
);
|
||||
}
|
||||
await relWriter.finish();
|
||||
|
||||
// Build result map — only include tables that have rows
|
||||
const nodeFiles = new Map<NodeTableName, { csvPath: string; rows: number }>();
|
||||
const tableMap: [NodeTableName, BufferedCSVWriter][] = [
|
||||
['File', fileWriter],
|
||||
['Folder', folderWriter],
|
||||
['Function', functionWriter],
|
||||
['Class', classWriter],
|
||||
['Interface', interfaceWriter],
|
||||
['Method', methodWriter],
|
||||
['CodeElement', codeElemWriter],
|
||||
['Community', communityWriter],
|
||||
['Process', processWriter],
|
||||
['Section' as NodeTableName, sectionWriter],
|
||||
['Route' as NodeTableName, routeWriter],
|
||||
['Tool' as NodeTableName, toolWriter],
|
||||
...Array.from(multiLangWriters.entries()).map(
|
||||
([name, w]) => [name as NodeTableName, w] as [NodeTableName, BufferedCSVWriter],
|
||||
),
|
||||
];
|
||||
for (const [name, writer] of tableMap) {
|
||||
if (writer.rows > 0) {
|
||||
nodeFiles.set(name, {
|
||||
csvPath: path.join(csvDir, `${name.toLowerCase()}.csv`),
|
||||
rows: writer.rows,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Restore original process listener limit
|
||||
process.setMaxListeners(prevMax);
|
||||
|
||||
return { nodeFiles, relCsvPath, relRows: relWriter.rows };
|
||||
};
|
||||
|
|
@ -1,530 +0,0 @@
|
|||
/**
|
||||
* LadybugDB Schema Definitions
|
||||
*
|
||||
* Hybrid Schema:
|
||||
* - Separate node tables for each code element type (File, Function, Class, etc.)
|
||||
* - Single CodeRelation table with 'type' property for all relationships
|
||||
*
|
||||
* This allows LLMs to write natural Cypher queries like:
|
||||
* MATCH (f:Function)-[r:CodeRelation {type: 'CALLS'}]->(g:Function) RETURN f, g
|
||||
*/
|
||||
|
||||
// Import from shared package (single source of truth) — used in DDL templates below
|
||||
import { NODE_TABLES, REL_TABLE_NAME, REL_TYPES, EMBEDDING_TABLE_NAME } from 'gitnexus-shared';
|
||||
// Re-export so downstream consumers keep the same import path
|
||||
export { NODE_TABLES, REL_TABLE_NAME, REL_TYPES, EMBEDDING_TABLE_NAME };
|
||||
export type { NodeTableName, RelType } from 'gitnexus-shared';
|
||||
|
||||
// ============================================================================
|
||||
// NODE TABLE SCHEMAS
|
||||
// ============================================================================
|
||||
|
||||
export const FILE_SCHEMA = `
|
||||
CREATE NODE TABLE File (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
content STRING,
|
||||
nodeCategory STRING,
|
||||
isPseudocode BOOLEAN,
|
||||
rawContent STRING,
|
||||
definedSymbols STRING[],
|
||||
calledSymbols STRING[],
|
||||
docType STRING,
|
||||
domain STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const FOLDER_SCHEMA = `
|
||||
CREATE NODE TABLE Folder (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const FUNCTION_SCHEMA = `
|
||||
CREATE NODE TABLE Function (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
description STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const CLASS_SCHEMA = `
|
||||
CREATE NODE TABLE Class (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
description STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const INTERFACE_SCHEMA = `
|
||||
CREATE NODE TABLE Interface (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
description STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const METHOD_SCHEMA = `
|
||||
CREATE NODE TABLE Method (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
description STRING,
|
||||
parameterCount INT32,
|
||||
returnType STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const CODE_ELEMENT_SCHEMA = `
|
||||
CREATE NODE TABLE CodeElement (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
description STRING,
|
||||
nodeCategory STRING,
|
||||
isPseudocode BOOLEAN,
|
||||
rawContent STRING,
|
||||
definedSymbols STRING[],
|
||||
calledSymbols STRING[],
|
||||
docType STRING,
|
||||
domain STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// ============================================================================
|
||||
// COMMUNITY NODE TABLE (for Leiden algorithm clusters)
|
||||
// ============================================================================
|
||||
|
||||
export const COMMUNITY_SCHEMA = `
|
||||
CREATE NODE TABLE Community (
|
||||
id STRING,
|
||||
label STRING,
|
||||
heuristicLabel STRING,
|
||||
keywords STRING[],
|
||||
description STRING,
|
||||
enrichedBy STRING,
|
||||
cohesion DOUBLE,
|
||||
symbolCount INT32,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// ============================================================================
|
||||
// PROCESS NODE TABLE (for execution flow detection)
|
||||
// ============================================================================
|
||||
|
||||
export const PROCESS_SCHEMA = `
|
||||
CREATE NODE TABLE Process (
|
||||
id STRING,
|
||||
label STRING,
|
||||
heuristicLabel STRING,
|
||||
processType STRING,
|
||||
stepCount INT32,
|
||||
communities STRING[],
|
||||
entryPointId STRING,
|
||||
terminalId STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// ============================================================================
|
||||
// MULTI-LANGUAGE NODE TABLE SCHEMAS
|
||||
// ============================================================================
|
||||
|
||||
// Generic code element with startLine/endLine for C, C++, Rust, Go, Java, C#
|
||||
// description: optional metadata (e.g. Eloquent $fillable fields, relationship targets)
|
||||
const CODE_ELEMENT_BASE = (name: string) => `
|
||||
CREATE NODE TABLE \`${name}\` (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
content STRING,
|
||||
description STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const STRUCT_SCHEMA = CODE_ELEMENT_BASE('Struct');
|
||||
export const ENUM_SCHEMA = CODE_ELEMENT_BASE('Enum');
|
||||
export const MACRO_SCHEMA = CODE_ELEMENT_BASE('Macro');
|
||||
export const TYPEDEF_SCHEMA = CODE_ELEMENT_BASE('Typedef');
|
||||
export const UNION_SCHEMA = CODE_ELEMENT_BASE('Union');
|
||||
export const NAMESPACE_SCHEMA = CODE_ELEMENT_BASE('Namespace');
|
||||
export const TRAIT_SCHEMA = CODE_ELEMENT_BASE('Trait');
|
||||
export const IMPL_SCHEMA = CODE_ELEMENT_BASE('Impl');
|
||||
export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias');
|
||||
export const CONST_SCHEMA = CODE_ELEMENT_BASE('Const');
|
||||
export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static');
|
||||
export const PROPERTY_SCHEMA = CODE_ELEMENT_BASE('Property');
|
||||
export const RECORD_SCHEMA = CODE_ELEMENT_BASE('Record');
|
||||
export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate');
|
||||
export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation');
|
||||
export const CONSTRUCTOR_SCHEMA = CODE_ELEMENT_BASE('Constructor');
|
||||
export const TEMPLATE_SCHEMA = CODE_ELEMENT_BASE('Template');
|
||||
export const MODULE_SCHEMA = CODE_ELEMENT_BASE('Module');
|
||||
// API route endpoints (Next.js, Express, etc.)
|
||||
export const ROUTE_SCHEMA = `
|
||||
CREATE NODE TABLE Route (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
responseKeys STRING[],
|
||||
errorKeys STRING[],
|
||||
middleware STRING[],
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// MCP tool definitions
|
||||
export const TOOL_SCHEMA = `
|
||||
CREATE NODE TABLE Tool (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
description STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// Markdown heading sections
|
||||
export const SECTION_SCHEMA = `
|
||||
CREATE NODE TABLE Section (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
level INT64,
|
||||
content STRING,
|
||||
description STRING,
|
||||
nodeCategory STRING,
|
||||
isPseudocode BOOLEAN,
|
||||
rawContent STRING,
|
||||
definedSymbols STRING[],
|
||||
calledSymbols STRING[],
|
||||
docType STRING,
|
||||
domain STRING,
|
||||
git_namespace STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
// ============================================================================
|
||||
// RELATION TABLE SCHEMA
|
||||
// Single table with 'type' property - connects all node tables
|
||||
// ============================================================================
|
||||
|
||||
export const RELATION_SCHEMA = `
|
||||
CREATE REL TABLE ${REL_TABLE_NAME} (
|
||||
FROM File TO File,
|
||||
FROM File TO Folder,
|
||||
FROM File TO Function,
|
||||
FROM File TO Class,
|
||||
FROM File TO Interface,
|
||||
FROM File TO Method,
|
||||
FROM File TO CodeElement,
|
||||
FROM File TO \`Struct\`,
|
||||
FROM File TO \`Enum\`,
|
||||
FROM File TO \`Macro\`,
|
||||
FROM File TO \`Typedef\`,
|
||||
FROM File TO \`Union\`,
|
||||
FROM File TO \`Namespace\`,
|
||||
FROM File TO \`Trait\`,
|
||||
FROM File TO \`Impl\`,
|
||||
FROM File TO \`TypeAlias\`,
|
||||
FROM File TO \`Const\`,
|
||||
FROM File TO \`Static\`,
|
||||
FROM File TO \`Property\`,
|
||||
FROM File TO \`Record\`,
|
||||
FROM File TO \`Delegate\`,
|
||||
FROM File TO \`Annotation\`,
|
||||
FROM File TO \`Constructor\`,
|
||||
FROM File TO \`Template\`,
|
||||
FROM File TO \`Module\`,
|
||||
FROM File TO Section,
|
||||
FROM Folder TO Folder,
|
||||
FROM Folder TO File,
|
||||
FROM Function TO Function,
|
||||
FROM Function TO Method,
|
||||
FROM Function TO Class,
|
||||
FROM Function TO Community,
|
||||
FROM Function TO \`Macro\`,
|
||||
FROM Function TO \`Struct\`,
|
||||
FROM Function TO \`Template\`,
|
||||
FROM Function TO \`Enum\`,
|
||||
FROM Function TO \`Namespace\`,
|
||||
FROM Function TO \`TypeAlias\`,
|
||||
FROM Function TO \`Module\`,
|
||||
FROM Function TO \`Impl\`,
|
||||
FROM Function TO Interface,
|
||||
FROM Function TO \`Constructor\`,
|
||||
FROM Function TO \`Const\`,
|
||||
FROM Function TO \`Typedef\`,
|
||||
FROM Function TO \`Union\`,
|
||||
FROM Function TO \`Property\`,
|
||||
FROM Function TO CodeElement,
|
||||
FROM Class TO Method,
|
||||
FROM Class TO Function,
|
||||
FROM Class TO Class,
|
||||
FROM Class TO Interface,
|
||||
FROM Class TO Community,
|
||||
FROM Class TO \`Template\`,
|
||||
FROM Class TO \`TypeAlias\`,
|
||||
FROM Class TO \`Struct\`,
|
||||
FROM Class TO \`Enum\`,
|
||||
FROM Class TO \`Annotation\`,
|
||||
FROM Class TO \`Constructor\`,
|
||||
FROM Class TO \`Trait\`,
|
||||
FROM Class TO \`Macro\`,
|
||||
FROM Class TO \`Impl\`,
|
||||
FROM Class TO \`Union\`,
|
||||
FROM Class TO \`Namespace\`,
|
||||
FROM Class TO \`Typedef\`,
|
||||
FROM Class TO \`Property\`,
|
||||
FROM Method TO Function,
|
||||
FROM Method TO Method,
|
||||
FROM Method TO Class,
|
||||
FROM Method TO Community,
|
||||
FROM Method TO \`Template\`,
|
||||
FROM Method TO \`Struct\`,
|
||||
FROM Method TO \`TypeAlias\`,
|
||||
FROM Method TO \`Enum\`,
|
||||
FROM Method TO \`Macro\`,
|
||||
FROM Method TO \`Namespace\`,
|
||||
FROM Method TO \`Module\`,
|
||||
FROM Method TO \`Impl\`,
|
||||
FROM Method TO Interface,
|
||||
FROM Method TO \`Constructor\`,
|
||||
FROM Method TO \`Property\`,
|
||||
FROM Method TO CodeElement,
|
||||
FROM \`Template\` TO \`Template\`,
|
||||
FROM \`Template\` TO Function,
|
||||
FROM \`Template\` TO Method,
|
||||
FROM \`Template\` TO Class,
|
||||
FROM \`Template\` TO \`Struct\`,
|
||||
FROM \`Template\` TO \`TypeAlias\`,
|
||||
FROM \`Template\` TO \`Enum\`,
|
||||
FROM \`Template\` TO \`Macro\`,
|
||||
FROM \`Template\` TO Interface,
|
||||
FROM \`Template\` TO \`Constructor\`,
|
||||
FROM \`Module\` TO \`Module\`,
|
||||
FROM Section TO Section,
|
||||
FROM Section TO File,
|
||||
FROM File TO Route,
|
||||
FROM Function TO Route,
|
||||
FROM Method TO Route,
|
||||
FROM File TO Tool,
|
||||
FROM Function TO Tool,
|
||||
FROM Method TO Tool,
|
||||
FROM CodeElement TO Community,
|
||||
FROM Interface TO Community,
|
||||
FROM Interface TO Function,
|
||||
FROM Interface TO Method,
|
||||
FROM Interface TO Class,
|
||||
FROM Interface TO Interface,
|
||||
FROM Interface TO \`TypeAlias\`,
|
||||
FROM Interface TO \`Struct\`,
|
||||
FROM Interface TO \`Constructor\`,
|
||||
FROM Interface TO \`Property\`,
|
||||
FROM \`Struct\` TO Community,
|
||||
FROM \`Struct\` TO \`Trait\`,
|
||||
FROM \`Struct\` TO \`Struct\`,
|
||||
FROM \`Struct\` TO Class,
|
||||
FROM \`Struct\` TO \`Enum\`,
|
||||
FROM \`Struct\` TO Function,
|
||||
FROM \`Struct\` TO Method,
|
||||
FROM \`Struct\` TO Interface,
|
||||
FROM \`Struct\` TO \`Constructor\`,
|
||||
FROM \`Struct\` TO \`Property\`,
|
||||
FROM \`Enum\` TO \`Enum\`,
|
||||
FROM \`Enum\` TO Community,
|
||||
FROM \`Enum\` TO Class,
|
||||
FROM \`Enum\` TO Interface,
|
||||
FROM \`Macro\` TO Community,
|
||||
FROM \`Macro\` TO Function,
|
||||
FROM \`Macro\` TO Method,
|
||||
FROM \`Module\` TO Function,
|
||||
FROM \`Module\` TO Method,
|
||||
FROM \`Typedef\` TO Community,
|
||||
FROM \`Union\` TO Community,
|
||||
FROM \`Namespace\` TO Community,
|
||||
FROM \`Namespace\` TO \`Struct\`,
|
||||
FROM \`Trait\` TO Method,
|
||||
FROM \`Trait\` TO \`Constructor\`,
|
||||
FROM \`Trait\` TO \`Property\`,
|
||||
FROM \`Trait\` TO Community,
|
||||
FROM \`Impl\` TO Method,
|
||||
FROM \`Impl\` TO \`Constructor\`,
|
||||
FROM \`Impl\` TO \`Property\`,
|
||||
FROM \`Impl\` TO Community,
|
||||
FROM \`Impl\` TO \`Trait\`,
|
||||
FROM \`Impl\` TO \`Struct\`,
|
||||
FROM \`Impl\` TO \`Impl\`,
|
||||
FROM \`TypeAlias\` TO Community,
|
||||
FROM \`TypeAlias\` TO \`Trait\`,
|
||||
FROM \`TypeAlias\` TO Class,
|
||||
FROM \`Const\` TO Community,
|
||||
FROM \`Static\` TO Community,
|
||||
FROM \`Property\` TO Community,
|
||||
FROM \`Record\` TO Method,
|
||||
FROM \`Record\` TO \`Constructor\`,
|
||||
FROM \`Record\` TO \`Property\`,
|
||||
FROM \`Record\` TO Community,
|
||||
FROM \`Delegate\` TO Community,
|
||||
FROM \`Annotation\` TO Community,
|
||||
FROM \`Constructor\` TO Community,
|
||||
FROM \`Constructor\` TO Interface,
|
||||
FROM \`Constructor\` TO Class,
|
||||
FROM \`Constructor\` TO Method,
|
||||
FROM \`Constructor\` TO Function,
|
||||
FROM \`Constructor\` TO \`Constructor\`,
|
||||
FROM \`Constructor\` TO \`Struct\`,
|
||||
FROM \`Constructor\` TO \`Macro\`,
|
||||
FROM \`Constructor\` TO \`Template\`,
|
||||
FROM \`Constructor\` TO \`TypeAlias\`,
|
||||
FROM \`Constructor\` TO \`Enum\`,
|
||||
FROM \`Constructor\` TO \`Annotation\`,
|
||||
FROM \`Constructor\` TO \`Impl\`,
|
||||
FROM \`Constructor\` TO \`Namespace\`,
|
||||
FROM \`Constructor\` TO \`Module\`,
|
||||
FROM \`Constructor\` TO \`Property\`,
|
||||
FROM \`Constructor\` TO \`Typedef\`,
|
||||
FROM \`Template\` TO Community,
|
||||
FROM \`Module\` TO Community,
|
||||
FROM Function TO Process,
|
||||
FROM Method TO Process,
|
||||
FROM Class TO Process,
|
||||
FROM Interface TO Process,
|
||||
FROM \`Struct\` TO Process,
|
||||
FROM \`Constructor\` TO Process,
|
||||
FROM \`Module\` TO Process,
|
||||
FROM \`Macro\` TO Process,
|
||||
FROM \`Impl\` TO Process,
|
||||
FROM \`Typedef\` TO Process,
|
||||
FROM \`TypeAlias\` TO Process,
|
||||
FROM \`Enum\` TO Process,
|
||||
FROM \`Union\` TO Process,
|
||||
FROM \`Namespace\` TO Process,
|
||||
FROM \`Trait\` TO Process,
|
||||
FROM \`Const\` TO Process,
|
||||
FROM \`Static\` TO Process,
|
||||
FROM \`Property\` TO Process,
|
||||
FROM \`Record\` TO Process,
|
||||
FROM \`Delegate\` TO Process,
|
||||
FROM \`Annotation\` TO Process,
|
||||
FROM \`Template\` TO Process,
|
||||
FROM CodeElement TO Process,
|
||||
FROM Route TO Process,
|
||||
FROM Tool TO Process,
|
||||
type STRING,
|
||||
confidence DOUBLE,
|
||||
reason STRING,
|
||||
step INT32
|
||||
)`;
|
||||
|
||||
// ============================================================================
|
||||
// EMBEDDING TABLE SCHEMA
|
||||
// Separate table for vector storage to avoid copy-on-write overhead
|
||||
// ============================================================================
|
||||
|
||||
/** Embedding vector dimensions. Default 384 (snowflake-arctic-embed-xs). */
|
||||
const _rawDims = parseInt(process.env.GITNEXUS_EMBEDDING_DIMS ?? '384', 10);
|
||||
if (Number.isNaN(_rawDims) || _rawDims <= 0) {
|
||||
throw new Error(
|
||||
`GITNEXUS_EMBEDDING_DIMS must be a positive integer, got "${process.env.GITNEXUS_EMBEDDING_DIMS}"`,
|
||||
);
|
||||
}
|
||||
export const EMBEDDING_DIMS = _rawDims;
|
||||
|
||||
export const EMBEDDING_SCHEMA = `
|
||||
CREATE NODE TABLE ${EMBEDDING_TABLE_NAME} (
|
||||
nodeId STRING,
|
||||
embedding FLOAT[${EMBEDDING_DIMS}],
|
||||
PRIMARY KEY (nodeId)
|
||||
)`;
|
||||
|
||||
/**
|
||||
* Create vector index for semantic search
|
||||
* Uses HNSW (Hierarchical Navigable Small World) algorithm with cosine similarity
|
||||
*/
|
||||
export const CREATE_VECTOR_INDEX_QUERY = `
|
||||
CALL CREATE_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', 'code_embedding_idx', 'embedding', metric := 'cosine')
|
||||
`;
|
||||
|
||||
// ============================================================================
|
||||
// ALL SCHEMA QUERIES IN ORDER
|
||||
// Node tables must be created before relationship tables that reference them
|
||||
// ============================================================================
|
||||
|
||||
export const NODE_SCHEMA_QUERIES = [
|
||||
FILE_SCHEMA,
|
||||
FOLDER_SCHEMA,
|
||||
FUNCTION_SCHEMA,
|
||||
CLASS_SCHEMA,
|
||||
INTERFACE_SCHEMA,
|
||||
METHOD_SCHEMA,
|
||||
CODE_ELEMENT_SCHEMA,
|
||||
COMMUNITY_SCHEMA,
|
||||
PROCESS_SCHEMA,
|
||||
// Multi-language support
|
||||
STRUCT_SCHEMA,
|
||||
ENUM_SCHEMA,
|
||||
MACRO_SCHEMA,
|
||||
TYPEDEF_SCHEMA,
|
||||
UNION_SCHEMA,
|
||||
NAMESPACE_SCHEMA,
|
||||
TRAIT_SCHEMA,
|
||||
IMPL_SCHEMA,
|
||||
TYPE_ALIAS_SCHEMA,
|
||||
CONST_SCHEMA,
|
||||
STATIC_SCHEMA,
|
||||
PROPERTY_SCHEMA,
|
||||
RECORD_SCHEMA,
|
||||
DELEGATE_SCHEMA,
|
||||
ANNOTATION_SCHEMA,
|
||||
CONSTRUCTOR_SCHEMA,
|
||||
TEMPLATE_SCHEMA,
|
||||
MODULE_SCHEMA,
|
||||
// Markdown support
|
||||
SECTION_SCHEMA,
|
||||
// API routes
|
||||
ROUTE_SCHEMA,
|
||||
// MCP tools
|
||||
TOOL_SCHEMA,
|
||||
];
|
||||
|
||||
export const REL_SCHEMA_QUERIES = [RELATION_SCHEMA];
|
||||
|
||||
export const SCHEMA_QUERIES = [...NODE_SCHEMA_QUERIES, ...REL_SCHEMA_QUERIES, EMBEDDING_SCHEMA];
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -106,7 +106,7 @@ describe('isWriteQuery', () => {
|
|||
|
||||
describe('VALID_RELATION_TYPES', () => {
|
||||
it('contains all expected relation types', () => {
|
||||
expect(VALID_RELATION_TYPES.size).toBe(15);
|
||||
expect(VALID_RELATION_TYPES.size).toBe(17);
|
||||
for (const t of [
|
||||
'CALLS',
|
||||
'IMPORTS',
|
||||
|
|
|
|||
|
|
@ -62,6 +62,9 @@ export default defineConfig({
|
|||
'test/integration/lbug-lock-retry.test.ts',
|
||||
'test/integration/api-impact-e2e.test.ts',
|
||||
'test/integration/shape-check-regression.test.ts',
|
||||
'test/integration/class-impact-all-languages.test.ts',
|
||||
'test/integration/java-class-impact.test.ts',
|
||||
'test/integration/namespace-isolation.test.ts',
|
||||
],
|
||||
fileParallelism: false,
|
||||
sequence: { groupOrder: 1 },
|
||||
|
|
@ -87,6 +90,9 @@ export default defineConfig({
|
|||
'test/integration/lbug-lock-retry.test.ts',
|
||||
'test/integration/api-impact-e2e.test.ts',
|
||||
'test/integration/shape-check-regression.test.ts',
|
||||
'test/integration/class-impact-all-languages.test.ts',
|
||||
'test/integration/java-class-impact.test.ts',
|
||||
'test/integration/namespace-isolation.test.ts',
|
||||
],
|
||||
},
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue