From 00758b102a2db528b74b8ee027a103ee1c8dcc1e Mon Sep 17 00:00:00 2001 From: Abhigyan Patwari <126312502+abhigyanpatwari@users.noreply.github.com> Date: Sat, 21 Mar 2026 03:11:42 +0530 Subject: [PATCH] feat: add markdown file indexing (headings + cross-links) (#399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat: add markdown file indexing (headings + cross-links) Ports #286 by @dp-web4 onto current main, resolving conflicts from kuzu→lbug rename. Closes #286 Co-Authored-By: Dennis Palatov Co-Authored-By: Claude Opus 4.6 (1M context) --- gitnexus/src/core/graph/types.ts | 5 +- .../src/core/ingestion/markdown-processor.ts | 157 ++++++++++++++++++ gitnexus/src/core/ingestion/pipeline.ts | 16 ++ gitnexus/src/core/lbug/csv-generator.ts | 20 ++- gitnexus/src/core/lbug/schema.ts | 15 ++ 5 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 gitnexus/src/core/ingestion/markdown-processor.ts diff --git a/gitnexus/src/core/graph/types.ts b/gitnexus/src/core/graph/types.ts index 60b358195..f3ddc0414 100644 --- a/gitnexus/src/core/graph/types.ts +++ b/gitnexus/src/core/graph/types.ts @@ -32,7 +32,8 @@ export type NodeLabel = | 'Delegate' | 'Annotation' | 'Constructor' - | 'Template'; + | 'Template' + | 'Section'; import { SupportedLanguages } from '../../config/supported-languages.js'; @@ -65,6 +66,8 @@ export type NodeProperties = { entryPointReason?: string, // Method signature (for MRO disambiguation) parameterCount?: number, + // Section-specific (markdown heading level, 1-6) + level?: number, returnType?: string, } diff --git a/gitnexus/src/core/ingestion/markdown-processor.ts b/gitnexus/src/core/ingestion/markdown-processor.ts new file mode 100644 index 000000000..d894cee43 --- /dev/null +++ b/gitnexus/src/core/ingestion/markdown-processor.ts @@ -0,0 +1,157 @@ +/** + * Markdown Processor + * + * Extracts structure from .md files using regex (no tree-sitter dependency). + * Creates Section nodes for headings with hierarchy, and IMPORTS edges for + * cross-file links. + */ + +import path from 'node:path'; +import { generateId } from '../../lib/utils.js'; +import { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types.js'; + +const HEADING_RE = /^(#{1,6})\s+(.+)$/; +const LINK_RE = /\[([^\]]*)\]\(([^)]+)\)/g; +const MD_EXTENSIONS = new Set(['.md', '.mdx']); + +interface MdFile { + path: string; + content: string; +} + +export const processMarkdown = ( + graph: KnowledgeGraph, + files: MdFile[], + allPathSet: Set, +): { sections: number; links: number } => { + let totalSections = 0; + let totalLinks = 0; + + for (const file of files) { + const ext = path.extname(file.path).toLowerCase(); + if (!MD_EXTENSIONS.has(ext)) continue; + + const fileNodeId = generateId('File', file.path); + // Skip if file node doesn't exist (shouldn't happen, structure-processor creates it) + if (!graph.getNode(fileNodeId)) continue; + + const lines = file.content.split('\n'); + + // --- Extract headings and build hierarchy --- + // First pass: collect all heading positions so we can compute endLine spans + const headings: { level: number; heading: string; lineNum: number }[] = []; + + for (let i = 0; i < lines.length; i++) { + const match = lines[i].match(HEADING_RE); + if (!match) continue; + + headings.push({ + level: match[1].length, + heading: match[2].trim(), + lineNum: i + 1, // 1-indexed + }); + } + + // Second pass: create nodes with proper endLine spans + const sectionStack: { level: number; id: string }[] = []; + + for (let h = 0; h < headings.length; h++) { + const { level, heading, lineNum } = headings[h]; + + // endLine = line before next heading at same or higher level, or EOF + let endLine = lines.length; + for (let j = h + 1; j < headings.length; j++) { + if (headings[j].level <= level) { + endLine = headings[j].lineNum - 1; + break; + } + } + + const sectionId = generateId('Section', `${file.path}:L${lineNum}:${heading}`); + + const node: GraphNode = { + id: sectionId, + label: 'Section', + properties: { + name: heading, + filePath: file.path, + startLine: lineNum, + endLine, + level, + description: `h${level}`, + }, + }; + graph.addNode(node); + totalSections++; + + // Find parent: pop stack until we find a level strictly less than current + while (sectionStack.length > 0 && sectionStack[sectionStack.length - 1].level >= level) { + sectionStack.pop(); + } + + const parentId = sectionStack.length > 0 + ? sectionStack[sectionStack.length - 1].id + : fileNodeId; + + graph.addRelationship({ + id: generateId('CONTAINS', `${parentId}->${sectionId}`), + type: 'CONTAINS', + sourceId: parentId, + targetId: sectionId, + confidence: 1.0, + reason: 'markdown-heading', + }); + + sectionStack.push({ level, id: sectionId }); + } + + // --- Extract links to other files in the repo --- + const fileDir = path.dirname(file.path); + const seenLinks = new Set(); + let linkMatch: RegExpExecArray | null; + LINK_RE.lastIndex = 0; + + while ((linkMatch = LINK_RE.exec(file.content)) !== null) { + const href = linkMatch[2]; + + // Skip external URLs, anchors, and mailto + if (href.startsWith('http://') || href.startsWith('https://') || + href.startsWith('#') || href.startsWith('mailto:')) { + continue; + } + + // Strip anchor fragments from local links + const cleanHref = href.split('#')[0]; + if (!cleanHref) continue; + + // Resolve relative to the file's directory, then normalize + const resolved = path.posix.normalize(path.posix.join(fileDir, cleanHref)); + + if (allPathSet.has(resolved)) { + const targetFileId = generateId('File', resolved); + + // Skip if target file node doesn't exist + if (!graph.getNode(targetFileId)) continue; + + // Dedup: skip if we've already linked this file pair + const linkKey = `${fileNodeId}->${targetFileId}`; + if (seenLinks.has(linkKey)) continue; + seenLinks.add(linkKey); + + const relId = generateId('IMPORTS', linkKey); + + graph.addRelationship({ + id: relId, + type: 'IMPORTS', + sourceId: fileNodeId, + targetId: targetFileId, + confidence: 0.8, + reason: 'markdown-link', + }); + totalLinks++; + } + } + } + + return { sections: totalSections, links: totalLinks }; +}; diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index dc2d4658b..535da40f4 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -1,5 +1,6 @@ import { createKnowledgeGraph } from '../graph/graph.js'; import { processStructure } from './structure-processor.js'; +import { processMarkdown } from './markdown-processor.js'; import { processParsing } from './parsing-processor.js'; import { processImports, @@ -99,6 +100,21 @@ export const runPipelineFromRepo = async ( stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: graph.nodeCount }, }); + + // ── Phase 2.5: Markdown processing (headings + cross-links) ──────── + const mdScanned = scannedFiles.filter(f => f.path.endsWith('.md') || f.path.endsWith('.mdx')); + if (mdScanned.length > 0) { + const mdContents = await readFileContents(repoPath, mdScanned.map(f => f.path)); + const mdFiles = mdScanned + .filter(f => mdContents.has(f.path)) + .map(f => ({ path: f.path, content: mdContents.get(f.path)! })); + const allPathSet = new Set(allPaths); + const mdResult = processMarkdown(graph, mdFiles, allPathSet); + if (isDev) { + console.log(` Markdown: ${mdResult.sections} sections, ${mdResult.links} cross-links from ${mdFiles.length} files`); + } + } + // ── Phase 3+4: Chunked read + parse ──────────────────────────────── // Group parseable files into byte-budget chunks so only ~20MB of source // is in memory at a time. Each chunk is: read → parse → extract → free. diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 9c99eb04b..e8f8a2fc8 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -238,6 +238,9 @@ export const streamAllCSVsToDisk = async ( const communityWriter = new BufferedCSVWriter(path.join(csvDir, 'community.csv'), 'id,label,heuristicLabel,keywords,description,enrichedBy,cohesion,symbolCount'); const processWriter = new BufferedCSVWriter(path.join(csvDir, 'process.csv'), 'id,label,heuristicLabel,processType,stepCount,communities,entryPointId,terminalId'); + // Section nodes have an extra 'level' column + const sectionWriter = new BufferedCSVWriter(path.join(csvDir, 'section.csv'), 'id,name,filePath,startLine,endLine,level,content,description'); + // Multi-language node types share the same CSV shape (no isExported column) const multiLangHeader = 'id,name,filePath,startLine,endLine,content,description'; const MULTI_LANG_TYPES = ['Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl', @@ -324,6 +327,20 @@ export const streamAllCSVsToDisk = async ( ].join(',')); break; } + case 'Section': { + const content = await extractContent(node, contentCache); + await sectionWriter.addRow([ + escapeCSVField(node.id), + escapeCSVField(node.properties.name || ''), + escapeCSVField(node.properties.filePath || ''), + escapeCSVNumber(node.properties.startLine, -1), + escapeCSVNumber(node.properties.endLine, -1), + escapeCSVNumber((node.properties as any).level, 1), + escapeCSVField(content), + escapeCSVField((node.properties as any).description || ''), + ].join(',')); + break; + } default: { // Code element nodes (Function, Class, Interface, CodeElement) const writer = codeWriterMap[node.label]; @@ -361,7 +378,7 @@ export const streamAllCSVsToDisk = async ( } // Finish all node writers - const allWriters = [fileWriter, folderWriter, functionWriter, classWriter, interfaceWriter, methodWriter, codeElemWriter, communityWriter, processWriter, ...multiLangWriters.values()]; + const allWriters = [fileWriter, folderWriter, functionWriter, classWriter, interfaceWriter, methodWriter, codeElemWriter, communityWriter, processWriter, sectionWriter, ...multiLangWriters.values()]; await Promise.all(allWriters.map(w => w.finish())); // --- Stream relationship CSV --- @@ -387,6 +404,7 @@ export const streamAllCSVsToDisk = async ( ['Interface', interfaceWriter], ['Method', methodWriter], ['CodeElement', codeElemWriter], ['Community', communityWriter], ['Process', processWriter], + ['Section' as NodeTableName, sectionWriter], ...Array.from(multiLangWriters.entries()).map(([name, w]) => [name as NodeTableName, w] as [NodeTableName, BufferedCSVWriter]), ]; for (const [name, writer] of tableMap) { diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index ef1fbad50..c08292466 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -192,6 +192,19 @@ export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation'); export const CONSTRUCTOR_SCHEMA = CODE_ELEMENT_BASE('Constructor'); export const TEMPLATE_SCHEMA = CODE_ELEMENT_BASE('Template'); export const MODULE_SCHEMA = CODE_ELEMENT_BASE('Module'); +// Markdown heading sections +export const SECTION_SCHEMA = ` +CREATE NODE TABLE Section ( + id STRING, + name STRING, + filePath STRING, + startLine INT64, + endLine INT64, + level INT64, + content STRING, + description STRING, + PRIMARY KEY (id) +)`; // ============================================================================ // RELATION TABLE SCHEMA @@ -289,6 +302,8 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM \`Template\` TO Interface, FROM \`Template\` TO \`Constructor\`, FROM \`Module\` TO \`Module\`, + FROM Section TO Section, + FROM Section TO File, FROM CodeElement TO Community, FROM Interface TO Community, FROM Interface TO Function,