diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts index 4c368c136..74e05b62a 100644 --- a/gitnexus/src/core/embeddings/chunker.ts +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -14,8 +14,7 @@ export { type Chunk, characterChunk } from './character-chunk.js'; import { characterChunk } from './character-chunk.js'; import type { Chunk } from './character-chunk.js'; -// Module-level parser cache — safe because Parser is stateless -// and language grammars are read-only +// Module-level parser cache — safe because Parser is stateless and language grammars are read-only let parserInstance: any = null; const loadedLanguages = new Set(); diff --git a/gitnexus/src/core/embeddings/language-patterns.ts b/gitnexus/src/core/embeddings/language-patterns.ts index 0625cbfe2..d26ae06a9 100644 --- a/gitnexus/src/core/embeddings/language-patterns.ts +++ b/gitnexus/src/core/embeddings/language-patterns.ts @@ -3,275 +3,116 @@ export interface LanguagePatterns { extractProperties(content: string): string[]; } -function cStyleMethods(modifiers: string): (content: string) => string[] { - const re = new RegExp( - `(?:${modifiers})\\s+` + - `(?:<[^>]+>\\s+)?` + - `(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` + - `(\\w+)\\s*\\(`, - 'gm', - ); +function regexExtractor(re: RegExp, group: number = 1): (content: string) => string[] { return (content: string): string[] => { re.lastIndex = 0; const out: string[] = []; let m: RegExpExecArray | null; while ((m = re.exec(content)) !== null) { - out.push(m[1]); + out.push(m[group]); } return out; }; } -function cStyleProperties(typePattern: string): (content: string) => string[] { - const re = new RegExp( - `(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` + - typePattern + - `\\s+(\\w+)\\s*(?:[;=]|\\{)`, - 'gm', - ); - return (content: string): string[] => { - re.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - out.push(m[1]); - } - return out; - }; -} +// --- Regex constants --- const tsMethodRe = /(?:(?:async|static|public|private|protected|readonly|abstract|override)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm; - const tsPropertyRe = /(?:private|public|protected|readonly|static|abstract|override)\s+(\w+)\s*[;=:]/gm; -function extractTsMethods(content: string): string[] { - tsMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = tsMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractTsProperties(content: string): string[] { - tsPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = tsPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - const pythonMethodRe = /def\s+(\w+)\s*\(/gm; const pythonPropertyRe = /self\.(\w+)\s*=/gm; -function extractPythonMethods(content: string): string[] { - pythonMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = pythonMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractPythonProperties(content: string): string[] { - pythonPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = pythonPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - const kotlinMethodRe = /(?:(?:public|private|protected|internal|suspend|inline|open|override)\s+)*fun\s+(\w+)\s*[(<]/gm; const kotlinPropertyRe = /(?:(?:public|private|protected|internal|lateinit)\s+)*(?:val|var)\s+(\w+)/gm; -function extractKotlinMethods(content: string): string[] { - kotlinMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = kotlinMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractKotlinProperties(content: string): string[] { - kotlinPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = kotlinPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - const goPropertyRe = /^\s+(\w+)\s+\w+/gm; +const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm; +const rustPropertyRe = /^\s+(\w+)\s*:/gm; + +const javaMethodRe = new RegExp( + `(?:public|private|protected|static)\\s+` + + `(?:<[^>]+>\\s+)?` + + `(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` + + `(\\w+)\\s*\\(`, + 'gm', +); +const javaPropertyRe = new RegExp( + `(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` + + `\\w+(?:<[^>]+>)?(?:\\[\\])?` + + `\\s+(\\w+)\\s*(?:[;=]|\\{)`, + 'gm', +); + +const csharpMethodRe = new RegExp( + `(?:public|private|protected|internal|static|virtual|override|async)\\s+` + + `(?:<[^>]+>\\s+)?` + + `(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` + + `(\\w+)\\s*\\(`, + 'gm', +); +const csharpPropertyRe = + /(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm; + +const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm; +const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm; + +const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm; +const rubyAttrLineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm; +const rubyAttrSymRe = /:(\w+)/g; + +const swiftMethodRe = + /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm; +const swiftPropertyRe = + /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm; + +const cppMethodRe = + /(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm; +const cppPropertyRe = + /(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm; + +const dartMethodRe = + /(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm; +const dartPropertyRe = + /(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm; + +const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm; +const jsPropertyRe = /this\.(\w+)\s*=/gm; + +// --- Special extractors with filtering --- + function extractGoProperties(content: string): string[] { goPropertyRe.lastIndex = 0; const out: string[] = []; let m: RegExpExecArray | null; while ((m = goPropertyRe.exec(content)) !== null) { const name = m[1]; - if (name[0] === name[0].toUpperCase()) { + if (name[0] >= 'A' && name[0] <= 'Z') { out.push(name); } } return out; } -const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm; -const rustPropertyRe = /^\s+(\w+)\s*:/gm; - -function extractRustMethods(content: string): string[] { - rustMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = rustMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractRustProperties(content: string): string[] { - rustPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = rustPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -const csharpPropertyRe = - /(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm; - -function extractCsharpProperties(content: string): string[] { - csharpPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = csharpPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm; -const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm; - -function extractPhpMethods(content: string): string[] { - phpMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = phpMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractPhpProperties(content: string): string[] { - phpPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = phpPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm; -const rubyAttrRe = /:(\w+)/g; - -function extractRubyMethods(content: string): string[] { - rubyMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = rubyMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - function extractRubyProperties(content: string): string[] { const out: string[] = []; - const lineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm; - lineRe.lastIndex = 0; + rubyAttrLineRe.lastIndex = 0; let line: RegExpExecArray | null; - while ((line = lineRe.exec(content)) !== null) { - const args = line[1]; - rubyAttrRe.lastIndex = 0; + while ((line = rubyAttrLineRe.exec(content)) !== null) { + rubyAttrSymRe.lastIndex = 0; let sym: RegExpExecArray | null; - while ((sym = rubyAttrRe.exec(args)) !== null) { + while ((sym = rubyAttrSymRe.exec(line[1])) !== null) { out.push(sym[1]); } } return out; } -const swiftMethodRe = - /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm; -const swiftPropertyRe = - /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm; - -function extractSwiftMethods(content: string): string[] { - swiftMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = swiftMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractSwiftProperties(content: string): string[] { - swiftPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = swiftPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -const cppMethodRe = - /(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm; -const cppPropertyRe = - /(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm; - -function extractCppMethods(content: string): string[] { - cppMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = cppMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractCppProperties(content: string): string[] { - cppPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = cppPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -const dartMethodRe = - /(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm; -const dartPropertyRe = - /(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm; - function extractDartMethods(content: string): string[] { dartMethodRe.lastIndex = 0; const out: string[] = []; @@ -303,83 +144,60 @@ function extractDartProperties(content: string): string[] { return out; } -const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm; -const jsPropertyRe = /this\.(\w+)\s*=/gm; - -function extractJsMethods(content: string): string[] { - jsMethodRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = jsMethodRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} - -function extractJsProperties(content: string): string[] { - jsPropertyRe.lastIndex = 0; - const out: string[] = []; - let m: RegExpExecArray | null; - while ((m = jsPropertyRe.exec(content)) !== null) { - out.push(m[1]); - } - return out; -} +// --- Pattern registry --- export const PATTERNS: Record = { typescript: { - extractMethods: extractTsMethods, - extractProperties: extractTsProperties, + extractMethods: regexExtractor(tsMethodRe), + extractProperties: regexExtractor(tsPropertyRe), }, javascript: { - extractMethods: extractJsMethods, - extractProperties: extractJsProperties, + extractMethods: regexExtractor(jsMethodRe), + extractProperties: regexExtractor(jsPropertyRe), }, python: { - extractMethods: extractPythonMethods, - extractProperties: extractPythonProperties, + extractMethods: regexExtractor(pythonMethodRe), + extractProperties: regexExtractor(pythonPropertyRe), }, java: { - extractMethods: cStyleMethods('public|private|protected|static'), - extractProperties: cStyleProperties('\\w+(?:<[^>]+>)?(?:\\[\\])?'), + extractMethods: regexExtractor(javaMethodRe), + extractProperties: regexExtractor(javaPropertyRe), }, kotlin: { - extractMethods: extractKotlinMethods, - extractProperties: extractKotlinProperties, + extractMethods: regexExtractor(kotlinMethodRe), + extractProperties: regexExtractor(kotlinPropertyRe), }, go: { extractMethods: () => [], extractProperties: extractGoProperties, }, rust: { - extractMethods: extractRustMethods, - extractProperties: extractRustProperties, + extractMethods: regexExtractor(rustMethodRe), + extractProperties: regexExtractor(rustPropertyRe), }, csharp: { - extractMethods: cStyleMethods( - 'public|private|protected|internal|static|virtual|override|async', - ), - extractProperties: extractCsharpProperties, + extractMethods: regexExtractor(csharpMethodRe), + extractProperties: regexExtractor(csharpPropertyRe), }, php: { - extractMethods: extractPhpMethods, - extractProperties: extractPhpProperties, + extractMethods: regexExtractor(phpMethodRe), + extractProperties: regexExtractor(phpPropertyRe), }, ruby: { - extractMethods: extractRubyMethods, + extractMethods: regexExtractor(rubyMethodRe), extractProperties: extractRubyProperties, }, swift: { - extractMethods: extractSwiftMethods, - extractProperties: extractSwiftProperties, + extractMethods: regexExtractor(swiftMethodRe), + extractProperties: regexExtractor(swiftPropertyRe), }, cpp: { - extractMethods: extractCppMethods, - extractProperties: extractCppProperties, + extractMethods: regexExtractor(cppMethodRe), + extractProperties: regexExtractor(cppPropertyRe), }, c: { - extractMethods: extractCppMethods, - extractProperties: extractCppProperties, + extractMethods: regexExtractor(cppMethodRe), + extractProperties: regexExtractor(cppPropertyRe), }, dart: { extractMethods: extractDartMethods, diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index 914911ae0..2f43d635b 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -90,24 +90,7 @@ const buildMetadataHeader = (node: EmbeddableNode, config: Partial, -): string => { - const header = buildMetadataHeader(node, config); - const cleaned = cleanContent(codeBody); - return `${header}\n\n${cleaned}`; -}; - -/** - * Generate embedding text for Constructor nodes - */ -const generateConstructorText = ( +const generateCodeBodyText = ( node: EmbeddableNode, codeBody: string, config: Partial, @@ -187,9 +170,9 @@ const generateClassText = ( return parts.join('\n'); }; -/** - * Extract class/interface/struct declaration lines (no method bodies) - */ +const DECL_START_RE = + /^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/; + const extractDeclarationOnly = (content: string): string => { const lines = content.split('\n'); const declLines: string[] = []; @@ -198,20 +181,14 @@ const extractDeclarationOnly = (content: string): string => { for (const line of lines) { const trimmed = line.trim(); - if ( - !started && - (trimmed.match(/^(?:export\s+)?(?:abstract\s+)?(?:data\s+)?class\s/) || - trimmed.match(/^(?:pub\s+)?struct\s/) || - trimmed.match(/^(?:pub\s+)?enum\s/) || - trimmed.match(/^type\s+\w+\s+struct/) || - trimmed.match(/^class\s/) || - trimmed.match(/^interface\s/)) - ) { + if (!started && DECL_START_RE.test(trimmed)) { started = true; } if (started) { - depth += (trimmed.match(/\{/g) || []).length; - depth -= (trimmed.match(/\}/g) || []).length; + for (const ch of trimmed) { + if (ch === '{') depth++; + else if (ch === '}') depth--; + } declLines.push(trimmed); if (depth <= 0 && declLines.length > 1) break; } @@ -220,29 +197,12 @@ const extractDeclarationOnly = (content: string): string => { return declLines.join('\n').trim(); }; -/** - * Generate embedding text for short nodes (TypeAlias, Const, etc.) - * No chunking, just metadata + full content - */ const generateShortNodeText = (node: EmbeddableNode, config: Partial): string => { const header = buildMetadataHeader(node, config); const cleaned = cleanContent(node.content); return `${header}\n\n${cleaned}`; }; -/** - * Generate embedding text for Interface/Struct/Enum/Trait/etc. (chunkable but non-function) - */ -const generateChunkableNonFunctionText = ( - node: EmbeddableNode, - codeBody: string, - config: Partial, -): string => { - const header = buildMetadataHeader(node, config); - const cleaned = cleanContent(codeBody); - return `${header}\n\n${cleaned}`; -}; - /** * Generate embedding text for any embeddable node * Dispatches to the appropriate generator based on node label @@ -260,16 +220,7 @@ export const generateEmbeddingText = ( return generateClassText(node, codeBody, config); } - if (node.label === 'Constructor') { - return generateConstructorText(node, codeBody, config); - } - - if (node.label === 'Function' || node.label === 'Method') { - return generateFunctionText(node, codeBody, config); - } - - // Other chunkable types (Interface, Struct, Enum, Trait, Impl, Macro, Namespace) - return generateChunkableNonFunctionText(node, codeBody, config); + return generateCodeBodyText(node, codeBody, config); }; /**