mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-12 23:02:45 +00:00
refactor(embeddings): eliminate duplicate code in language-patterns and text-generator
- Replace ~20 identical extract functions with regexExtractor factory - Merge 3 identical text generators (generateConstructorText, generateFunctionText, generateChunkableNonFunctionText) into generateCodeBodyText - Consolidate 6 regex checks in extractDeclarationOnly into single DECL_START_RE, use character loop for brace counting - Hoist rubyAttrLineRe to module-level constant - Use range check instead of toUpperCase() in extractGoProperties
This commit is contained in:
wangjichao
parent
0c84b39b99
commit
de49e3027b
3 changed files with 97 additions and 329 deletions
|
|
@ -14,8 +14,7 @@ export { type Chunk, characterChunk } from './character-chunk.js';
|
|||
import { characterChunk } from './character-chunk.js';
|
||||
import type { Chunk } from './character-chunk.js';
|
||||
|
||||
// Module-level parser cache — safe because Parser is stateless
|
||||
// and language grammars are read-only
|
||||
// Module-level parser cache — safe because Parser is stateless and language grammars are read-only
|
||||
let parserInstance: any = null;
|
||||
const loadedLanguages = new Set<string>();
|
||||
|
||||
|
|
|
|||
|
|
@ -3,275 +3,116 @@ export interface LanguagePatterns {
|
|||
extractProperties(content: string): string[];
|
||||
}
|
||||
|
||||
function cStyleMethods(modifiers: string): (content: string) => string[] {
|
||||
const re = new RegExp(
|
||||
`(?:${modifiers})\\s+` +
|
||||
`(?:<[^>]+>\\s+)?` +
|
||||
`(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` +
|
||||
`(\\w+)\\s*\\(`,
|
||||
'gm',
|
||||
);
|
||||
function regexExtractor(re: RegExp, group: number = 1): (content: string) => string[] {
|
||||
return (content: string): string[] => {
|
||||
re.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
out.push(m[group]);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
}
|
||||
|
||||
function cStyleProperties(typePattern: string): (content: string) => string[] {
|
||||
const re = new RegExp(
|
||||
`(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` +
|
||||
typePattern +
|
||||
`\\s+(\\w+)\\s*(?:[;=]|\\{)`,
|
||||
'gm',
|
||||
);
|
||||
return (content: string): string[] => {
|
||||
re.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
}
|
||||
// --- Regex constants ---
|
||||
|
||||
const tsMethodRe =
|
||||
/(?:(?:async|static|public|private|protected|readonly|abstract|override)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm;
|
||||
|
||||
const tsPropertyRe =
|
||||
/(?:private|public|protected|readonly|static|abstract|override)\s+(\w+)\s*[;=:]/gm;
|
||||
|
||||
function extractTsMethods(content: string): string[] {
|
||||
tsMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tsMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractTsProperties(content: string): string[] {
|
||||
tsPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = tsPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const pythonMethodRe = /def\s+(\w+)\s*\(/gm;
|
||||
const pythonPropertyRe = /self\.(\w+)\s*=/gm;
|
||||
|
||||
function extractPythonMethods(content: string): string[] {
|
||||
pythonMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = pythonMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractPythonProperties(content: string): string[] {
|
||||
pythonPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = pythonPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const kotlinMethodRe =
|
||||
/(?:(?:public|private|protected|internal|suspend|inline|open|override)\s+)*fun\s+(\w+)\s*[(<]/gm;
|
||||
const kotlinPropertyRe =
|
||||
/(?:(?:public|private|protected|internal|lateinit)\s+)*(?:val|var)\s+(\w+)/gm;
|
||||
|
||||
function extractKotlinMethods(content: string): string[] {
|
||||
kotlinMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = kotlinMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractKotlinProperties(content: string): string[] {
|
||||
kotlinPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = kotlinPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const goPropertyRe = /^\s+(\w+)\s+\w+/gm;
|
||||
|
||||
const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm;
|
||||
const rustPropertyRe = /^\s+(\w+)\s*:/gm;
|
||||
|
||||
const javaMethodRe = new RegExp(
|
||||
`(?:public|private|protected|static)\\s+` +
|
||||
`(?:<[^>]+>\\s+)?` +
|
||||
`(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` +
|
||||
`(\\w+)\\s*\\(`,
|
||||
'gm',
|
||||
);
|
||||
const javaPropertyRe = new RegExp(
|
||||
`(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` +
|
||||
`\\w+(?:<[^>]+>)?(?:\\[\\])?` +
|
||||
`\\s+(\\w+)\\s*(?:[;=]|\\{)`,
|
||||
'gm',
|
||||
);
|
||||
|
||||
const csharpMethodRe = new RegExp(
|
||||
`(?:public|private|protected|internal|static|virtual|override|async)\\s+` +
|
||||
`(?:<[^>]+>\\s+)?` +
|
||||
`(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` +
|
||||
`(\\w+)\\s*\\(`,
|
||||
'gm',
|
||||
);
|
||||
const csharpPropertyRe =
|
||||
/(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm;
|
||||
|
||||
const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm;
|
||||
const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm;
|
||||
|
||||
const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm;
|
||||
const rubyAttrLineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm;
|
||||
const rubyAttrSymRe = /:(\w+)/g;
|
||||
|
||||
const swiftMethodRe =
|
||||
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm;
|
||||
const swiftPropertyRe =
|
||||
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm;
|
||||
|
||||
const cppMethodRe =
|
||||
/(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm;
|
||||
const cppPropertyRe =
|
||||
/(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm;
|
||||
|
||||
const dartMethodRe =
|
||||
/(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm;
|
||||
const dartPropertyRe =
|
||||
/(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm;
|
||||
|
||||
const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm;
|
||||
const jsPropertyRe = /this\.(\w+)\s*=/gm;
|
||||
|
||||
// --- Special extractors with filtering ---
|
||||
|
||||
function extractGoProperties(content: string): string[] {
|
||||
goPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = goPropertyRe.exec(content)) !== null) {
|
||||
const name = m[1];
|
||||
if (name[0] === name[0].toUpperCase()) {
|
||||
if (name[0] >= 'A' && name[0] <= 'Z') {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm;
|
||||
const rustPropertyRe = /^\s+(\w+)\s*:/gm;
|
||||
|
||||
function extractRustMethods(content: string): string[] {
|
||||
rustMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = rustMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractRustProperties(content: string): string[] {
|
||||
rustPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = rustPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const csharpPropertyRe =
|
||||
/(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm;
|
||||
|
||||
function extractCsharpProperties(content: string): string[] {
|
||||
csharpPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = csharpPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm;
|
||||
const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm;
|
||||
|
||||
function extractPhpMethods(content: string): string[] {
|
||||
phpMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = phpMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractPhpProperties(content: string): string[] {
|
||||
phpPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = phpPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm;
|
||||
const rubyAttrRe = /:(\w+)/g;
|
||||
|
||||
function extractRubyMethods(content: string): string[] {
|
||||
rubyMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = rubyMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractRubyProperties(content: string): string[] {
|
||||
const out: string[] = [];
|
||||
const lineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm;
|
||||
lineRe.lastIndex = 0;
|
||||
rubyAttrLineRe.lastIndex = 0;
|
||||
let line: RegExpExecArray | null;
|
||||
while ((line = lineRe.exec(content)) !== null) {
|
||||
const args = line[1];
|
||||
rubyAttrRe.lastIndex = 0;
|
||||
while ((line = rubyAttrLineRe.exec(content)) !== null) {
|
||||
rubyAttrSymRe.lastIndex = 0;
|
||||
let sym: RegExpExecArray | null;
|
||||
while ((sym = rubyAttrRe.exec(args)) !== null) {
|
||||
while ((sym = rubyAttrSymRe.exec(line[1])) !== null) {
|
||||
out.push(sym[1]);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const swiftMethodRe =
|
||||
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm;
|
||||
const swiftPropertyRe =
|
||||
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm;
|
||||
|
||||
function extractSwiftMethods(content: string): string[] {
|
||||
swiftMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = swiftMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractSwiftProperties(content: string): string[] {
|
||||
swiftPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = swiftPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const cppMethodRe =
|
||||
/(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm;
|
||||
const cppPropertyRe =
|
||||
/(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm;
|
||||
|
||||
function extractCppMethods(content: string): string[] {
|
||||
cppMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = cppMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractCppProperties(content: string): string[] {
|
||||
cppPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = cppPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const dartMethodRe =
|
||||
/(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm;
|
||||
const dartPropertyRe =
|
||||
/(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm;
|
||||
|
||||
function extractDartMethods(content: string): string[] {
|
||||
dartMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
|
|
@ -303,83 +144,60 @@ function extractDartProperties(content: string): string[] {
|
|||
return out;
|
||||
}
|
||||
|
||||
const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm;
|
||||
const jsPropertyRe = /this\.(\w+)\s*=/gm;
|
||||
|
||||
function extractJsMethods(content: string): string[] {
|
||||
jsMethodRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = jsMethodRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractJsProperties(content: string): string[] {
|
||||
jsPropertyRe.lastIndex = 0;
|
||||
const out: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = jsPropertyRe.exec(content)) !== null) {
|
||||
out.push(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// --- Pattern registry ---
|
||||
|
||||
export const PATTERNS: Record<string, LanguagePatterns> = {
|
||||
typescript: {
|
||||
extractMethods: extractTsMethods,
|
||||
extractProperties: extractTsProperties,
|
||||
extractMethods: regexExtractor(tsMethodRe),
|
||||
extractProperties: regexExtractor(tsPropertyRe),
|
||||
},
|
||||
javascript: {
|
||||
extractMethods: extractJsMethods,
|
||||
extractProperties: extractJsProperties,
|
||||
extractMethods: regexExtractor(jsMethodRe),
|
||||
extractProperties: regexExtractor(jsPropertyRe),
|
||||
},
|
||||
python: {
|
||||
extractMethods: extractPythonMethods,
|
||||
extractProperties: extractPythonProperties,
|
||||
extractMethods: regexExtractor(pythonMethodRe),
|
||||
extractProperties: regexExtractor(pythonPropertyRe),
|
||||
},
|
||||
java: {
|
||||
extractMethods: cStyleMethods('public|private|protected|static'),
|
||||
extractProperties: cStyleProperties('\\w+(?:<[^>]+>)?(?:\\[\\])?'),
|
||||
extractMethods: regexExtractor(javaMethodRe),
|
||||
extractProperties: regexExtractor(javaPropertyRe),
|
||||
},
|
||||
kotlin: {
|
||||
extractMethods: extractKotlinMethods,
|
||||
extractProperties: extractKotlinProperties,
|
||||
extractMethods: regexExtractor(kotlinMethodRe),
|
||||
extractProperties: regexExtractor(kotlinPropertyRe),
|
||||
},
|
||||
go: {
|
||||
extractMethods: () => [],
|
||||
extractProperties: extractGoProperties,
|
||||
},
|
||||
rust: {
|
||||
extractMethods: extractRustMethods,
|
||||
extractProperties: extractRustProperties,
|
||||
extractMethods: regexExtractor(rustMethodRe),
|
||||
extractProperties: regexExtractor(rustPropertyRe),
|
||||
},
|
||||
csharp: {
|
||||
extractMethods: cStyleMethods(
|
||||
'public|private|protected|internal|static|virtual|override|async',
|
||||
),
|
||||
extractProperties: extractCsharpProperties,
|
||||
extractMethods: regexExtractor(csharpMethodRe),
|
||||
extractProperties: regexExtractor(csharpPropertyRe),
|
||||
},
|
||||
php: {
|
||||
extractMethods: extractPhpMethods,
|
||||
extractProperties: extractPhpProperties,
|
||||
extractMethods: regexExtractor(phpMethodRe),
|
||||
extractProperties: regexExtractor(phpPropertyRe),
|
||||
},
|
||||
ruby: {
|
||||
extractMethods: extractRubyMethods,
|
||||
extractMethods: regexExtractor(rubyMethodRe),
|
||||
extractProperties: extractRubyProperties,
|
||||
},
|
||||
swift: {
|
||||
extractMethods: extractSwiftMethods,
|
||||
extractProperties: extractSwiftProperties,
|
||||
extractMethods: regexExtractor(swiftMethodRe),
|
||||
extractProperties: regexExtractor(swiftPropertyRe),
|
||||
},
|
||||
cpp: {
|
||||
extractMethods: extractCppMethods,
|
||||
extractProperties: extractCppProperties,
|
||||
extractMethods: regexExtractor(cppMethodRe),
|
||||
extractProperties: regexExtractor(cppPropertyRe),
|
||||
},
|
||||
c: {
|
||||
extractMethods: extractCppMethods,
|
||||
extractProperties: extractCppProperties,
|
||||
extractMethods: regexExtractor(cppMethodRe),
|
||||
extractProperties: regexExtractor(cppPropertyRe),
|
||||
},
|
||||
dart: {
|
||||
extractMethods: extractDartMethods,
|
||||
|
|
|
|||
|
|
@ -90,24 +90,7 @@ const buildMetadataHeader = (node: EmbeddableNode, config: Partial<EmbeddingConf
|
|||
return parts.join('\n');
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for Function/Method nodes
|
||||
* Includes metadata header + code body (chunk text passed separately)
|
||||
*/
|
||||
const generateFunctionText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
): string => {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const cleaned = cleanContent(codeBody);
|
||||
return `${header}\n\n${cleaned}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for Constructor nodes
|
||||
*/
|
||||
const generateConstructorText = (
|
||||
const generateCodeBodyText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
|
|
@ -187,9 +170,9 @@ const generateClassText = (
|
|||
return parts.join('\n');
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract class/interface/struct declaration lines (no method bodies)
|
||||
*/
|
||||
const DECL_START_RE =
|
||||
/^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/;
|
||||
|
||||
const extractDeclarationOnly = (content: string): string => {
|
||||
const lines = content.split('\n');
|
||||
const declLines: string[] = [];
|
||||
|
|
@ -198,20 +181,14 @@ const extractDeclarationOnly = (content: string): string => {
|
|||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (
|
||||
!started &&
|
||||
(trimmed.match(/^(?:export\s+)?(?:abstract\s+)?(?:data\s+)?class\s/) ||
|
||||
trimmed.match(/^(?:pub\s+)?struct\s/) ||
|
||||
trimmed.match(/^(?:pub\s+)?enum\s/) ||
|
||||
trimmed.match(/^type\s+\w+\s+struct/) ||
|
||||
trimmed.match(/^class\s/) ||
|
||||
trimmed.match(/^interface\s/))
|
||||
) {
|
||||
if (!started && DECL_START_RE.test(trimmed)) {
|
||||
started = true;
|
||||
}
|
||||
if (started) {
|
||||
depth += (trimmed.match(/\{/g) || []).length;
|
||||
depth -= (trimmed.match(/\}/g) || []).length;
|
||||
for (const ch of trimmed) {
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
}
|
||||
declLines.push(trimmed);
|
||||
if (depth <= 0 && declLines.length > 1) break;
|
||||
}
|
||||
|
|
@ -220,29 +197,12 @@ const extractDeclarationOnly = (content: string): string => {
|
|||
return declLines.join('\n').trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for short nodes (TypeAlias, Const, etc.)
|
||||
* No chunking, just metadata + full content
|
||||
*/
|
||||
const generateShortNodeText = (node: EmbeddableNode, config: Partial<EmbeddingConfig>): string => {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const cleaned = cleanContent(node.content);
|
||||
return `${header}\n\n${cleaned}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for Interface/Struct/Enum/Trait/etc. (chunkable but non-function)
|
||||
*/
|
||||
const generateChunkableNonFunctionText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
): string => {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const cleaned = cleanContent(codeBody);
|
||||
return `${header}\n\n${cleaned}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for any embeddable node
|
||||
* Dispatches to the appropriate generator based on node label
|
||||
|
|
@ -260,16 +220,7 @@ export const generateEmbeddingText = (
|
|||
return generateClassText(node, codeBody, config);
|
||||
}
|
||||
|
||||
if (node.label === 'Constructor') {
|
||||
return generateConstructorText(node, codeBody, config);
|
||||
}
|
||||
|
||||
if (node.label === 'Function' || node.label === 'Method') {
|
||||
return generateFunctionText(node, codeBody, config);
|
||||
}
|
||||
|
||||
// Other chunkable types (Interface, Struct, Enum, Trait, Impl, Macro, Namespace)
|
||||
return generateChunkableNonFunctionText(node, codeBody, config);
|
||||
return generateCodeBodyText(node, codeBody, config);
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue