From bd4d15c4603473c0892164f1f0d996980ba96706 Mon Sep 17 00:00:00 2001 From: wangjichao Date: Tue, 14 Apr 2026 09:46:26 +0800 Subject: [PATCH] feat(embeddings): add language-patterns module for multi-language class text --- .../src/core/embeddings/language-patterns.ts | 392 ++++++++++++++++++ gitnexus/test/unit/language-patterns.test.ts | 352 ++++++++++++++++ 2 files changed, 744 insertions(+) create mode 100644 gitnexus/src/core/embeddings/language-patterns.ts create mode 100644 gitnexus/test/unit/language-patterns.test.ts diff --git a/gitnexus/src/core/embeddings/language-patterns.ts b/gitnexus/src/core/embeddings/language-patterns.ts new file mode 100644 index 000000000..0625cbfe2 --- /dev/null +++ b/gitnexus/src/core/embeddings/language-patterns.ts @@ -0,0 +1,392 @@ +export interface LanguagePatterns { + extractMethods(content: string): string[]; + extractProperties(content: string): string[]; +} + +function cStyleMethods(modifiers: string): (content: string) => string[] { + const re = new RegExp( + `(?:${modifiers})\\s+` + + `(?:<[^>]+>\\s+)?` + + `(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` + + `(\\w+)\\s*\\(`, + 'gm', + ); + return (content: string): string[] => { + re.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = re.exec(content)) !== null) { + out.push(m[1]); + } + return out; + }; +} + +function cStyleProperties(typePattern: string): (content: string) => string[] { + const re = new RegExp( + `(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` + + typePattern + + `\\s+(\\w+)\\s*(?:[;=]|\\{)`, + 'gm', + ); + return (content: string): string[] => { + re.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = re.exec(content)) !== null) { + out.push(m[1]); + } + return out; + }; +} + +const tsMethodRe = + /(?:(?:async|static|public|private|protected|readonly|abstract|override)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm; + +const tsPropertyRe = + /(?:private|public|protected|readonly|static|abstract|override)\s+(\w+)\s*[;=:]/gm; + +function extractTsMethods(content: string): string[] { + tsMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = tsMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractTsProperties(content: string): string[] { + tsPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = tsPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const pythonMethodRe = /def\s+(\w+)\s*\(/gm; +const pythonPropertyRe = /self\.(\w+)\s*=/gm; + +function extractPythonMethods(content: string): string[] { + pythonMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = pythonMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractPythonProperties(content: string): string[] { + pythonPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = pythonPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const kotlinMethodRe = + /(?:(?:public|private|protected|internal|suspend|inline|open|override)\s+)*fun\s+(\w+)\s*[(<]/gm; +const kotlinPropertyRe = + /(?:(?:public|private|protected|internal|lateinit)\s+)*(?:val|var)\s+(\w+)/gm; + +function extractKotlinMethods(content: string): string[] { + kotlinMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = kotlinMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractKotlinProperties(content: string): string[] { + kotlinPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = kotlinPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const goPropertyRe = /^\s+(\w+)\s+\w+/gm; + +function extractGoProperties(content: string): string[] { + goPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = goPropertyRe.exec(content)) !== null) { + const name = m[1]; + if (name[0] === name[0].toUpperCase()) { + out.push(name); + } + } + return out; +} + +const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm; +const rustPropertyRe = /^\s+(\w+)\s*:/gm; + +function extractRustMethods(content: string): string[] { + rustMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = rustMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractRustProperties(content: string): string[] { + rustPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = rustPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const csharpPropertyRe = + /(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm; + +function extractCsharpProperties(content: string): string[] { + csharpPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = csharpPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm; +const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm; + +function extractPhpMethods(content: string): string[] { + phpMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = phpMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractPhpProperties(content: string): string[] { + phpPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = phpPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm; +const rubyAttrRe = /:(\w+)/g; + +function extractRubyMethods(content: string): string[] { + rubyMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = rubyMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractRubyProperties(content: string): string[] { + const out: string[] = []; + const lineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm; + lineRe.lastIndex = 0; + let line: RegExpExecArray | null; + while ((line = lineRe.exec(content)) !== null) { + const args = line[1]; + rubyAttrRe.lastIndex = 0; + let sym: RegExpExecArray | null; + while ((sym = rubyAttrRe.exec(args)) !== null) { + out.push(sym[1]); + } + } + return out; +} + +const swiftMethodRe = + /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm; +const swiftPropertyRe = + /(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm; + +function extractSwiftMethods(content: string): string[] { + swiftMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = swiftMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractSwiftProperties(content: string): string[] { + swiftPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = swiftPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const cppMethodRe = + /(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm; +const cppPropertyRe = + /(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm; + +function extractCppMethods(content: string): string[] { + cppMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = cppMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractCppProperties(content: string): string[] { + cppPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = cppPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +const dartMethodRe = + /(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm; +const dartPropertyRe = + /(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm; + +function extractDartMethods(content: string): string[] { + dartMethodRe.lastIndex = 0; + const out: string[] = []; + const seen = new Set(); + let m: RegExpExecArray | null; + while ((m = dartMethodRe.exec(content)) !== null) { + const name = m[1]; + if (!seen.has(name)) { + seen.add(name); + out.push(name); + } + } + return out; +} + +function extractDartProperties(content: string): string[] { + dartPropertyRe.lastIndex = 0; + const out: string[] = []; + const seen = new Set(); + let m: RegExpExecArray | null; + while ((m = dartPropertyRe.exec(content)) !== null) { + const name = m[1]; + if (name === 'class' || name === 'void' || name === 'async') continue; + if (!seen.has(name)) { + seen.add(name); + out.push(name); + } + } + return out; +} + +const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm; +const jsPropertyRe = /this\.(\w+)\s*=/gm; + +function extractJsMethods(content: string): string[] { + jsMethodRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = jsMethodRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +function extractJsProperties(content: string): string[] { + jsPropertyRe.lastIndex = 0; + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = jsPropertyRe.exec(content)) !== null) { + out.push(m[1]); + } + return out; +} + +export const PATTERNS: Record = { + typescript: { + extractMethods: extractTsMethods, + extractProperties: extractTsProperties, + }, + javascript: { + extractMethods: extractJsMethods, + extractProperties: extractJsProperties, + }, + python: { + extractMethods: extractPythonMethods, + extractProperties: extractPythonProperties, + }, + java: { + extractMethods: cStyleMethods('public|private|protected|static'), + extractProperties: cStyleProperties('\\w+(?:<[^>]+>)?(?:\\[\\])?'), + }, + kotlin: { + extractMethods: extractKotlinMethods, + extractProperties: extractKotlinProperties, + }, + go: { + extractMethods: () => [], + extractProperties: extractGoProperties, + }, + rust: { + extractMethods: extractRustMethods, + extractProperties: extractRustProperties, + }, + csharp: { + extractMethods: cStyleMethods( + 'public|private|protected|internal|static|virtual|override|async', + ), + extractProperties: extractCsharpProperties, + }, + php: { + extractMethods: extractPhpMethods, + extractProperties: extractPhpProperties, + }, + ruby: { + extractMethods: extractRubyMethods, + extractProperties: extractRubyProperties, + }, + swift: { + extractMethods: extractSwiftMethods, + extractProperties: extractSwiftProperties, + }, + cpp: { + extractMethods: extractCppMethods, + extractProperties: extractCppProperties, + }, + c: { + extractMethods: extractCppMethods, + extractProperties: extractCppProperties, + }, + dart: { + extractMethods: extractDartMethods, + extractProperties: extractDartProperties, + }, +}; + +export function getPatternsForLanguage(language: string): LanguagePatterns | undefined { + return PATTERNS[language]; +} diff --git a/gitnexus/test/unit/language-patterns.test.ts b/gitnexus/test/unit/language-patterns.test.ts new file mode 100644 index 000000000..bcec48df2 --- /dev/null +++ b/gitnexus/test/unit/language-patterns.test.ts @@ -0,0 +1,352 @@ +import { describe, it, expect } from 'vitest'; +import { PATTERNS, getPatternsForLanguage } from '../../src/core/embeddings/language-patterns.js'; + +describe('language-patterns', () => { + describe('getPatternsForLanguage', () => { + it('returns patterns for known languages', () => { + expect(getPatternsForLanguage('typescript')).toBeDefined(); + expect(getPatternsForLanguage('python')).toBeDefined(); + expect(getPatternsForLanguage('rust')).toBeDefined(); + }); + + it('returns undefined for unknown language', () => { + expect(getPatternsForLanguage('brainfuck')).toBeUndefined(); + }); + }); + + describe('TypeScript/JavaScript', () => { + const patterns = PATTERNS['typescript']!; + + it('extracts method names', () => { + const content = `class Foo { + async getData(): Promise { } + handleClick(event: Event) { } + private _internal() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['getData', 'handleClick', '_internal'])); + }); + + it('extracts property names', () => { + const content = `class Foo { + private name: string; + public age: number; + readonly id: string; +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'age', 'id'])); + }); + }); + + describe('Python', () => { + const patterns = PATTERNS['python']!; + + it('extracts method names', () => { + const content = `class User: + def __init__(self, name): + self.name = name + + def get_full_name(self): + return self.name + + @staticmethod + def from_dict(data): + return User(data["name"]) +`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['__init__', 'get_full_name', 'from_dict'])); + }); + + it('extracts property names', () => { + const content = `class User: + def __init__(self, name): + self.name = name + self._email = "" + self.age = 0 +`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', '_email', 'age'])); + }); + }); + + describe('Java', () => { + const patterns = PATTERNS['java']!; + + it('extracts method names', () => { + const content = `public class Service { + public String getName() { } + private void process(Data d) { } + static Builder builder() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['getName', 'process', 'builder'])); + }); + + it('extracts property names', () => { + const content = `public class Service { + private String name; + protected int count; + public static final int MAX = 100; +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'count'])); + }); + }); + + describe('Kotlin', () => { + const patterns = PATTERNS['kotlin']!; + + it('extracts method names', () => { + const content = `class User(val name: String) { + fun greet(): String = "Hello" + suspend fun fetchData() { } + private fun validate() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['greet', 'fetchData', 'validate'])); + }); + + it('extracts property names', () => { + const content = `class User { + val name: String + var age: Int = 0 + private val _id: UUID +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'age', '_id'])); + }); + }); + + describe('Go', () => { + const patterns = PATTERNS['go']!; + + it('extracts no methods (Go methods are external to struct)', () => { + const content = `type User struct { + Name string + Age int +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual([]); + }); + + it('extracts struct fields', () => { + const content = `type User struct { + Name string + Age int + json string +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['Name', 'Age'])); + }); + }); + + describe('Rust', () => { + const patterns = PATTERNS['rust']!; + + it('extracts method names from impl block', () => { + const content = `impl User { + pub fn new(name: &str) -> Self { } + fn validate(&self) -> bool { } + pub async fn fetch(&self) -> Result { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['new', 'validate', 'fetch'])); + }); + + it('extracts struct fields', () => { + const content = `struct User { + name: String, + age: u32, + active: bool, +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'age', 'active'])); + }); + }); + + describe('C#', () => { + const patterns = PATTERNS['csharp']!; + + it('extracts method names', () => { + const content = `public class Service { + public async Task QueryAsync() { } + private void Process() { } + public static Service Create() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['QueryAsync', 'Process', 'Create'])); + }); + + it('extracts property names', () => { + const content = `public class Service { + public string Name { get; set; } + private int _count; + public bool IsActive { get; } +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['Name', '_count', 'IsActive'])); + }); + }); + + describe('PHP', () => { + const patterns = PATTERNS['php']!; + + it('extracts method names', () => { + const content = `class UserService { + public function find($id) { } + private function validate($data) { } + public static function create(array $data) { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['find', 'validate', 'create'])); + }); + + it('extracts property names', () => { + const content = `class UserService { + private string $name; + public int $age; + protected array $data = []; +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'age', 'data'])); + }); + }); + + describe('Ruby', () => { + const patterns = PATTERNS['ruby']!; + + it('extracts method names', () => { + const content = `class User + def initialize(name) + @name = name + end + + def full_name + @name + end + + def self.find(id) + end +end`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['initialize', 'full_name', 'find'])); + }); + + it('extracts property names', () => { + const content = `class User + attr_accessor :name, :email + attr_reader :id + attr_writer :status + + def initialize(name) + @name = name + @secret = "hidden" + end +end`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'email', 'id', 'status'])); + }); + }); + + describe('Swift', () => { + const patterns = PATTERNS['swift']!; + + it('extracts method names', () => { + const content = `class UserService { + func fetch() async throws -> [User] { } + private func validate(_ input: String) -> Bool { } + static func create() -> UserService { } + class func shared() -> Self { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['fetch', 'validate', 'create', 'shared'])); + }); + + it('extracts property names', () => { + const content = `class UserService { + var name: String + let id: UUID + private var _cache: [String: Any] +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'id', '_cache'])); + }); + }); + + describe('C/C++', () => { + const patterns = PATTERNS['cpp']!; + + it('extracts method names from class body', () => { + const content = `class Engine { +public: + void start(); + virtual void stop(); + static Engine* create(); +private: + bool validate(); +};`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['start', 'stop', 'create', 'validate'])); + }); + + it('extracts member variables', () => { + const content = `class Engine { + std::string name; + int power; + bool active; +};`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'power', 'active'])); + }); + }); + + describe('Dart', () => { + const patterns = PATTERNS['dart']!; + + it('extracts method names', () => { + const content = `class Repository { + Future save(User user) async { } + User? findById(int id) { } + static Repository create() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['save', 'findById', 'create'])); + }); + + it('extracts property names', () => { + const content = `class Repository { + final String name; + int? count; + late List users; +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', 'count', 'users'])); + }); + }); + + describe('JavaScript', () => { + const patterns = PATTERNS['javascript']!; + + it('extracts method names', () => { + const content = `class Handler { + async process(event) { } + validate(input) { } + static create() { } +}`; + const methods = patterns.extractMethods(content); + expect(methods).toEqual(expect.arrayContaining(['process', 'validate', 'create'])); + }); + + it('extracts property names from constructor assignments', () => { + const content = `class Handler { + constructor() { + this.name = ""; + this._cache = new Map(); + this.count = 0; + } +}`; + const props = patterns.extractProperties(content); + expect(props).toEqual(expect.arrayContaining(['name', '_cache', 'count'])); + }); + }); +});