feat(embeddings): add language-patterns module for multi-language class text

This commit is contained in:
wangjichao 2026-04-14 09:46:26 +08:00
parent 844529f999
commit bd4d15c460
2 changed files with 744 additions and 0 deletions

View file

@ -0,0 +1,392 @@
export interface LanguagePatterns {
extractMethods(content: string): string[];
extractProperties(content: string): string[];
}
function cStyleMethods(modifiers: string): (content: string) => string[] {
const re = new RegExp(
`(?:${modifiers})\\s+` +
`(?:<[^>]+>\\s+)?` +
`(?:\\w+(?:<[^>]+>)?(?:\\[\\])?\\s+)?` +
`(\\w+)\\s*\\(`,
'gm',
);
return (content: string): string[] => {
re.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
out.push(m[1]);
}
return out;
};
}
function cStyleProperties(typePattern: string): (content: string) => string[] {
const re = new RegExp(
`(?:private|protected|public)?\\s*(?:static)?\\s*(?:final)?\\s*(?:readonly)?\\s*` +
typePattern +
`\\s+(\\w+)\\s*(?:[;=]|\\{)`,
'gm',
);
return (content: string): string[] => {
re.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
out.push(m[1]);
}
return out;
};
}
const tsMethodRe =
/(?:(?:async|static|public|private|protected|readonly|abstract|override)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm;
const tsPropertyRe =
/(?:private|public|protected|readonly|static|abstract|override)\s+(\w+)\s*[;=:]/gm;
function extractTsMethods(content: string): string[] {
tsMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = tsMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractTsProperties(content: string): string[] {
tsPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = tsPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const pythonMethodRe = /def\s+(\w+)\s*\(/gm;
const pythonPropertyRe = /self\.(\w+)\s*=/gm;
function extractPythonMethods(content: string): string[] {
pythonMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = pythonMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractPythonProperties(content: string): string[] {
pythonPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = pythonPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const kotlinMethodRe =
/(?:(?:public|private|protected|internal|suspend|inline|open|override)\s+)*fun\s+(\w+)\s*[(<]/gm;
const kotlinPropertyRe =
/(?:(?:public|private|protected|internal|lateinit)\s+)*(?:val|var)\s+(\w+)/gm;
function extractKotlinMethods(content: string): string[] {
kotlinMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = kotlinMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractKotlinProperties(content: string): string[] {
kotlinPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = kotlinPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const goPropertyRe = /^\s+(\w+)\s+\w+/gm;
function extractGoProperties(content: string): string[] {
goPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = goPropertyRe.exec(content)) !== null) {
const name = m[1];
if (name[0] === name[0].toUpperCase()) {
out.push(name);
}
}
return out;
}
const rustMethodRe = /(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*[(<]/gm;
const rustPropertyRe = /^\s+(\w+)\s*:/gm;
function extractRustMethods(content: string): string[] {
rustMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = rustMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractRustProperties(content: string): string[] {
rustPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = rustPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const csharpPropertyRe =
/(?:(?:public|private|protected|internal|static|readonly|virtual|override|sealed|abstract)\s+)+(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)(\w+)\s*[;{]/gm;
function extractCsharpProperties(content: string): string[] {
csharpPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = csharpPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const phpMethodRe = /(?:public|private|protected)\s+(?:static\s+)?function\s+(\w+)\s*\(/gm;
const phpPropertyRe = /(?:public|private|protected)\s+(?:\w+\s+)?\$(\w+)/gm;
function extractPhpMethods(content: string): string[] {
phpMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = phpMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractPhpProperties(content: string): string[] {
phpPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = phpPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const rubyMethodRe = /def\s+(?:self\.)?(\w+)/gm;
const rubyAttrRe = /:(\w+)/g;
function extractRubyMethods(content: string): string[] {
rubyMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = rubyMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractRubyProperties(content: string): string[] {
const out: string[] = [];
const lineRe = /attr_(?:accessor|reader|writer)\s+(.+)/gm;
lineRe.lastIndex = 0;
let line: RegExpExecArray | null;
while ((line = lineRe.exec(content)) !== null) {
const args = line[1];
rubyAttrRe.lastIndex = 0;
let sym: RegExpExecArray | null;
while ((sym = rubyAttrRe.exec(args)) !== null) {
out.push(sym[1]);
}
}
return out;
}
const swiftMethodRe =
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*func\s+(\w+)\s*[(<]/gm;
const swiftPropertyRe =
/(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:var|let)\s+(\w+)\s*[:=]/gm;
function extractSwiftMethods(content: string): string[] {
swiftMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = swiftMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractSwiftProperties(content: string): string[] {
swiftPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = swiftPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const cppMethodRe =
/(?:(?:virtual|static|inline|explicit|constexpr|const)\s+)*(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+)\s*)(\w+)\s*\(/gm;
const cppPropertyRe =
/(?:(?:public|private|protected)\s*:\s*)?(?:(?:static|const|mutable|volatile)\s+)*(?:std::)?(?:\w+(?:<[^>]+>)?(?:\s*\*\s*|\s*&\s*|\s+))(\w+)\s*[;=]/gm;
function extractCppMethods(content: string): string[] {
cppMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = cppMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractCppProperties(content: string): string[] {
cppPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = cppPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
const dartMethodRe =
/(?:(?:static|async|factory)\s+)*(?:\w+(?:<[^>]+>)?(?:\?)?(?:\s*\(\))?\s+)?(\w+)\s*\(/gm;
const dartPropertyRe =
/(?:(?:final|late|static|const)\s+)*(?:\w+(?:<[^>]+>)?)?\??\s+(?:<[^>]+>\s+)?(\w+)\s*[;=]/gm;
function extractDartMethods(content: string): string[] {
dartMethodRe.lastIndex = 0;
const out: string[] = [];
const seen = new Set<string>();
let m: RegExpExecArray | null;
while ((m = dartMethodRe.exec(content)) !== null) {
const name = m[1];
if (!seen.has(name)) {
seen.add(name);
out.push(name);
}
}
return out;
}
function extractDartProperties(content: string): string[] {
dartPropertyRe.lastIndex = 0;
const out: string[] = [];
const seen = new Set<string>();
let m: RegExpExecArray | null;
while ((m = dartPropertyRe.exec(content)) !== null) {
const name = m[1];
if (name === 'class' || name === 'void' || name === 'async') continue;
if (!seen.has(name)) {
seen.add(name);
out.push(name);
}
}
return out;
}
const jsMethodRe = /(?:(?:async|static|get|set)\s+)*(?:\w+(?:<[^>]+>)?(?:\[\])?\s+)?(\w+)\s*\(/gm;
const jsPropertyRe = /this\.(\w+)\s*=/gm;
function extractJsMethods(content: string): string[] {
jsMethodRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = jsMethodRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
function extractJsProperties(content: string): string[] {
jsPropertyRe.lastIndex = 0;
const out: string[] = [];
let m: RegExpExecArray | null;
while ((m = jsPropertyRe.exec(content)) !== null) {
out.push(m[1]);
}
return out;
}
export const PATTERNS: Record<string, LanguagePatterns> = {
typescript: {
extractMethods: extractTsMethods,
extractProperties: extractTsProperties,
},
javascript: {
extractMethods: extractJsMethods,
extractProperties: extractJsProperties,
},
python: {
extractMethods: extractPythonMethods,
extractProperties: extractPythonProperties,
},
java: {
extractMethods: cStyleMethods('public|private|protected|static'),
extractProperties: cStyleProperties('\\w+(?:<[^>]+>)?(?:\\[\\])?'),
},
kotlin: {
extractMethods: extractKotlinMethods,
extractProperties: extractKotlinProperties,
},
go: {
extractMethods: () => [],
extractProperties: extractGoProperties,
},
rust: {
extractMethods: extractRustMethods,
extractProperties: extractRustProperties,
},
csharp: {
extractMethods: cStyleMethods(
'public|private|protected|internal|static|virtual|override|async',
),
extractProperties: extractCsharpProperties,
},
php: {
extractMethods: extractPhpMethods,
extractProperties: extractPhpProperties,
},
ruby: {
extractMethods: extractRubyMethods,
extractProperties: extractRubyProperties,
},
swift: {
extractMethods: extractSwiftMethods,
extractProperties: extractSwiftProperties,
},
cpp: {
extractMethods: extractCppMethods,
extractProperties: extractCppProperties,
},
c: {
extractMethods: extractCppMethods,
extractProperties: extractCppProperties,
},
dart: {
extractMethods: extractDartMethods,
extractProperties: extractDartProperties,
},
};
export function getPatternsForLanguage(language: string): LanguagePatterns | undefined {
return PATTERNS[language];
}

View file

@ -0,0 +1,352 @@
import { describe, it, expect } from 'vitest';
import { PATTERNS, getPatternsForLanguage } from '../../src/core/embeddings/language-patterns.js';
describe('language-patterns', () => {
describe('getPatternsForLanguage', () => {
it('returns patterns for known languages', () => {
expect(getPatternsForLanguage('typescript')).toBeDefined();
expect(getPatternsForLanguage('python')).toBeDefined();
expect(getPatternsForLanguage('rust')).toBeDefined();
});
it('returns undefined for unknown language', () => {
expect(getPatternsForLanguage('brainfuck')).toBeUndefined();
});
});
describe('TypeScript/JavaScript', () => {
const patterns = PATTERNS['typescript']!;
it('extracts method names', () => {
const content = `class Foo {
async getData(): Promise<T> { }
handleClick(event: Event) { }
private _internal() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['getData', 'handleClick', '_internal']));
});
it('extracts property names', () => {
const content = `class Foo {
private name: string;
public age: number;
readonly id: string;
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'age', 'id']));
});
});
describe('Python', () => {
const patterns = PATTERNS['python']!;
it('extracts method names', () => {
const content = `class User:
def __init__(self, name):
self.name = name
def get_full_name(self):
return self.name
@staticmethod
def from_dict(data):
return User(data["name"])
`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['__init__', 'get_full_name', 'from_dict']));
});
it('extracts property names', () => {
const content = `class User:
def __init__(self, name):
self.name = name
self._email = ""
self.age = 0
`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', '_email', 'age']));
});
});
describe('Java', () => {
const patterns = PATTERNS['java']!;
it('extracts method names', () => {
const content = `public class Service {
public String getName() { }
private void process(Data d) { }
static Builder builder() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['getName', 'process', 'builder']));
});
it('extracts property names', () => {
const content = `public class Service {
private String name;
protected int count;
public static final int MAX = 100;
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'count']));
});
});
describe('Kotlin', () => {
const patterns = PATTERNS['kotlin']!;
it('extracts method names', () => {
const content = `class User(val name: String) {
fun greet(): String = "Hello"
suspend fun fetchData() { }
private fun validate() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['greet', 'fetchData', 'validate']));
});
it('extracts property names', () => {
const content = `class User {
val name: String
var age: Int = 0
private val _id: UUID
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'age', '_id']));
});
});
describe('Go', () => {
const patterns = PATTERNS['go']!;
it('extracts no methods (Go methods are external to struct)', () => {
const content = `type User struct {
Name string
Age int
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual([]);
});
it('extracts struct fields', () => {
const content = `type User struct {
Name string
Age int
json string
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['Name', 'Age']));
});
});
describe('Rust', () => {
const patterns = PATTERNS['rust']!;
it('extracts method names from impl block', () => {
const content = `impl User {
pub fn new(name: &str) -> Self { }
fn validate(&self) -> bool { }
pub async fn fetch(&self) -> Result<Data> { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['new', 'validate', 'fetch']));
});
it('extracts struct fields', () => {
const content = `struct User {
name: String,
age: u32,
active: bool,
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'age', 'active']));
});
});
describe('C#', () => {
const patterns = PATTERNS['csharp']!;
it('extracts method names', () => {
const content = `public class Service {
public async Task<T> QueryAsync() { }
private void Process() { }
public static Service Create() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['QueryAsync', 'Process', 'Create']));
});
it('extracts property names', () => {
const content = `public class Service {
public string Name { get; set; }
private int _count;
public bool IsActive { get; }
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['Name', '_count', 'IsActive']));
});
});
describe('PHP', () => {
const patterns = PATTERNS['php']!;
it('extracts method names', () => {
const content = `class UserService {
public function find($id) { }
private function validate($data) { }
public static function create(array $data) { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['find', 'validate', 'create']));
});
it('extracts property names', () => {
const content = `class UserService {
private string $name;
public int $age;
protected array $data = [];
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'age', 'data']));
});
});
describe('Ruby', () => {
const patterns = PATTERNS['ruby']!;
it('extracts method names', () => {
const content = `class User
def initialize(name)
@name = name
end
def full_name
@name
end
def self.find(id)
end
end`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['initialize', 'full_name', 'find']));
});
it('extracts property names', () => {
const content = `class User
attr_accessor :name, :email
attr_reader :id
attr_writer :status
def initialize(name)
@name = name
@secret = "hidden"
end
end`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'email', 'id', 'status']));
});
});
describe('Swift', () => {
const patterns = PATTERNS['swift']!;
it('extracts method names', () => {
const content = `class UserService {
func fetch() async throws -> [User] { }
private func validate(_ input: String) -> Bool { }
static func create() -> UserService { }
class func shared() -> Self { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['fetch', 'validate', 'create', 'shared']));
});
it('extracts property names', () => {
const content = `class UserService {
var name: String
let id: UUID
private var _cache: [String: Any]
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'id', '_cache']));
});
});
describe('C/C++', () => {
const patterns = PATTERNS['cpp']!;
it('extracts method names from class body', () => {
const content = `class Engine {
public:
void start();
virtual void stop();
static Engine* create();
private:
bool validate();
};`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['start', 'stop', 'create', 'validate']));
});
it('extracts member variables', () => {
const content = `class Engine {
std::string name;
int power;
bool active;
};`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'power', 'active']));
});
});
describe('Dart', () => {
const patterns = PATTERNS['dart']!;
it('extracts method names', () => {
const content = `class Repository {
Future<void> save(User user) async { }
User? findById(int id) { }
static Repository create() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['save', 'findById', 'create']));
});
it('extracts property names', () => {
const content = `class Repository {
final String name;
int? count;
late List<User> users;
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', 'count', 'users']));
});
});
describe('JavaScript', () => {
const patterns = PATTERNS['javascript']!;
it('extracts method names', () => {
const content = `class Handler {
async process(event) { }
validate(input) { }
static create() { }
}`;
const methods = patterns.extractMethods(content);
expect(methods).toEqual(expect.arrayContaining(['process', 'validate', 'create']));
});
it('extracts property names from constructor assignments', () => {
const content = `class Handler {
constructor() {
this.name = "";
this._cache = new Map();
this.count = 0;
}
}`;
const props = patterns.extractProperties(content);
expect(props).toEqual(expect.arrayContaining(['name', '_cache', 'count']));
});
});
});