mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-30 01:51:20 +00:00
feat(group): extract thrift idl contracts
This commit is contained in:
liyipeng06
parent
b9ceb08cc1
commit
544a852dd9
2 changed files with 534 additions and 0 deletions
279
gitnexus/src/core/group/extractors/thrift-extractor.ts
Normal file
279
gitnexus/src/core/group/extractors/thrift-extractor.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
import { glob } from 'glob';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
|
||||
export interface ThriftServiceInfo {
|
||||
namespace: string;
|
||||
serviceName: string;
|
||||
methods: string[];
|
||||
thriftPath: string;
|
||||
}
|
||||
|
||||
export interface ThriftContext {
|
||||
namespacesByThrift: Map<string, string>;
|
||||
servicesByName: Map<string, ThriftServiceInfo[]>;
|
||||
}
|
||||
|
||||
function normalizeThriftPath(rel: string): string {
|
||||
return rel.replace(/\\/g, '/');
|
||||
}
|
||||
|
||||
export function thriftMethodContractId(
|
||||
namespace: string,
|
||||
serviceName: string,
|
||||
methodName: string,
|
||||
): string {
|
||||
const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
|
||||
return `thrift::${prefix}/${methodName}`;
|
||||
}
|
||||
|
||||
export function thriftServiceContractId(namespace: string, serviceName: string): string {
|
||||
const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
|
||||
return `thrift::${prefix}/*`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace Thrift comments and string literals with spaces while preserving
|
||||
* newlines and character offsets. Service block scanning can then count braces
|
||||
* without being confused by examples or comments inside the IDL.
|
||||
*/
|
||||
function stripThriftCommentsAndStrings(content: string): string {
|
||||
const out = new Array<string>(content.length);
|
||||
let i = 0;
|
||||
|
||||
while (i < content.length) {
|
||||
const ch = content[i];
|
||||
const next = content[i + 1];
|
||||
|
||||
if (ch === '/' && next === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length && content[i] !== '\n') {
|
||||
out[i] = content[i] === '\r' ? '\r' : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '#') {
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
while (i < content.length && content[i] !== '\n') {
|
||||
out[i] = content[i] === '\r' ? '\r' : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '/' && next === '*') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length) {
|
||||
if (content[i] === '*' && content[i + 1] === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ch === '"' || ch === "'") {
|
||||
const quote = ch;
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
while (i < content.length) {
|
||||
const c = content[i];
|
||||
if (c === '\\' && i + 1 < content.length) {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === quote) {
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
out[i] = c === '\n' || c === '\r' ? c : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out[i] = ch;
|
||||
i++;
|
||||
}
|
||||
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
function extractNamespace(sanitizedContent: string): string {
|
||||
const namespaces: Array<{ language: string; namespace: string }> = [];
|
||||
const namespaceRe = /^\s*namespace\s+([A-Za-z_*][\w.*-]*)\s+([A-Za-z_][\w.]*)\s*$/gm;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = namespaceRe.exec(sanitizedContent)) !== null) {
|
||||
namespaces.push({ language: match[1], namespace: match[2] });
|
||||
}
|
||||
|
||||
return (
|
||||
namespaces.find((entry) => entry.language === 'java')?.namespace ??
|
||||
namespaces[0]?.namespace ??
|
||||
''
|
||||
);
|
||||
}
|
||||
|
||||
function extractServiceBlocks(sanitizedContent: string): Array<{ name: string; body: string }> {
|
||||
const results: Array<{ name: string; body: string }> = [];
|
||||
const headerRe = /service\s+([A-Za-z_]\w*)\s*(?:extends\s+[A-Za-z_][\w.]*)?\s*\{/g;
|
||||
let headerMatch: RegExpExecArray | null;
|
||||
|
||||
while ((headerMatch = headerRe.exec(sanitizedContent)) !== null) {
|
||||
const serviceName = headerMatch[1];
|
||||
const bodyStart = headerMatch.index + headerMatch[0].length;
|
||||
let depth = 1;
|
||||
let pos = bodyStart;
|
||||
|
||||
while (pos < sanitizedContent.length && depth > 0) {
|
||||
const ch = sanitizedContent[pos];
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
pos++;
|
||||
}
|
||||
|
||||
if (depth !== 0) continue;
|
||||
|
||||
results.push({
|
||||
name: serviceName,
|
||||
body: sanitizedContent.slice(bodyStart, pos - 1),
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function extractMethods(sanitizedServiceBody: string): string[] {
|
||||
const methods: string[] = [];
|
||||
const methodRe =
|
||||
/(?:^|[;,\n\r])\s*(?:oneway\s+)?[A-Za-z_][\w.]*(?:\s*<[^(){};]*>)?\s+([A-Za-z_]\w*)\s*\(/g;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = methodRe.exec(sanitizedServiceBody)) !== null) {
|
||||
methods.push(match[1]);
|
||||
}
|
||||
|
||||
return methods;
|
||||
}
|
||||
|
||||
function makeContract(
|
||||
cid: string,
|
||||
filePath: string,
|
||||
symbolName: string,
|
||||
meta: Record<string, unknown>,
|
||||
): ExtractedContract {
|
||||
return {
|
||||
contractId: cid,
|
||||
type: 'thrift',
|
||||
role: 'provider',
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath: normalizeThriftPath(filePath), name: symbolName },
|
||||
symbolName,
|
||||
confidence: 0.85,
|
||||
meta: { ...meta, extractionStrategy: 'source_scan' },
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildThriftContext(repoPath: string): Promise<ThriftContext> {
|
||||
const thriftFiles = await glob('**/*.thrift', {
|
||||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
});
|
||||
const namespacesByThrift = new Map<string, string>();
|
||||
const servicesByName = new Map<string, ThriftServiceInfo[]>();
|
||||
|
||||
for (const rel of thriftFiles) {
|
||||
const thriftPath = normalizeThriftPath(rel);
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
|
||||
const sanitized = stripThriftCommentsAndStrings(content);
|
||||
const namespace = extractNamespace(sanitized);
|
||||
namespacesByThrift.set(thriftPath, namespace);
|
||||
|
||||
for (const block of extractServiceBlocks(sanitized)) {
|
||||
const methods = extractMethods(block.body);
|
||||
const info: ThriftServiceInfo = {
|
||||
namespace,
|
||||
serviceName: block.name,
|
||||
methods,
|
||||
thriftPath,
|
||||
};
|
||||
const existing = servicesByName.get(block.name) ?? [];
|
||||
existing.push(info);
|
||||
servicesByName.set(block.name, existing);
|
||||
}
|
||||
}
|
||||
|
||||
return { namespacesByThrift, servicesByName };
|
||||
}
|
||||
|
||||
export class ThriftExtractor implements ContractExtractor {
|
||||
type = 'thrift' as const;
|
||||
|
||||
async canExtract(_repo: RepoHandle): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
|
||||
async extract(
|
||||
_dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const out: ExtractedContract[] = [];
|
||||
const context = await buildThriftContext(repoPath);
|
||||
|
||||
for (const infos of context.servicesByName.values()) {
|
||||
for (const info of infos) {
|
||||
for (const methodName of info.methods) {
|
||||
const symbolName = `${info.serviceName}.${methodName}`;
|
||||
out.push(
|
||||
makeContract(
|
||||
thriftMethodContractId(info.namespace, info.serviceName, methodName),
|
||||
info.thriftPath,
|
||||
symbolName,
|
||||
{
|
||||
namespace: info.namespace,
|
||||
service: info.serviceName,
|
||||
method: methodName,
|
||||
source: 'thrift_idl',
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this.dedupe(out);
|
||||
}
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const byKey = new Map<string, ExtractedContract>();
|
||||
for (const c of items) {
|
||||
const key = `${c.contractId}|${c.role}|${c.symbolRef.filePath}|${c.symbolName}`;
|
||||
const existing = byKey.get(key);
|
||||
if (!existing || c.confidence > existing.confidence) {
|
||||
byKey.set(key, c);
|
||||
}
|
||||
}
|
||||
return Array.from(byKey.values());
|
||||
}
|
||||
}
|
||||
255
gitnexus/test/unit/group/thrift-extractor.test.ts
Normal file
255
gitnexus/test/unit/group/thrift-extractor.test.ts
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import fsp from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import {
|
||||
ThriftExtractor,
|
||||
buildThriftContext,
|
||||
thriftMethodContractId,
|
||||
thriftServiceContractId,
|
||||
} from '../../../src/core/group/extractors/thrift-extractor.js';
|
||||
import type { RepoHandle } from '../../../src/core/group/types.js';
|
||||
|
||||
describe('ThriftExtractor', () => {
|
||||
let tmpDir: string;
|
||||
let extractor: ThriftExtractor;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'gitnexus-thrift-'));
|
||||
extractor = new ThriftExtractor();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function writeFile(relPath: string, content: string): void {
|
||||
const full = path.join(tmpDir, relPath);
|
||||
fs.mkdirSync(path.dirname(full), { recursive: true });
|
||||
fs.writeFileSync(full, content);
|
||||
}
|
||||
|
||||
const makeRepo = (repoPath: string): RepoHandle => ({
|
||||
id: 'test-repo',
|
||||
path: 'test/app',
|
||||
repoPath,
|
||||
storagePath: path.join(repoPath, '.gitnexus'),
|
||||
});
|
||||
|
||||
it('test_extract_thrift_single_method_returns_idl_provider', async () => {
|
||||
writeFile(
|
||||
'idl/order.thrift',
|
||||
`namespace java billing.v1
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts).toHaveLength(1);
|
||||
expect(contracts[0]).toMatchObject({
|
||||
contractId: 'thrift::billing.v1.OrderService/PlaceOrder',
|
||||
type: 'thrift',
|
||||
role: 'provider',
|
||||
symbolName: 'OrderService.PlaceOrder',
|
||||
confidence: 0.85,
|
||||
meta: {
|
||||
namespace: 'billing.v1',
|
||||
service: 'OrderService',
|
||||
method: 'PlaceOrder',
|
||||
source: 'thrift_idl',
|
||||
},
|
||||
});
|
||||
expect(contracts[0].symbolRef).toEqual({
|
||||
filePath: 'idl/order.thrift',
|
||||
name: 'OrderService.PlaceOrder',
|
||||
});
|
||||
});
|
||||
|
||||
it('test_extract_thrift_multiple_services_and_methods_returns_all', async () => {
|
||||
writeFile(
|
||||
'contracts/orders.thrift',
|
||||
`namespace java billing.v1
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
OrderStatus GetOrderStatus(1: string orderId)
|
||||
}
|
||||
|
||||
service InvoiceService {
|
||||
Invoice CreateInvoice(1: string orderId)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts.map((c) => c.contractId).sort()).toEqual([
|
||||
'thrift::billing.v1.InvoiceService/CreateInvoice',
|
||||
'thrift::billing.v1.OrderService/GetOrderStatus',
|
||||
'thrift::billing.v1.OrderService/PlaceOrder',
|
||||
]);
|
||||
});
|
||||
|
||||
it('test_extract_thrift_prefers_java_namespace_over_other_namespaces', async () => {
|
||||
writeFile(
|
||||
'order.thrift',
|
||||
`namespace py billing_python.v1
|
||||
namespace java billing.v1
|
||||
namespace go billinggo
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts[0].contractId).toBe('thrift::billing.v1.OrderService/PlaceOrder');
|
||||
expect(contracts[0].meta.namespace).toBe('billing.v1');
|
||||
});
|
||||
|
||||
it('test_extract_thrift_uses_first_non_java_namespace_when_java_missing', async () => {
|
||||
writeFile(
|
||||
'order.thrift',
|
||||
`namespace py billing_python.v1
|
||||
namespace go billinggo
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts[0].contractId).toBe('thrift::billing_python.v1.OrderService/PlaceOrder');
|
||||
expect(contracts[0].meta.namespace).toBe('billing_python.v1');
|
||||
});
|
||||
|
||||
it('test_extract_thrift_without_namespace_uses_service_only', async () => {
|
||||
writeFile(
|
||||
'order.thrift',
|
||||
`service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts[0].contractId).toBe('thrift::OrderService/PlaceOrder');
|
||||
expect(contracts[0].meta.namespace).toBe('');
|
||||
});
|
||||
|
||||
it('test_extract_thrift_ignores_braces_inside_comments_and_strings', async () => {
|
||||
writeFile(
|
||||
'idl/tricky.thrift',
|
||||
`namespace java billing.v1
|
||||
|
||||
service OrderService {
|
||||
// A comment with } should not close the service.
|
||||
/* A block comment with { and } should not affect depth. */
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
const string NOTE = "literal with } and { braces"
|
||||
OrderStatus GetOrderStatus(1: string orderId)
|
||||
}`,
|
||||
);
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts.map((c) => c.symbolName).sort()).toEqual([
|
||||
'OrderService.GetOrderStatus',
|
||||
'OrderService.PlaceOrder',
|
||||
]);
|
||||
});
|
||||
|
||||
it('test_extract_thrift_malformed_unclosed_service_is_skipped', async () => {
|
||||
writeFile(
|
||||
'idl/broken.thrift',
|
||||
`namespace java billing.v1
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
`,
|
||||
);
|
||||
|
||||
await expect(extractor.extract(null, tmpDir, makeRepo(tmpDir))).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('test_extract_repo_without_thrift_returns_empty', async () => {
|
||||
writeFile('src/index.ts', 'console.log("hello")');
|
||||
|
||||
const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir));
|
||||
|
||||
expect(contracts).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildThriftContext', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'gitnexus-thrift-context-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('test_buildThriftContext_parses_namespace_service_methods_and_path', async () => {
|
||||
await fsp.mkdir(path.join(tmpDir, 'idl'), { recursive: true });
|
||||
await fsp.writeFile(
|
||||
path.join(tmpDir, 'idl', 'order.thrift'),
|
||||
`namespace java billing.v1
|
||||
|
||||
service OrderService {
|
||||
PlaceOrderResponse PlaceOrder(1: PlaceOrderRequest request)
|
||||
OrderStatus GetOrderStatus(1: string orderId)
|
||||
}`,
|
||||
);
|
||||
|
||||
const context = await buildThriftContext(tmpDir);
|
||||
|
||||
expect(context.namespacesByThrift.get('idl/order.thrift')).toBe('billing.v1');
|
||||
expect(context.servicesByName.get('OrderService')).toEqual([
|
||||
{
|
||||
namespace: 'billing.v1',
|
||||
serviceName: 'OrderService',
|
||||
methods: ['PlaceOrder', 'GetOrderStatus'],
|
||||
thriftPath: 'idl/order.thrift',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('test_buildThriftContext_without_files_returns_empty_maps', async () => {
|
||||
const context = await buildThriftContext(tmpDir);
|
||||
|
||||
expect(context.namespacesByThrift.size).toBe(0);
|
||||
expect(context.servicesByName.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Thrift contract id helpers', () => {
|
||||
it('test_thriftMethodContractId_with_namespace', () => {
|
||||
expect(thriftMethodContractId('billing.v1', 'OrderService', 'PlaceOrder')).toBe(
|
||||
'thrift::billing.v1.OrderService/PlaceOrder',
|
||||
);
|
||||
});
|
||||
|
||||
it('test_thriftMethodContractId_without_namespace', () => {
|
||||
expect(thriftMethodContractId('', 'OrderService', 'PlaceOrder')).toBe(
|
||||
'thrift::OrderService/PlaceOrder',
|
||||
);
|
||||
});
|
||||
|
||||
it('test_thriftServiceContractId_with_namespace', () => {
|
||||
expect(thriftServiceContractId('billing.v1', 'OrderService')).toBe(
|
||||
'thrift::billing.v1.OrderService/*',
|
||||
);
|
||||
});
|
||||
|
||||
it('test_thriftServiceContractId_without_namespace', () => {
|
||||
expect(thriftServiceContractId('', 'OrderService')).toBe('thrift::OrderService/*');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue