From 54d681a434e78242eae07571561a4345a0a3fc9b Mon Sep 17 00:00:00 2001 From: ivkond Date: Thu, 9 Apr 2026 13:22:25 +0300 Subject: [PATCH] feat(group): expand grpc extraction and proto resolution coverage --- .../2026-03-31-cross-index-impact-design.md | 6 +- .../core/group/extractors/grpc-extractor.ts | 205 +++++++++++++++--- .../test/unit/group/grpc-extractor.test.ts | 193 +++++++++++++++++ 3 files changed, 368 insertions(+), 36 deletions(-) diff --git a/docs/specs/2026-03-31-cross-index-impact-design.md b/docs/specs/2026-03-31-cross-index-impact-design.md index 61be405c4..e90a5c43c 100644 --- a/docs/specs/2026-03-31-cross-index-impact-design.md +++ b/docs/specs/2026-03-31-cross-index-impact-design.md @@ -363,11 +363,13 @@ Note: `responseKeys`/`accessedKeys` from the original design require response sh `.proto` files are not a supported language in GitNexus — no symbols are extracted during indexing. This extractor is **source-scan only**. - Scan for `.proto` files in repo directory -- Parse `service` and `rpc` declarations with regex -- For consumers: scan source files for generated stub/client class usage patterns +- Parse `service` and `rpc` declarations with regex, inheriting package context through imported proto files when definitions are split +- For consumers: scan source files for generated stub/client class usage patterns (`@GrpcClient`, `ClientGrpc.getService(...)`, `new XxxServiceClient(...)`, `grpc.loadPackageDefinition(...)`-based construction) In MVP, `canExtract` returns `true` only when `.proto` files exist in the repo's file tree. Extraction confidence is lower (0.7) due to regex-only parsing. +Explicitly unsupported in this extractor today: `C#`, `Ruby`, and `Rust` gRPC client/server ecosystems. Those require follow-up extractor coverage and should not be treated as implicitly supported by the TypeScript/Go/Java/Python patterns above. + **Contract ID:** `grpc::{package}.{Service}/{Method}` ### MessageTopicExtractor diff --git a/gitnexus/src/core/group/extractors/grpc-extractor.ts b/gitnexus/src/core/group/extractors/grpc-extractor.ts index 42fb9c852..c5fe11167 100644 --- a/gitnexus/src/core/group/extractors/grpc-extractor.ts +++ b/gitnexus/src/core/group/extractors/grpc-extractor.ts @@ -82,21 +82,100 @@ export interface ProtoServiceInfo { protoPath: string; } -export async function buildProtoMap(repoPath: string): Promise> { - const map = new Map(); +function normalizeProtoPath(rel: string): string { + return rel.replace(/\\/g, '/'); +} + +function extractProtoImports(content: string): string[] { + const imports: string[] = []; + const re = /^\s*import\s+"([^"]+)"\s*;/gm; + let match: RegExpExecArray | null; + while ((match = re.exec(content)) !== null) { + imports.push(match[1]); + } + return imports; +} + +function longestSharedSegmentRun(aPath: string, bPath: string): number { + const a = aPath.split('/').filter(Boolean); + const b = bPath.split('/').filter(Boolean); + let best = 0; + + for (let i = 0; i < a.length; i++) { + for (let j = 0; j < b.length; j++) { + let run = 0; + while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) { + run++; + } + if (run > best) best = run; + } + } + + return best; +} + +async function buildProtoContext(repoPath: string): Promise<{ + packagesByProto: Map; + servicesByName: Map; +}> { + const servicesByName = new Map(); const protoFiles = await glob('**/*.proto', { cwd: repoPath, absolute: false, nodir: true, ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'], }); + const contents = new Map(); for (const rel of protoFiles) { const content = readSafe(repoPath, rel); if (!content) continue; + contents.set(normalizeProtoPath(rel), content); + } + const packagesByProto = new Map(); + + const resolvePackage = (protoPath: string, seen = new Set()): string => { + if (packagesByProto.has(protoPath)) return packagesByProto.get(protoPath) ?? ''; + if (seen.has(protoPath)) return ''; + + const content = contents.get(protoPath); + if (!content) return ''; + + seen.add(protoPath); const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m); - const pkg = pkgMatch?.[1] ?? ''; + if (pkgMatch?.[1]) { + packagesByProto.set(protoPath, pkgMatch[1]); + return pkgMatch[1]; + } + + for (const importPath of extractProtoImports(content)) { + const normalizedImport = normalizeProtoPath(importPath); + const candidates = [ + normalizeProtoPath( + path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport)), + ), + normalizedImport, + ]; + for (const candidate of candidates) { + if (!contents.has(candidate)) continue; + const inheritedPackage = resolvePackage(candidate, seen); + if (inheritedPackage) { + packagesByProto.set(protoPath, inheritedPackage); + return inheritedPackage; + } + } + } + + packagesByProto.set(protoPath, ''); + return ''; + }; + + for (const rel of protoFiles) { + const normalizedRel = normalizeProtoPath(rel); + const content = contents.get(normalizedRel); + if (!content) continue; + const pkg = resolvePackage(normalizedRel); const serviceBlocks = extractServiceBlocks(content); for (const block of serviceBlocks) { @@ -110,14 +189,20 @@ export async function buildProtoMap(repoPath: string): Promise> { + const { servicesByName } = await buildProtoContext(repoPath); + return servicesByName; } export function resolveProtoConflict( @@ -128,20 +213,14 @@ export function resolveProtoConflict( if (candidates.length === 0) return null; if (candidates.length === 1) return candidates[0]; - // Directory proximity heuristic - const sourceDir = path.dirname(sourceFilePath).replace(/\\/g, '/'); + const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath)); let best = candidates[0]; - let bestScore = 0; + let bestScore = -1; for (const c of candidates) { - const protoDir = path.dirname(c.protoPath).replace(/\\/g, '/'); - let shared = 0; - const min = Math.min(sourceDir.length, protoDir.length); - for (let i = 0; i < min; i++) { - if (sourceDir[i] === protoDir[i]) shared++; - else break; - } - if (shared > bestScore) { - bestScore = shared; + const protoDir = normalizeProtoPath(path.dirname(c.protoPath)); + const sharedRun = longestSharedSegmentRun(sourceDir, protoDir); + if (sharedRun > bestScore) { + bestScore = sharedRun; best = c; } } @@ -166,6 +245,7 @@ export class GrpcExtractor implements ContractExtractor { _repo: RepoHandle, ): Promise { const out: ExtractedContract[] = []; + const protoContext = await buildProtoContext(repoPath); // Proto files — definitive provider source const protoFiles = await glob('**/*.proto', { @@ -175,11 +255,17 @@ export class GrpcExtractor implements ContractExtractor { }); for (const rel of protoFiles) { const content = readSafe(repoPath, rel); - if (content) out.push(...this.parseProtoFile(content, rel)); + if (content) { + out.push( + ...this.parseProtoFile( + content, + rel, + protoContext.packagesByProto.get(normalizeProtoPath(rel)) ?? '', + ), + ); + } } - - // Build proto map for source scanner resolution - const protoMap = await buildProtoMap(repoPath); + const protoMap = protoContext.servicesByName; // Source files — server/client detection const sourceFiles = await glob('**/*.{go,java,py,ts,tsx,js,jsx}', { @@ -203,18 +289,16 @@ export class GrpcExtractor implements ContractExtractor { out.push(...this.scanPythonConsumers(content, rel, protoMap)); } else if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) { out.push(...this.scanTsProviders(content, rel, protoMap)); + out.push(...this.scanTsConsumers(content, rel, protoMap)); } } return this.dedupe(out); } - private parseProtoFile(content: string, filePath: string): ExtractedContract[] { + private parseProtoFile(content: string, filePath: string, pkg: string): ExtractedContract[] { const out: ExtractedContract[] = []; - const pkgMatch = content.match(/^package\s+([\w.]+)\s*;/m); - const pkg = pkgMatch ? pkgMatch[1] : ''; - for (const { name: serviceName, body } of extractServiceBlocks(content)) { const rpcRe = /rpc\s+(\w+)\s*\(/g; let rpcMatch: RegExpExecArray | null; @@ -484,15 +568,68 @@ export class GrpcExtractor implements ContractExtractor { return out; } - private dedupe(items: ExtractedContract[]): ExtractedContract[] { - const seen = new Set(); + private scanTsConsumers( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; - for (const c of items) { - const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; - if (seen.has(k)) continue; - seen.add(k); - out.push(c); + const pushConsumer = ( + serviceName: string, + symbolName: string, + source: string, + confidenceWithProto = 0.75, + confidenceWithoutProto = 0.55, + ): void => { + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? confidenceWithProto : confidenceWithoutProto; + out.push( + makeContract(cid, 'consumer', filePath, symbolName, conf, { + service: serviceName, + source, + }), + ); + }; + + const getServiceRe = /\.getService(?:<[^>]+>)?\s*\(\s*['"](\w+)['"]\s*\)/g; + let match: RegExpExecArray | null; + while ((match = getServiceRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_client_grpc_get_service'); } + + const clientCtorRe = /new\s+(\w+)Client\s*\(/g; + while ((match = clientCtorRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_generated_client'); + } + + if (content.includes('loadPackageDefinition')) { + const packageCtorRe = /new\s+[\w$.]*\.([A-Z]\w+)\s*\(/g; + while ((match = packageCtorRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_load_package_definition'); + } + } + return out; } + + private dedupe(items: ExtractedContract[]): ExtractedContract[] { + const byKey = new Map(); + for (const c of items) { + const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; + const existing = byKey.get(k); + if ( + !existing || + c.confidence > existing.confidence || + (c.confidence === existing.confidence && + String(c.meta.source) < String(existing.meta.source)) + ) { + byKey.set(k, c); + } + } + return Array.from(byKey.values()); + } } diff --git a/gitnexus/test/unit/group/grpc-extractor.test.ts b/gitnexus/test/unit/group/grpc-extractor.test.ts index d38135ba4..421093e15 100644 --- a/gitnexus/test/unit/group/grpc-extractor.test.ts +++ b/gitnexus/test/unit/group/grpc-extractor.test.ts @@ -379,6 +379,146 @@ export class AuthController { expect(providers[0].contractId).toContain('Login'); expect(providers[0].confidence).toBe(0.8); }); + + it('test_extract_ts_grpc_client_decorator_and_getService_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import { ClientGrpc, GrpcClient } from '@nestjs/microservices'; + +export class AuthGateway { + @GrpcClient('AUTH_PACKAGE') + private readonly client!: ClientGrpc; + + onModuleInit(): void { + this.client.getService('AuthService'); + } +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_getService_without_decorator_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import type { ClientGrpc } from '@nestjs/microservices'; + +export function createAuthClient(client: ClientGrpc) { + return client.getService('AuthService'); +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_generated_client_constructor_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import { credentials } from '@grpc/grpc-js'; +import { AuthServiceClient } from './generated/auth'; + +export const authClient = new AuthServiceClient('localhost:50051', credentials.createInsecure());`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_loadPackageDefinition_constructor_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; + +const definition = protoLoader.loadSync('proto/auth.proto'); +const authProto = grpc.loadPackageDefinition(definition) as any; +export const authClient = new authProto.auth.v1.AuthService( + 'localhost:50051', + grpc.credentials.createInsecure(), +);`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_duplicate_consumer_patterns_in_one_file_dedupes_deterministically', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import * as grpc from '@grpc/grpc-js'; +import type { ClientGrpc } from '@nestjs/microservices'; +import { AuthServiceClient } from './generated/auth'; + +export class AuthGateway { + constructor(private readonly client: ClientGrpc) {} + + connect() { + this.client.getService('AuthService'); + return new AuthServiceClient('localhost:50051', grpc.credentials.createInsecure()); + } +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); }); describe('edge cases', () => { @@ -459,6 +599,25 @@ service Foo { rpc Bar (Req) returns (Res); }`; const map = await buildProtoMap(tmpDir); expect(map.get('Svc')).toHaveLength(2); }); + + it('test_buildProtoMap_imported_package_is_inherited_for_split_service_definition', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto', 'shared'), { recursive: true }); + await fsp.mkdir(path.join(tmpDir, 'proto', 'services'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'shared', 'package.proto'), + 'package auth.v1;\nmessage LoginRequest {}', + ); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'services', 'auth.proto'), + 'import "../shared/package.proto";\nservice AuthService { rpc Login (LoginRequest) returns (LoginRequest); }', + ); + + const map = await buildProtoMap(tmpDir); + const entries = map.get('AuthService')!; + + expect(entries).toHaveLength(1); + expect(entries[0].package).toBe('auth.v1'); + }); }); describe('resolveProtoConflict', () => { @@ -483,6 +642,15 @@ describe('resolveProtoConflict', () => { expect(result?.package).toBe('close'); }); + it('test_centralized_proto_layout_prefers_shared_path_segments_over_prefix_only', () => { + const candidates = [ + makeInfo('billing', 'proto/services/billing/svc.proto'), + makeInfo('auth', 'proto/services/auth/svc.proto'), + ]; + const result = resolveProtoConflict('Svc', 'services/auth/src/server.ts', candidates); + expect(result?.package).toBe('auth'); + }); + it('test_no_candidates_returns_null', () => { expect(resolveProtoConflict('Svc', 'src/main.go', [])).toBeNull(); }); @@ -636,4 +804,29 @@ stub = UserServiceStub(channel)`, expect(tsProvider!.contractId).toBe('grpc::com.example.UserService/GetUser'); expect(tsProvider!.confidence).toBe(0.8); }); + + it('test_proto_provider_inherits_package_from_imported_definition', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto', 'shared'), { recursive: true }); + await fsp.mkdir(path.join(tmpDir, 'proto', 'services'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'shared', 'package.proto'), + 'package auth.v1;\nmessage LoginRequest {}', + ); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'services', 'auth.proto'), + `syntax = "proto3"; +import "../shared/package.proto"; +service AuthService { + rpc Login (LoginRequest) returns (LoginRequest); +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const protoProvider = contracts.find( + (c) => c.symbolRef.filePath === 'proto/services/auth.proto', + ); + expect(protoProvider).toBeDefined(); + expect(protoProvider!.contractId).toBe('grpc::auth.v1.AuthService/Login'); + }); });