diff --git a/gitnexus/src/core/group/extractors/grpc-extractor.ts b/gitnexus/src/core/group/extractors/grpc-extractor.ts index b4cefadc5..6e52c0b74 100644 --- a/gitnexus/src/core/group/extractors/grpc-extractor.ts +++ b/gitnexus/src/core/group/extractors/grpc-extractor.ts @@ -25,20 +25,110 @@ function serviceOnlyContractId(serviceName: string): string { return `grpc::${serviceName}/*`; } +/** + * Replace all .proto comments and string literals with spaces, preserving the + * original length and character offsets of the input. This lets downstream + * regex / brace-depth parsers run on a "sanitized" copy without having to + * understand proto syntax, while any RegExp.exec/index-based lookups that + * were already positional against `content` continue to work against the + * original string. + * + * Supported comment forms: `// line comment`, `/* block comment * /`. + * Supported strings: double-quoted ("…") and single-quoted ('…') with `\` + * escape handling. Raw/unterminated strings are not supported — we stop + * on a line break for line-style comments and on EOF for unterminated + * strings/blocks, which matches how most real proto files parse. + */ +function stripProtoCommentsAndStrings(content: string): string { + const out = new Array(content.length); + let i = 0; + while (i < content.length) { + const ch = content[i]; + const next = content[i + 1]; + + // Line comment: // ... \n + if (ch === '/' && next === '/') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + while (i < content.length && content[i] !== '\n') { + out[i] = content[i] === '\r' ? '\r' : ' '; + i++; + } + continue; + } + + // Block comment: /* ... */ + if (ch === '/' && next === '*') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + while (i < content.length) { + if (content[i] === '*' && content[i + 1] === '/') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + break; + } + // Preserve newlines so line numbers stay stable for downstream code. + out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' '; + i++; + } + continue; + } + + // String literal: "..." or '...' + if (ch === '"' || ch === "'") { + const quote = ch; + out[i] = ' '; // replace opening quote + i++; + while (i < content.length) { + const c = content[i]; + if (c === '\\' && i + 1 < content.length) { + // Skip escaped pair (e.g. \" \n \\) + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + continue; + } + if (c === quote) { + out[i] = ' '; + i++; + break; + } + // Preserve newlines; proto technically disallows unescaped newlines + // inside strings, but real files occasionally have them. + out[i] = c === '\n' || c === '\r' ? c : ' '; + i++; + } + continue; + } + + out[i] = ch; + i++; + } + return out.join(''); +} + function extractServiceBlocks(content: string): Array<{ name: string; body: string }> { const results: Array<{ name: string; body: string }> = []; - // v1: brace-depth only — braces inside comments or string literals are not filtered (see spec Fix 2) + // Sanitize comments and string literals so braces inside them don't + // throw off the depth counter. The sanitized copy has the same length + // and offsets as the original, so we use it ONLY to scan for service + // headers and braces; the service body we return is sliced from the + // ORIGINAL content to preserve exact source text for downstream use. + const sanitized = stripProtoCommentsAndStrings(content); const headerRe = /service\s+(\w+)\s*\{/g; let headerMatch: RegExpExecArray | null; - while ((headerMatch = headerRe.exec(content)) !== null) { + while ((headerMatch = headerRe.exec(sanitized)) !== null) { const serviceName = headerMatch[1]; const bodyStart = headerMatch.index + headerMatch[0].length; let depth = 1; let pos = bodyStart; - while (pos < content.length && depth > 0) { - const ch = content[pos]; + while (pos < sanitized.length && depth > 0) { + const ch = sanitized[pos]; if (ch === '{') depth++; else if (ch === '}') depth--; pos++; @@ -75,6 +165,163 @@ function makeContract( }; } +export interface ProtoServiceInfo { + package: string; + serviceName: string; + methods: string[]; + protoPath: string; +} + +function normalizeProtoPath(rel: string): string { + return rel.replace(/\\/g, '/'); +} + +function extractProtoImports(content: string): string[] { + const imports: string[] = []; + const re = /^\s*import\s+"([^"]+)"\s*;/gm; + let match: RegExpExecArray | null; + while ((match = re.exec(content)) !== null) { + imports.push(match[1]); + } + return imports; +} + +function longestSharedSegmentRun(aPath: string, bPath: string): number { + const a = aPath.split('/').filter(Boolean); + const b = bPath.split('/').filter(Boolean); + let best = 0; + + for (let i = 0; i < a.length; i++) { + for (let j = 0; j < b.length; j++) { + let run = 0; + while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) { + run++; + } + if (run > best) best = run; + } + } + + return best; +} + +async function buildProtoContext(repoPath: string): Promise<{ + packagesByProto: Map; + servicesByName: Map; +}> { + const servicesByName = new Map(); + const protoFiles = await glob('**/*.proto', { + cwd: repoPath, + absolute: false, + nodir: true, + ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'], + }); + const contents = new Map(); + + for (const rel of protoFiles) { + const content = readSafe(repoPath, rel); + if (!content) continue; + contents.set(normalizeProtoPath(rel), content); + } + + const packagesByProto = new Map(); + + const resolvePackage = (protoPath: string, seen = new Set()): string => { + if (packagesByProto.has(protoPath)) return packagesByProto.get(protoPath) ?? ''; + if (seen.has(protoPath)) return ''; + + const content = contents.get(protoPath); + if (!content) return ''; + + seen.add(protoPath); + const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m); + if (pkgMatch?.[1]) { + packagesByProto.set(protoPath, pkgMatch[1]); + return pkgMatch[1]; + } + + for (const importPath of extractProtoImports(content)) { + const normalizedImport = normalizeProtoPath(importPath); + const candidates = [ + normalizeProtoPath( + path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport)), + ), + normalizedImport, + ]; + for (const candidate of candidates) { + if (!contents.has(candidate)) continue; + const inheritedPackage = resolvePackage(candidate, seen); + if (inheritedPackage) { + packagesByProto.set(protoPath, inheritedPackage); + return inheritedPackage; + } + } + } + + packagesByProto.set(protoPath, ''); + return ''; + }; + + for (const rel of protoFiles) { + const normalizedRel = normalizeProtoPath(rel); + const content = contents.get(normalizedRel); + if (!content) continue; + const pkg = resolvePackage(normalizedRel); + + const serviceBlocks = extractServiceBlocks(content); + for (const block of serviceBlocks) { + const rpcRe = /rpc\s+(\w+)\s*\(/g; + const methods: string[] = []; + let m: RegExpExecArray | null; + while ((m = rpcRe.exec(block.body)) !== null) { + methods.push(m[1]); + } + const info: ProtoServiceInfo = { + package: pkg, + serviceName: block.name, + methods, + protoPath: normalizedRel, + }; + const existing = servicesByName.get(block.name) ?? []; + existing.push(info); + servicesByName.set(block.name, existing); + } + } + + return { packagesByProto, servicesByName }; +} + +export async function buildProtoMap(repoPath: string): Promise> { + const { servicesByName } = await buildProtoContext(repoPath); + return servicesByName; +} + +export function resolveProtoConflict( + _serviceName: string, + sourceFilePath: string, + candidates: ProtoServiceInfo[], +): ProtoServiceInfo | null { + if (candidates.length === 0) return null; + if (candidates.length === 1) return candidates[0]; + + const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath)); + let best = candidates[0]; + let bestScore = -1; + for (const c of candidates) { + const protoDir = normalizeProtoPath(path.dirname(c.protoPath)); + const sharedRun = longestSharedSegmentRun(sourceDir, protoDir); + if (sharedRun > bestScore) { + bestScore = sharedRun; + best = c; + } + } + return best; +} + +export function serviceContractId(pkg: string, serviceName: string): string { + const prefix = pkg ? `${pkg}.${serviceName}` : serviceName; + return `grpc::${prefix}/*`; +} + export class GrpcExtractor implements ContractExtractor { type = 'grpc' as const; @@ -88,6 +335,7 @@ export class GrpcExtractor implements ContractExtractor { _repo: RepoHandle, ): Promise { const out: ExtractedContract[] = []; + const protoContext = await buildProtoContext(repoPath); // Proto files — definitive provider source const protoFiles = await glob('**/*.proto', { @@ -97,8 +345,17 @@ export class GrpcExtractor implements ContractExtractor { }); for (const rel of protoFiles) { const content = readSafe(repoPath, rel); - if (content) out.push(...this.parseProtoFile(content, rel)); + if (content) { + out.push( + ...this.parseProtoFile( + content, + rel, + protoContext.packagesByProto.get(normalizeProtoPath(rel)) ?? '', + ), + ); + } } + const protoMap = protoContext.servicesByName; // Source files — server/client detection const sourceFiles = await glob('**/*.{go,java,py,ts,tsx,js,jsx}', { @@ -112,28 +369,26 @@ export class GrpcExtractor implements ContractExtractor { const ext = path.extname(rel).toLowerCase(); if (ext === '.go') { - out.push(...this.scanGoProviders(content, rel)); - out.push(...this.scanGoConsumers(content, rel)); + out.push(...this.scanGoProviders(content, rel, protoMap)); + out.push(...this.scanGoConsumers(content, rel, protoMap)); } else if (ext === '.java') { - out.push(...this.scanJavaProviders(content, rel)); - out.push(...this.scanJavaConsumers(content, rel)); + out.push(...this.scanJavaProviders(content, rel, protoMap)); + out.push(...this.scanJavaConsumers(content, rel, protoMap)); } else if (ext === '.py') { - out.push(...this.scanPythonProviders(content, rel)); - out.push(...this.scanPythonConsumers(content, rel)); + out.push(...this.scanPythonProviders(content, rel, protoMap)); + out.push(...this.scanPythonConsumers(content, rel, protoMap)); } else if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) { - out.push(...this.scanTsProviders(content, rel)); + out.push(...this.scanTsProviders(content, rel, protoMap)); + out.push(...this.scanTsConsumers(content, rel, protoMap)); } } return this.dedupe(out); } - private parseProtoFile(content: string, filePath: string): ExtractedContract[] { + private parseProtoFile(content: string, filePath: string, pkg: string): ExtractedContract[] { const out: ExtractedContract[] = []; - const pkgMatch = content.match(/^package\s+([\w.]+)\s*;/m); - const pkg = pkgMatch ? pkgMatch[1] : ''; - for (const { name: serviceName, body } of extractServiceBlocks(content)) { const rpcRe = /rpc\s+(\w+)\s*\(/g; let rpcMatch: RegExpExecArray | null; @@ -154,7 +409,11 @@ export class GrpcExtractor implements ContractExtractor { return out; } - private scanGoProviders(content: string, filePath: string): ExtractedContract[] { + private scanGoProviders( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; // pb.RegisterXxxServer( @@ -162,15 +421,17 @@ export class GrpcExtractor implements ContractExtractor { let m: RegExpExecArray | null; while ((m = registerRe.exec(content)) !== null) { const serviceName = m[1]; + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? 0.8 : 0.65; out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `Register${serviceName}Server`, - 0.8, - { service: serviceName, source: 'go_register' }, - ), + makeContract(cid, 'provider', filePath, `Register${serviceName}Server`, conf, { + service: serviceName, + source: 'go_register', + }), ); } @@ -178,51 +439,74 @@ export class GrpcExtractor implements ContractExtractor { const unimplRe = /\w+\.Unimplemented(\w+)Server\b/g; while ((m = unimplRe.exec(content)) !== null) { const serviceName = m[1]; + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? 0.8 : 0.65; out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `Unimplemented${serviceName}Server`, - 0.8, - { service: serviceName, source: 'go_unimplemented' }, - ), + makeContract(cid, 'provider', filePath, `Unimplemented${serviceName}Server`, conf, { + service: serviceName, + source: 'go_unimplemented', + }), ); } return out; } - private scanGoConsumers(content: string, filePath: string): ExtractedContract[] { + private scanGoConsumers( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; const re = /\w+\.New(\w+)Client\s*\(/g; let m: RegExpExecArray | null; while ((m = re.exec(content)) !== null) { const serviceName = m[1]; + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? 0.75 : 0.55; out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'consumer', - filePath, - `New${serviceName}Client`, - 0.7, - { service: serviceName, source: 'go_client' }, - ), + makeContract(cid, 'consumer', filePath, `New${serviceName}Client`, conf, { + service: serviceName, + source: 'go_client', + }), ); } return out; } - private scanJavaProviders(content: string, filePath: string): ExtractedContract[] { + private scanJavaProviders( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; + const resolveJava = (svcName: string): { cid: string; conf: number } => { + const candidates = protoMap.get(svcName); + const proto = resolveProtoConflict(svcName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(svcName); + const conf = proto ? 0.8 : 0.65; + return { cid, conf }; + }; + // @GrpcService if (content.includes('@GrpcService')) { const implBaseRe = /extends\s+(\w+)Grpc\.(\w+)ImplBase/; const m = content.match(implBaseRe); if (m) { + const { cid, conf } = resolveJava(m[1]); out.push( - makeContract(serviceOnlyContractId(m[1]), 'provider', filePath, m[2], 0.8, { + makeContract(cid, 'provider', filePath, m[2], conf, { service: m[1], source: 'java_grpc_service', }), @@ -234,8 +518,9 @@ export class GrpcExtractor implements ContractExtractor { const cm = content.match(classRe); if (cm) { const svcName = cm[2].replace(/Grpc$/, ''); + const { cid, conf } = resolveJava(svcName); out.push( - makeContract(serviceOnlyContractId(svcName), 'provider', filePath, cm[1], 0.8, { + makeContract(cid, 'provider', filePath, cm[1], conf, { service: svcName, source: 'java_grpc_service', }), @@ -250,8 +535,9 @@ export class GrpcExtractor implements ContractExtractor { const m = content.match(implRe); if (m) { const svcName = m[2] || m[1].replace(/Grpc$/, ''); + const { cid, conf } = resolveJava(svcName); out.push( - makeContract(serviceOnlyContractId(svcName), 'provider', filePath, svcName, 0.8, { + makeContract(cid, 'provider', filePath, svcName, conf, { service: svcName, source: 'java_impl_base', }), @@ -262,49 +548,65 @@ export class GrpcExtractor implements ContractExtractor { return out; } - private scanJavaConsumers(content: string, filePath: string): ExtractedContract[] { + private scanJavaConsumers( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; // XxxGrpc.newBlockingStub( or XxxGrpc.newStub( const re = /(\w+)Grpc\.new(?:Blocking)?Stub\s*\(/g; let m: RegExpExecArray | null; while ((m = re.exec(content)) !== null) { const serviceName = m[1]; + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? 0.75 : 0.55; out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'consumer', - filePath, - `${serviceName}Stub`, - 0.7, - { service: serviceName, source: 'java_stub' }, - ), + makeContract(cid, 'consumer', filePath, `${serviceName}Stub`, conf, { + service: serviceName, + source: 'java_stub', + }), ); } return out; } - private scanPythonProviders(content: string, filePath: string): ExtractedContract[] { + private scanPythonProviders( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; // add_XxxServicer_to_server( const re = /add_(\w+?)Servicer_to_server\s*\(/g; let m: RegExpExecArray | null; while ((m = re.exec(content)) !== null) { const serviceName = m[1]; + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? 0.8 : 0.65; out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `add_${serviceName}Servicer_to_server`, - 0.8, - { service: serviceName, source: 'python_servicer' }, - ), + makeContract(cid, 'provider', filePath, `add_${serviceName}Servicer_to_server`, conf, { + service: serviceName, + source: 'python_servicer', + }), ); } return out; } - private scanPythonConsumers(content: string, filePath: string): ExtractedContract[] { + private scanPythonConsumers( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; // XxxStub( const re = /(\w+)Stub\s*\(/g; @@ -313,8 +615,14 @@ export class GrpcExtractor implements ContractExtractor { const name = m[1]; // Filter out common false positives if (['Mock', 'Test', 'Fake', 'Stub'].includes(name)) continue; + const candidates = protoMap.get(name); + const proto = resolveProtoConflict(name, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(name); + const conf = proto ? 0.75 : 0.55; out.push( - makeContract(serviceOnlyContractId(name), 'consumer', filePath, `${name}Stub`, 0.7, { + makeContract(cid, 'consumer', filePath, `${name}Stub`, conf, { service: name, source: 'python_stub', }), @@ -323,7 +631,11 @@ export class GrpcExtractor implements ContractExtractor { return out; } - private scanTsProviders(content: string, filePath: string): ExtractedContract[] { + private scanTsProviders( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; // @GrpcMethod('ServiceName', 'MethodName') const re = /@GrpcMethod\s*\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g; @@ -331,7 +643,10 @@ export class GrpcExtractor implements ContractExtractor { while ((m = re.exec(content)) !== null) { const serviceName = m[1]; const methodName = m[2]; - const cid = contractId('', serviceName, methodName); + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const pkg = proto?.package ?? ''; + const cid = contractId(pkg, serviceName, methodName); out.push( makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.8, { service: serviceName, @@ -343,15 +658,74 @@ export class GrpcExtractor implements ContractExtractor { return out; } - private dedupe(items: ExtractedContract[]): ExtractedContract[] { - const seen = new Set(); + private scanTsConsumers( + content: string, + filePath: string, + protoMap: Map, + ): ExtractedContract[] { const out: ExtractedContract[] = []; - for (const c of items) { - const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; - if (seen.has(k)) continue; - seen.add(k); - out.push(c); + const pushConsumer = ( + serviceName: string, + symbolName: string, + source: string, + confidenceWithProto = 0.75, + confidenceWithoutProto = 0.55, + ): void => { + const candidates = protoMap.get(serviceName); + const proto = resolveProtoConflict(serviceName, filePath, candidates ?? []); + const cid = proto + ? serviceContractId(proto.package, proto.serviceName) + : serviceOnlyContractId(serviceName); + const conf = proto ? confidenceWithProto : confidenceWithoutProto; + out.push( + makeContract(cid, 'consumer', filePath, symbolName, conf, { + service: serviceName, + source, + }), + ); + }; + + const grpcClientDecoratorRe = + /@GrpcClient\s*\([^)]*\)\s*(?:private|protected|public)?\s*(?:readonly\s+)?\w+[!?]?\s*:\s*(\w+Service)Client\b/g; + let match: RegExpExecArray | null; + while ((match = grpcClientDecoratorRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_grpc_client_decorator'); } + + const getServiceRe = /\.getService(?:<[^>]+>)?\s*\(\s*['"](\w+)['"]\s*\)/g; + while ((match = getServiceRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_client_grpc_get_service'); + } + + const clientCtorRe = /new\s+(\w+Service)Client\s*\(/g; + while ((match = clientCtorRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_generated_client'); + } + + if (content.includes('loadPackageDefinition')) { + const packageCtorRe = /new\s+[\w$.]*\.([A-Z]\w+)\s*\(/g; + while ((match = packageCtorRe.exec(content)) !== null) { + pushConsumer(match[1], `${match[1]}Client`, 'ts_load_package_definition'); + } + } + return out; } + + private dedupe(items: ExtractedContract[]): ExtractedContract[] { + const byKey = new Map(); + for (const c of items) { + const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; + const existing = byKey.get(k); + if ( + !existing || + c.confidence > existing.confidence || + (c.confidence === existing.confidence && + String(c.meta.source) < String(existing.meta.source)) + ) { + byKey.set(k, c); + } + } + return Array.from(byKey.values()); + } } diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index ebb4c668d..2a9f662c3 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -226,7 +226,7 @@ export class HttpRouteExtractor implements ContractExtractor { } private async extractProvidersSourceScan(repoPath: string): Promise { - const files = await glob('**/*.{ts,tsx,js,jsx,java,vue,svelte,php,py}', { + const files = await glob('**/*.{ts,tsx,js,jsx,java,vue,svelte,php,py,go}', { cwd: repoPath, ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'], nodir: true, @@ -236,7 +236,9 @@ export class HttpRouteExtractor implements ContractExtractor { const content = readSafe(repoPath, rel); if (!content) continue; out.push(...this.scanSpringProviders(content, rel)); + out.push(...this.scanNestProviders(content, rel)); out.push(...this.scanExpressProviders(content, rel)); + out.push(...this.scanGoProviders(content, rel)); out.push(...this.scanLaravelProviders(content, rel)); out.push(...this.scanFastApiProviders(content, rel)); } @@ -257,18 +259,6 @@ export class HttpRouteExtractor implements ContractExtractor { private scanSpringProviders(content: string, filePath: string): ExtractedContract[] { const out: ExtractedContract[] = []; - - // Skip Feign/client interfaces — annotated methods in interfaces are - // consumers (Feign, JAX-RS proxies), not provider endpoints. - // Anchored to line start (with optional access modifier) so we do not - // match "interface" inside comments or string literals. - if ( - /^\s*(?:public\s+)?interface\s+\w+/m.test(content) && - !/@(?:Rest)?Controller\b/.test(content) - ) { - return out; - } - let classPrefix = ''; const classRm = content.match(/@RequestMapping\s*\(\s*"([^"]+)"/); if (classRm) classPrefix = classRm[1].replace(/\/+$/, ''); @@ -324,6 +314,48 @@ export class HttpRouteExtractor implements ContractExtractor { return out; } + private scanNestProviders(content: string, filePath: string): ExtractedContract[] { + const out: ExtractedContract[] = []; + const controllerMatch = content.match(/@Controller\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/); + const controllerPrefix = controllerMatch ? controllerMatch[1].replace(/\/+$/, '') : ''; + const re = /@(Get|Post|Put|Delete|Patch)\s*\(\s*['"`]?([^'"`)]*)['"`]?\s*\)/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(content)) !== null) { + const method = m[1].toUpperCase(); + const routePath = String(m[2] || ''); + const fullPath = controllerPrefix + ? `${controllerPrefix}/${routePath.replace(/^\/+/, '')}` + : routePath; + const pathNorm = normalizeHttpPath(fullPath.startsWith('/') ? fullPath : `/${fullPath}`); + const sub = content.slice(m.index); + const nameMatch = sub.match( + /(?:public|protected|private)?\s*(?:async\s+)?(\w+)\s*\([^)]*\)\s*\{/, + ); + const name = nameMatch ? nameMatch[1] : m[0]; + out.push(this.makeProvider(filePath, method, pathNorm, name, 0.8)); + } + return out; + } + + private scanGoProviders(content: string, filePath: string): ExtractedContract[] { + const out: ExtractedContract[] = []; + const frameworkRe = + /(?:^|\W)\w+\.(GET|POST|PUT|DELETE|PATCH)\s*\(\s*['"]([^'"]+)['"]\s*,\s*(\w+)/gim; + let m: RegExpExecArray | null; + while ((m = frameworkRe.exec(content)) !== null) { + out.push(this.makeProvider(filePath, m[1].toUpperCase(), normalizeHttpPath(m[2]), m[3], 0.8)); + } + + const handleFuncRe = + /(?:http|\w+)\.HandleFunc\s*\(\s*['"]([^'"]+)['"]\s*,\s*(\w+)\s*\)(?:\s*\.\s*Methods\s*\(\s*['"](\w+)['"]\s*\))?/gim; + while ((m = handleFuncRe.exec(content)) !== null) { + const method = (m[3] || 'GET').toUpperCase(); + out.push(this.makeProvider(filePath, method, normalizeHttpPath(m[1]), m[2], 0.8)); + } + + return out; + } + private makeProvider( filePath: string, method: string, @@ -419,7 +451,7 @@ export class HttpRouteExtractor implements ContractExtractor { } private async extractConsumersSourceScan(repoPath: string): Promise { - const files = await glob('**/*.{ts,tsx,js,jsx,vue,svelte}', { + const files = await glob('**/*.{ts,tsx,js,jsx,vue,svelte,py,java,go}', { cwd: repoPath, ignore: ['**/node_modules/**', '**/.git/**'], nodir: true, @@ -430,6 +462,9 @@ export class HttpRouteExtractor implements ContractExtractor { if (!content) continue; out.push(...this.scanFetchConsumers(content, rel)); out.push(...this.scanAxiosConsumers(content, rel)); + out.push(...this.scanPythonRequestsConsumers(content, rel)); + out.push(...this.scanJavaConsumers(content, rel)); + out.push(...this.scanGoConsumers(content, rel)); } return this.dedupeContracts(out); } @@ -451,18 +486,127 @@ export class HttpRouteExtractor implements ContractExtractor { return url.replace(/\$\{[^}]+\}/g, '{param}'); } + private normalizeConsumerPath(url: string): string { + const templated = this.templateToPattern(url.trim()); + let pathOnly = templated; + if (/^https?:\/\//i.test(templated)) { + try { + pathOnly = new URL(templated).pathname; + } catch { + pathOnly = templated.replace(/^https?:\/\/[^/]+/i, ''); + } + } + + const normalized = normalizeHttpPath(pathOnly || '/'); + const segments = normalized + .split('/') + .filter(Boolean) + .map((segment) => { + if (/^\d+$/.test(segment)) return '{param}'; + return segment; + }); + return `/${segments.join('/')}`.replace(/\/+$/, '') || '/'; + } + private scanAxiosConsumers(content: string, filePath: string): ExtractedContract[] { const out: ExtractedContract[] = []; const re = /axios\.(get|post|put|delete|patch)\s*\(\s*[`'"]([^`'"]+)[`'"]/gi; let m: RegExpExecArray | null; while ((m = re.exec(content)) !== null) { const method = m[1].toUpperCase(); - const pathNorm = normalizeHttpPath(this.templateToPattern(m[2])); + const pathNorm = this.normalizeConsumerPath(m[2]); out.push(this.makeConsumer(filePath, method, pathNorm, 0.7)); } return out; } + private scanPythonRequestsConsumers(content: string, filePath: string): ExtractedContract[] { + const out: ExtractedContract[] = []; + const methodRe = /requests\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi; + let m: RegExpExecArray | null; + while ((m = methodRe.exec(content)) !== null) { + out.push( + this.makeConsumer(filePath, m[1].toUpperCase(), this.normalizeConsumerPath(m[2]), 0.7), + ); + } + + const genericRe = /requests\.request\s*\(\s*['"](\w+)['"]\s*,\s*['"]([^'"]+)['"]/gi; + while ((m = genericRe.exec(content)) !== null) { + out.push( + this.makeConsumer(filePath, m[1].toUpperCase(), this.normalizeConsumerPath(m[2]), 0.7), + ); + } + + return out; + } + + private scanJavaConsumers(content: string, filePath: string): ExtractedContract[] { + const out: ExtractedContract[] = []; + const restTemplateMethods: Array<[RegExp, string]> = [ + [/restTemplate\.getFor(?:Object|Entity)\s*\(\s*['"]([^'"]+)['"]/gi, 'GET'], + [/restTemplate\.postFor(?:Object|Entity)\s*\(\s*['"]([^'"]+)['"]/gi, 'POST'], + [/restTemplate\.put\s*\(\s*['"]([^'"]+)['"]/gi, 'PUT'], + [/restTemplate\.delete\s*\(\s*['"]([^'"]+)['"]/gi, 'DELETE'], + [/restTemplate\.patchForObject\s*\(\s*['"]([^'"]+)['"]/gi, 'PATCH'], + ]; + for (const [re, method] of restTemplateMethods) { + let m: RegExpExecArray | null; + while ((m = re.exec(content)) !== null) { + out.push(this.makeConsumer(filePath, method, this.normalizeConsumerPath(m[1]), 0.7)); + } + } + + const webClientMethodRe = + /webClient\.method\s*\(\s*HttpMethod\.(GET|POST|PUT|DELETE|PATCH)\s*,\s*['"]([^'"]+)['"]/gi; + let m: RegExpExecArray | null; + while ((m = webClientMethodRe.exec(content)) !== null) { + out.push( + this.makeConsumer(filePath, m[1].toUpperCase(), this.normalizeConsumerPath(m[2]), 0.7), + ); + } + + const okHttpRe = + /new\s+Request\.Builder\s*\(\)\s*\.url\s*\(\s*['"]([^'"]+)['"]\s*\)(?:\s*\.\s*method\s*\(\s*['"](\w+)['"])?/gim; + while ((m = okHttpRe.exec(content)) !== null) { + out.push( + this.makeConsumer( + filePath, + (m[2] || 'GET').toUpperCase(), + this.normalizeConsumerPath(m[1]), + 0.7, + ), + ); + } + + return out; + } + + private scanGoConsumers(content: string, filePath: string): ExtractedContract[] { + const out: ExtractedContract[] = []; + const httpMethodRe = /\bhttp\.(Get|Post|Head)\s*\(\s*['"]([^'"]+)['"]/gi; + let m: RegExpExecArray | null; + while ((m = httpMethodRe.exec(content)) !== null) { + const method = m[1].toUpperCase() === 'HEAD' ? 'GET' : m[1].toUpperCase(); + out.push(this.makeConsumer(filePath, method, this.normalizeConsumerPath(m[2]), 0.7)); + } + + const newRequestRe = /\bhttp\.NewRequest\s*\(\s*['"](\w+)['"]\s*,\s*['"]([^'"]+)['"]/gi; + while ((m = newRequestRe.exec(content)) !== null) { + out.push( + this.makeConsumer(filePath, m[1].toUpperCase(), this.normalizeConsumerPath(m[2]), 0.7), + ); + } + + const restyRe = /\b\w+\.R\(\)\.(Get|Post|Put|Delete|Patch)\s*\(\s*['"]([^'"]+)['"]/gi; + while ((m = restyRe.exec(content)) !== null) { + out.push( + this.makeConsumer(filePath, m[1].toUpperCase(), this.normalizeConsumerPath(m[2]), 0.7), + ); + } + + return out; + } + private makeConsumer( filePath: string, method: string, diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts new file mode 100644 index 000000000..09817b356 --- /dev/null +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -0,0 +1,228 @@ +import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js'; +import type { CypherExecutor } from '../contract-extractor.js'; + +export interface ManifestExtractResult { + contracts: StoredContract[]; + crossLinks: CrossLink[]; +} + +/** + * Canonicalize an HTTP path for matching against Route.name in the graph. + * Mirrors core/ingestion/pipeline.ts ensureSlash semantics: + * - Ensures a leading slash. + * - Strips trailing slashes (except the root "/"). + * - Normalizes consecutive slashes. + * - Does NOT lowercase (route matching is case-sensitive). + */ +function normalizeRoutePath(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return '/'; + const withLeading = trimmed.startsWith('/') ? trimmed : `/${trimmed}`; + const collapsed = withLeading.replace(/\/+/g, '/'); + if (collapsed === '/') return '/'; + return collapsed.replace(/\/+$/, ''); +} + +/** + * Stable synthetic symbolUid for a manifest-declared contract whose target + * symbol could not be resolved against the per-repo graph (resolveSymbol + * returned null). Two reasons we don't leave the uid empty: + * + * 1. The bridge stores Contract nodes keyed in part by symbolUid; an empty + * uid means downstream Cypher queries that anchor on `provider.symbolUid` + * can't tell two different unresolved manifest contracts apart. + * 2. The cross-impact bridge query in cross-impact.ts joins local impact + * results to bridge contracts via `WHERE provider.symbolUid IN $localUids`. + * If the local impact engine produces a deterministic identifier for the + * unresolved target, it must agree with the value the bridge stored. A + * synthetic uid keyed off (repo, contractId) is the only thing both sides + * can derive without knowing about each other. + * + * Format: `manifest::::`. Stable across syncs, scoped to a + * single repo within a group, and never collides with real indexer uids + * (which never start with `manifest::`). + */ +export function manifestSymbolUid(repo: string, contractId: string): string { + return `manifest::${repo}::${contractId}`; +} + +export class ManifestExtractor { + async extractFromManifest( + links: GroupManifestLink[], + dbExecutors?: Map, + ): Promise { + const contracts: StoredContract[] = []; + const crossLinks: CrossLink[] = []; + + for (const link of links) { + const contractId = this.buildContractId(link.type, link.contract); + + const providerRepo = link.role === 'provider' ? link.from : link.to; + const consumerRepo = link.role === 'provider' ? link.to : link.from; + + const providerSymbol = await this.resolveSymbol(providerRepo, link, dbExecutors); + const consumerSymbol = await this.resolveSymbol(consumerRepo, link, dbExecutors); + const providerRef = providerSymbol || { filePath: '', name: link.contract }; + const consumerRef = consumerSymbol || { filePath: '', name: link.contract }; + // When the resolver finds a real graph symbol we keep its uid, otherwise + // fall back to the deterministic synthetic uid (see manifestSymbolUid). + const providerUid = providerSymbol?.uid || manifestSymbolUid(providerRepo, contractId); + const consumerUid = consumerSymbol?.uid || manifestSymbolUid(consumerRepo, contractId); + + contracts.push({ + contractId, + type: link.type, + role: 'provider', + symbolUid: providerUid, + symbolRef: providerRef, + symbolName: link.contract, + confidence: 1.0, + meta: { source: 'manifest' }, + repo: providerRepo, + }); + + contracts.push({ + contractId, + type: link.type, + role: 'consumer', + symbolUid: consumerUid, + symbolRef: consumerRef, + symbolName: link.contract, + confidence: 1.0, + meta: { source: 'manifest' }, + repo: consumerRepo, + }); + + crossLinks.push({ + from: { repo: consumerRepo, symbolUid: consumerUid, symbolRef: consumerRef }, + to: { repo: providerRepo, symbolUid: providerUid, symbolRef: providerRef }, + type: link.type, + contractId, + matchType: 'manifest', + confidence: 1.0, + }); + } + + return { contracts, crossLinks }; + } + + private async resolveSymbol( + repoPathKey: string, + link: GroupManifestLink, + dbExecutors?: Map, + ): Promise<{ filePath: string; name: string; uid: string } | null> { + const executor = dbExecutors?.get(repoPathKey); + if (!executor) return null; + + // NOTE: All lookups use EXACT equality on the relevant name field and + // deterministic ORDER BY before LIMIT 1. Previous versions used CONTAINS + // for fuzzy matching (plus an unconditional ".proto" fallback for gRPC) + // which produced silent false positives: e.g. manifest "/orders" would + // match "/suborders", and a gRPC manifest entry in a repo with any + // .proto file would attach to a random proto symbol. + // + // If resolveSymbol returns null, the extractor creates a contract with + // an empty symbolUid/ref — cross-impact still works via name-based + // matching through the `hint` path in runGroupImpact. + try { + let rows: Record[]; + if (link.type === 'http') { + // Route.name is the canonicalized URL path (see + // core/ingestion/pipeline.ts ensureSlash + generateId('Route', ...)). + // Normalize the manifest contract the same way so a user-written + // "/api/orders" matches "api/orders" in the graph. + const normalized = normalizeRoutePath(link.contract); + rows = await executor( + `MATCH (handler)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route) + WHERE route.name = $normalized + RETURN handler.id AS uid, handler.name AS name, handler.filePath AS filePath + ORDER BY handler.filePath ASC + LIMIT 1`, + { normalized }, + ); + } else if (link.type === 'topic') { + rows = await executor( + `MATCH (n) WHERE n.name = $contract + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); + } else if (link.type === 'grpc') { + // Contract is "Service/Method" or just "Service" (or package.Service + // variants). Prefer matching by method name when present, otherwise + // by service name. NO .proto path fallback — that's guaranteed to + // return a wrong symbol in any repo with more than one proto file. + const parts = link.contract.split('/'); + const serviceName = parts[0]?.trim() ?? ''; + const methodName = parts[1]?.trim() ?? ''; + if (methodName) { + rows = await executor( + `MATCH (n) WHERE n.name = $methodName + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { methodName }, + ); + } else if (serviceName) { + rows = await executor( + `MATCH (n) WHERE n.name = $serviceName + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { serviceName }, + ); + } else { + rows = []; + } + } else if (link.type === 'lib') { + // Only exact match on the symbol's name. Previous fallback to + // CONTAINS on n.filePath would promote "react" to "react-native" + // or "@types/react" — silent wrong attribution. + rows = await executor( + `MATCH (n) WHERE n.name = $contract + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); + } else { + return null; + } + if (rows.length > 0) { + return { + filePath: rows[0].filePath as string, + name: rows[0].name as string, + uid: String(rows[0].uid ?? ''), + }; + } + } catch (err) { + // Log but don't throw: a broken graph query in one repo shouldn't + // fail the whole manifest extraction. Unresolved contracts still + // get a synthetic symbolUid below, so cross-impact can proceed. + const message = err instanceof Error ? err.message : String(err); + console.warn( + `[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` + + `in ${repoPathKey}: ${message}`, + ); + } + return null; + } + + private buildContractId(type: ContractType, contract: string): string { + switch (type) { + case 'http': { + if (/^[A-Za-z]+::/.test(contract)) return `http::${contract}`; + return `http::*::${contract}`; + } + case 'grpc': + return `grpc::${contract}`; + case 'topic': + return `topic::${contract}`; + case 'lib': + return `lib::${contract}`; + case 'custom': + return `custom::${contract}`; + } + } +} diff --git a/gitnexus/src/core/group/extractors/topic-extractor.ts b/gitnexus/src/core/group/extractors/topic-extractor.ts index c27b419bb..3d5280d86 100644 --- a/gitnexus/src/core/group/extractors/topic-extractor.ts +++ b/gitnexus/src/core/group/extractors/topic-extractor.ts @@ -6,6 +6,9 @@ import type { ExtractedContract, RepoHandle } from '../types.js'; type Broker = 'kafka' | 'rabbitmq' | 'nats'; +const KAFKAJS_CONSUMER_RUN_RE = /consumer\.run\s*\(\s*\{\s*eachMessage:/; +const KAFKAJS_SUBSCRIBE_RE = /consumer\.subscribe\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g; + function readSafe(repoPath: string, rel: string): string | null { const abs = path.resolve(repoPath, rel); const base = path.resolve(repoPath); @@ -116,6 +119,54 @@ const KAFKA_PATTERNS: PatternDef[] = [ topicGroup: 1, symbolName: 'producer.send', }, + // Go: sarama.ProducerMessage{Topic: "xxx"} struct literal (emitted by + // both NewSyncProducer and NewAsyncProducer client code paths). + // + // Previous pattern was `sarama.NewSyncProducer[\s\S]{0,300}?Topic:...` + // which anchored to the producer constructor and used a 300-char + // lookahead. In a loop like + // producer := sarama.NewSyncProducer(...) + // for _, item := range items { + // msg1 := &sarama.ProducerMessage{Topic: "order.created"} + // msg2 := &sarama.ProducerMessage{Topic: "order.shipped"} + // } + // the regex captured only "order.created" (first Topic after the + // constructor) and silently missed "order.shipped". Matching on the + // struct literal directly fixes both the false negative in loops and + // the spurious cross-message capture when multiple unrelated messages + // sit within 300 chars of the constructor. + { + regex: /sarama\.ProducerMessage\s*\{[\s\S]{0,200}?Topic:\s*"([^"]+)"/g, + role: 'provider', + broker: 'kafka', + confidence: 0.75, + topicGroup: 1, + symbolName: 'sarama.ProducerMessage', + }, + // Go: kafka-go writer construction. kafka-go does NOT wrap messages in + // a struct with a Topic field (the writer owns the topic), so we match + // the Writer itself. A 200-char window bridges the gap between + // `kafka.NewWriter(...)` / `kafka.Writer{` and the Topic field inside + // the config literal — kafka-go writer configs are small and rarely + // contain more than one Topic field, so the risk of cross-message + // capture is low here. + { + regex: /kafka\.(?:NewWriter|Writer)\b[\s\S]{0,200}?Topic:\s*"([^"]+)"/g, + role: 'provider', + broker: 'kafka', + confidence: 0.75, + topicGroup: 1, + symbolName: 'kafka.Writer', + }, + // Go: kafka-go reader construction, mirrors Writer above. + { + regex: /kafka\.(?:NewReader|Reader)\b[\s\S]{0,200}?Topic:\s*"([^"]+)"/g, + role: 'consumer', + broker: 'kafka', + confidence: 0.75, + topicGroup: 1, + symbolName: 'kafka.Reader', + }, ]; // --- RabbitMQ patterns --- @@ -205,6 +256,42 @@ const NATS_PATTERNS: PatternDef[] = [ topicGroup: 1, symbolName: 'nc.Publish', }, + // Go/Node JetStream: js.Subscribe("xxx" + { + regex: /js\.(?:S|s)ubscribe\s*\(\s*"([^"]+)"/g, + role: 'consumer', + broker: 'nats', + confidence: 0.8, + topicGroup: 1, + symbolName: 'js.Subscribe', + }, + // Go/Node JetStream: js.Publish("xxx" + { + regex: /js\.(?:P|p)ublish\s*\(\s*"([^"]+)"/g, + role: 'provider', + broker: 'nats', + confidence: 0.8, + topicGroup: 1, + symbolName: 'js.Publish', + }, + // Python: await nc.subscribe("xxx") + { + regex: /await\s+nc\.subscribe\s*\(\s*['"]([^'"]+)['"]/g, + role: 'consumer', + broker: 'nats', + confidence: 0.75, + topicGroup: 1, + symbolName: 'nc.subscribe', + }, + // Python: await nc.publish("xxx") + { + regex: /await\s+nc\.publish\s*\(\s*['"]([^'"]+)['"]/g, + role: 'provider', + broker: 'nats', + confidence: 0.75, + topicGroup: 1, + symbolName: 'nc.publish', + }, ]; const ALL_PATTERNS: PatternDef[] = [...KAFKA_PATTERNS, ...RABBITMQ_PATTERNS, ...NATS_PATTERNS]; @@ -229,6 +316,7 @@ export class TopicExtractor implements ContractExtractor { const out: ExtractedContract[] = []; for (const rel of files) { + if (rel.endsWith('_test.go')) continue; const content = readSafe(repoPath, rel); if (!content) continue; out.push(...this.scanFile(content, rel)); @@ -260,6 +348,16 @@ export class TopicExtractor implements ContractExtractor { } } + if (KAFKAJS_CONSUMER_RUN_RE.test(content)) { + const subscribeRe = new RegExp(KAFKAJS_SUBSCRIBE_RE.source, KAFKAJS_SUBSCRIBE_RE.flags); + let subscribeMatch: RegExpExecArray | null; + while ((subscribeMatch = subscribeRe.exec(content)) !== null) { + const topicName = subscribeMatch[1]; + if (!topicName) continue; + out.push(makeContract(topicName, 'consumer', filePath, 'consumer.run', 0.75, 'kafka')); + } + } + return out; } diff --git a/gitnexus/test/unit/group/grpc-extractor.test.ts b/gitnexus/test/unit/group/grpc-extractor.test.ts index b4fc63b5c..82d79cbd6 100644 --- a/gitnexus/test/unit/group/grpc-extractor.test.ts +++ b/gitnexus/test/unit/group/grpc-extractor.test.ts @@ -1,17 +1,23 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as fs from 'node:fs'; +import fsp from 'node:fs/promises'; import * as path from 'node:path'; import * as os from 'node:os'; -import { GrpcExtractor } from '../../../src/core/group/extractors/grpc-extractor.js'; +import { + GrpcExtractor, + buildProtoMap, + resolveProtoConflict, + serviceContractId, +} from '../../../src/core/group/extractors/grpc-extractor.js'; +import type { ProtoServiceInfo } from '../../../src/core/group/extractors/grpc-extractor.js'; import type { RepoHandle } from '../../../src/core/group/types.js'; describe('GrpcExtractor', () => { let tmpDir: string; let extractor: GrpcExtractor; - beforeEach(() => { - tmpDir = path.join(os.tmpdir(), `gitnexus-grpc-${Date.now()}`); - fs.mkdirSync(tmpDir, { recursive: true }); + beforeEach(async () => { + tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'gitnexus-grpc-')); extractor = new GrpcExtractor(); }); @@ -205,6 +211,66 @@ service IncompleteService { // The old regex would find partial match; the new parser should skip it expect(providers).toHaveLength(0); }); + + it('test_extract_proto_ignores_braces_inside_string_literals', async () => { + // Regression for a known parser limitation: braces inside string + // literals used to be counted as real service-body braces, which + // would terminate the service early and drop methods after the + // offending string. + writeFile( + 'api/strings.proto', + `syntax = "proto3"; +package strings; + +service TrickyService { + rpc First (Req) returns (Res) { + option (google.api.http).additional_bindings = { + post: "/v1/first"; + }; + } + // Previously the "{" inside this literal would close the service body. + option deprecated_reason = "use NewService { instead"; + rpc Second (Req) returns (Res); + rpc Third (Req) returns (Res); +} +`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const protoProviders = contracts.filter( + (c) => c.role === 'provider' && c.symbolRef.filePath === 'api/strings.proto', + ); + // All three methods must be extracted even though a string literal + // contains an unbalanced "{". + expect(protoProviders.map((c) => c.symbolName).sort()).toEqual([ + 'TrickyService.First', + 'TrickyService.Second', + 'TrickyService.Third', + ]); + }); + + it('test_extract_proto_ignores_braces_inside_comments', async () => { + writeFile( + 'api/commented.proto', + `syntax = "proto3"; +package commented; + +service Svc { + // TODO: move { or } from this comment — parser used to count them + /* A block comment with { unbalanced braces } */ + rpc Alpha (Req) returns (Res); + // }} end of the method block (in comment) + rpc Beta (Req) returns (Res); +} +`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const protoProviders = contracts.filter( + (c) => c.role === 'provider' && c.symbolRef.filePath === 'api/commented.proto', + ); + expect(protoProviders.map((c) => c.symbolName).sort()).toEqual(['Svc.Alpha', 'Svc.Beta']); + }); }); describe('Go server detection', () => { @@ -228,7 +294,7 @@ func main() { expect(providers.length).toBeGreaterThanOrEqual(1); expect(providers[0].contractId).toContain('grpc::'); expect(providers[0].contractId).toContain('AuthService'); - expect(providers[0].confidence).toBe(0.8); + expect(providers[0].confidence).toBe(0.65); }); it('test_extract_go_unimplemented_server_returns_provider', async () => { @@ -267,7 +333,7 @@ func NewAuthClient(conn *grpc.ClientConn) pb.AuthServiceClient { expect(consumers.length).toBeGreaterThanOrEqual(1); expect(consumers[0].contractId).toContain('AuthService'); - expect(consumers[0].confidence).toBe(0.7); + expect(consumers[0].confidence).toBe(0.55); }); }); @@ -287,7 +353,7 @@ public class AuthGrpcService extends AuthServiceGrpc.AuthServiceImplBase { expect(providers.length).toBeGreaterThanOrEqual(1); expect(providers[0].contractId).toContain('AuthService'); - expect(providers[0].confidence).toBe(0.8); + expect(providers[0].confidence).toBe(0.65); }); it('test_extract_java_blocking_stub_returns_consumer', async () => { @@ -306,7 +372,7 @@ public class AuthGrpcService extends AuthServiceGrpc.AuthServiceImplBase { expect(consumers.length).toBeGreaterThanOrEqual(1); expect(consumers[0].contractId).toContain('AuthService'); - expect(consumers[0].confidence).toBe(0.7); + expect(consumers[0].confidence).toBe(0.55); }); }); @@ -328,7 +394,7 @@ def serve(): expect(providers.length).toBeGreaterThanOrEqual(1); expect(providers[0].contractId).toContain('AuthService'); - expect(providers[0].confidence).toBe(0.8); + expect(providers[0].confidence).toBe(0.65); }); it('test_extract_python_stub_returns_consumer', async () => { @@ -346,7 +412,7 @@ stub = auth_pb2_grpc.AuthServiceStub(channel)`, expect(consumers.length).toBeGreaterThanOrEqual(1); expect(consumers[0].contractId).toContain('AuthService'); - expect(consumers[0].confidence).toBe(0.7); + expect(consumers[0].confidence).toBe(0.55); }); }); @@ -372,6 +438,165 @@ export class AuthController { expect(providers[0].contractId).toContain('Login'); expect(providers[0].confidence).toBe(0.8); }); + + it('test_extract_ts_grpc_client_decorator_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import { GrpcClient } from '@nestjs/microservices'; +import type { AuthServiceClient } from './generated/auth'; + +export class AuthGateway { + @GrpcClient({ package: 'auth.v1', protoPath: 'proto/auth.proto' }) + private readonly authClient!: AuthServiceClient; +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_getService_without_decorator_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import type { ClientGrpc } from '@nestjs/microservices'; + +export function createAuthClient(client: ClientGrpc) { + return client.getService('AuthService'); +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_generated_client_constructor_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import { credentials } from '@grpc/grpc-js'; +import { AuthServiceClient } from './generated/auth'; + +export const authClient = new AuthServiceClient('localhost:50051', credentials.createInsecure());`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_non_service_client_constructor_is_ignored', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import { AuthClient } from './generated/auth'; + +export const authClient = new AuthClient('localhost:50051');`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + + it('test_extract_ts_loadPackageDefinition_constructor_returns_consumer', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; + +const definition = protoLoader.loadSync('proto/auth.proto'); +const authProto = grpc.loadPackageDefinition(definition) as any; +export const authClient = new authProto.auth.v1.AuthService( + 'localhost:50051', + grpc.credentials.createInsecure(), +);`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); + + it('test_extract_ts_duplicate_consumer_patterns_in_one_file_dedupes_deterministically', async () => { + writeFile( + 'proto/auth.proto', + `syntax = "proto3"; +package auth.v1; +service AuthService { + rpc Login (LoginRequest) returns (LoginResponse); +}`, + ); + writeFile( + 'src/auth.client.ts', + `import * as grpc from '@grpc/grpc-js'; +import type { ClientGrpc } from '@nestjs/microservices'; +import { AuthServiceClient } from './generated/auth'; + +export class AuthGateway { + constructor(private readonly client: ClientGrpc) {} + + connect() { + this.client.getService('AuthService'); + return new AuthServiceClient('localhost:50051', grpc.credentials.createInsecure()); + } +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('grpc::auth.v1.AuthService/*'); + }); }); describe('edge cases', () => { @@ -389,3 +614,297 @@ export class AuthController { }); }); }); + +describe('buildProtoMap', () => { + let tmpDir: string; + beforeEach(async () => { + tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'proto-test-')); + }); + afterEach(async () => { + await fsp.rm(tmpDir, { recursive: true, force: true }); + }); + + it('test_buildProtoMap_single_proto_parses_package_service_methods', async () => { + const protoContent = ` +syntax = "proto3"; +package com.example; + +service UserService { + rpc GetUser (GetUserRequest) returns (GetUserResponse); + rpc ListUsers (ListUsersRequest) returns (ListUsersResponse); +}`; + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile(path.join(tmpDir, 'proto', 'user.proto'), protoContent); + + const map = await buildProtoMap(tmpDir); + expect(map.has('UserService')).toBe(true); + const entries = map.get('UserService')!; + expect(entries).toHaveLength(1); + expect(entries[0].package).toBe('com.example'); + expect(entries[0].serviceName).toBe('UserService'); + expect(entries[0].methods).toEqual(['GetUser', 'ListUsers']); + expect(entries[0].protoPath).toBe('proto/user.proto'); + }); + + it('test_buildProtoMap_no_package_declaration', async () => { + const protoContent = ` +syntax = "proto3"; +service Foo { rpc Bar (Req) returns (Res); }`; + await fsp.writeFile(path.join(tmpDir, 'foo.proto'), protoContent); + + const map = await buildProtoMap(tmpDir); + const entries = map.get('Foo')!; + expect(entries[0].package).toBe(''); + }); + + it('test_buildProtoMap_no_protos_returns_empty', async () => { + const map = await buildProtoMap(tmpDir); + expect(map.size).toBe(0); + }); + + it('test_buildProtoMap_conflicting_names', async () => { + await fsp.mkdir(path.join(tmpDir, 'a'), { recursive: true }); + await fsp.mkdir(path.join(tmpDir, 'b'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'a', 'svc.proto'), + 'package pkg.a;\nservice Svc { rpc Do (R) returns (R); }', + ); + await fsp.writeFile( + path.join(tmpDir, 'b', 'svc.proto'), + 'package pkg.b;\nservice Svc { rpc Do (R) returns (R); }', + ); + + const map = await buildProtoMap(tmpDir); + expect(map.get('Svc')).toHaveLength(2); + }); + + it('test_buildProtoMap_imported_package_is_inherited_for_split_service_definition', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto', 'shared'), { recursive: true }); + await fsp.mkdir(path.join(tmpDir, 'proto', 'services'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'shared', 'package.proto'), + 'package auth.v1;\nmessage LoginRequest {}', + ); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'services', 'auth.proto'), + 'import "../shared/package.proto";\nservice AuthService { rpc Login (LoginRequest) returns (LoginRequest); }', + ); + + const map = await buildProtoMap(tmpDir); + const entries = map.get('AuthService')!; + + expect(entries).toHaveLength(1); + expect(entries[0].package).toBe('auth.v1'); + }); +}); + +describe('resolveProtoConflict', () => { + const makeInfo = (pkg: string, protoPath: string): ProtoServiceInfo => ({ + package: pkg, + serviceName: 'Svc', + methods: ['Do'], + protoPath, + }); + + it('test_single_candidate_returns_it', () => { + const result = resolveProtoConflict('Svc', 'src/main.go', [makeInfo('pkg', 'proto/svc.proto')]); + expect(result?.package).toBe('pkg'); + }); + + it('test_multiple_candidates_picks_closest_directory', () => { + const candidates = [ + makeInfo('far', 'other/dir/svc.proto'), + makeInfo('close', 'src/proto/svc.proto'), + ]; + const result = resolveProtoConflict('Svc', 'src/server.go', candidates); + expect(result?.package).toBe('close'); + }); + + it('test_centralized_proto_layout_prefers_shared_path_segments_over_prefix_only', () => { + const candidates = [ + makeInfo('billing', 'proto/services/billing/svc.proto'), + makeInfo('auth', 'proto/services/auth/svc.proto'), + ]; + const result = resolveProtoConflict('Svc', 'services/auth/src/server.ts', candidates); + expect(result?.package).toBe('auth'); + }); + + it('test_no_candidates_returns_null', () => { + expect(resolveProtoConflict('Svc', 'src/main.go', [])).toBeNull(); + }); +}); + +describe('serviceContractId', () => { + it('test_with_package', () => { + expect(serviceContractId('com.example', 'UserService')).toBe('grpc::com.example.UserService/*'); + }); + + it('test_without_package', () => { + expect(serviceContractId('', 'UserService')).toBe('grpc::UserService/*'); + }); +}); + +describe('proto-aware source scanners', () => { + let tmpDir: string; + let extractor: GrpcExtractor; + + beforeEach(async () => { + tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'scanner-test-')); + extractor = new GrpcExtractor(); + }); + afterEach(async () => { + await fsp.rm(tmpDir, { recursive: true, force: true }); + }); + + const makeRepo = (repoPath: string): RepoHandle => ({ + id: 'test-repo', + path: '', + repoPath, + storagePath: '', + }); + + it('test_go_provider_with_proto_uses_canonical_service_id', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'user.proto'), + 'package com.example;\nservice UserService { rpc GetUser (R) returns (R); }', + ); + await fsp.mkdir(path.join(tmpDir, 'src'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'src', 'server.go'), + 'package main\nfunc init() { pb.RegisterUserServiceServer(srv, &impl{}) }', + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const goProvider = contracts.find((c) => c.meta.source === 'go_register'); + expect(goProvider).toBeDefined(); + expect(goProvider!.contractId).toBe('grpc::com.example.UserService/*'); + expect(goProvider!.confidence).toBe(0.8); + }); + + it('test_go_provider_without_proto_reduced_confidence', async () => { + await fsp.mkdir(path.join(tmpDir, 'src'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'src', 'server.go'), + 'package main\nfunc init() { pb.RegisterFooServer(srv, &impl{}) }', + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const goProvider = contracts.find((c) => c.meta.source === 'go_register'); + expect(goProvider).toBeDefined(); + expect(goProvider!.contractId).toBe('grpc::Foo/*'); + expect(goProvider!.confidence).toBe(0.65); + }); + + it('test_go_consumer_with_proto_uses_canonical_service_id', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'user.proto'), + 'package com.example;\nservice UserService { rpc GetUser (R) returns (R); }', + ); + await fsp.mkdir(path.join(tmpDir, 'src'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'src', 'client.go'), + 'package main\nfunc init() { client := pb.NewUserServiceClient(conn) }', + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const goConsumer = contracts.find((c) => c.meta.source === 'go_client'); + expect(goConsumer).toBeDefined(); + expect(goConsumer!.contractId).toBe('grpc::com.example.UserService/*'); + expect(goConsumer!.confidence).toBe(0.75); + }); + + it('test_java_provider_with_proto_uses_canonical_service_id', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'user.proto'), + 'package com.example;\nservice UserService { rpc GetUser (R) returns (R); }', + ); + await fsp.mkdir(path.join(tmpDir, 'src', 'main', 'java'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'src', 'main', 'java', 'UserGrpcService.java'), + `@GrpcService +public class UserGrpcService extends UserServiceGrpc.UserServiceImplBase { + @Override + public void getUser(GetUserRequest req, StreamObserver obs) {} +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const javaProvider = contracts.find((c) => c.meta.source === 'java_grpc_service'); + expect(javaProvider).toBeDefined(); + expect(javaProvider!.contractId).toBe('grpc::com.example.UserService/*'); + expect(javaProvider!.confidence).toBe(0.8); + }); + + it('test_python_consumer_with_proto_uses_canonical_service_id', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'user.proto'), + 'package com.example;\nservice UserService { rpc GetUser (R) returns (R); }', + ); + await fsp.writeFile( + path.join(tmpDir, 'client.py'), + `import grpc +channel = grpc.insecure_channel('localhost:50051') +stub = UserServiceStub(channel)`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const pyConsumer = contracts.find((c) => c.meta.source === 'python_stub'); + expect(pyConsumer).toBeDefined(); + expect(pyConsumer!.contractId).toBe('grpc::com.example.UserService/*'); + expect(pyConsumer!.confidence).toBe(0.75); + }); + + it('test_ts_provider_with_proto_adds_package', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'user.proto'), + 'package com.example;\nservice UserService { rpc GetUser (R) returns (R); }', + ); + await fsp.mkdir(path.join(tmpDir, 'src'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'src', 'controller.ts'), + "@GrpcMethod('UserService', 'GetUser')\nasync getUser() {}", + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const tsProvider = contracts.find((c) => c.meta.source === 'ts_grpc_method'); + expect(tsProvider).toBeDefined(); + expect(tsProvider!.contractId).toBe('grpc::com.example.UserService/GetUser'); + expect(tsProvider!.confidence).toBe(0.8); + }); + + it('test_proto_provider_inherits_package_from_imported_definition', async () => { + await fsp.mkdir(path.join(tmpDir, 'proto', 'shared'), { recursive: true }); + await fsp.mkdir(path.join(tmpDir, 'proto', 'services'), { recursive: true }); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'shared', 'package.proto'), + 'package auth.v1;\nmessage LoginRequest {}', + ); + await fsp.writeFile( + path.join(tmpDir, 'proto', 'services', 'auth.proto'), + `syntax = "proto3"; +import "../shared/package.proto"; +service AuthService { + rpc Login (LoginRequest) returns (LoginRequest); +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + const protoProvider = contracts.find( + (c) => c.symbolRef.filePath === 'proto/services/auth.proto', + ); + expect(protoProvider).toBeDefined(); + expect(protoProvider!.contractId).toBe('grpc::auth.v1.AuthService/Login'); + }); +}); diff --git a/gitnexus/test/unit/group/http-route-extractor.test.ts b/gitnexus/test/unit/group/http-route-extractor.test.ts index d4c0db3eb..653b4952c 100644 --- a/gitnexus/test/unit/group/http-route-extractor.test.ts +++ b/gitnexus/test/unit/group/http-route-extractor.test.ts @@ -157,6 +157,89 @@ export default router; providers.find((c) => c.contractId === 'http::DELETE::/api/users/{param}'), ).toBeDefined(); }); + + it('extracts Go Gin and Echo route registrations', async () => { + const dir = path.join(tmpDir, 'go-frameworks'); + fs.mkdirSync(path.join(dir, 'cmd'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'cmd', 'server.go'), + ` +package main + +func createOrder(c *gin.Context) {} +func listOrders(c echo.Context) error { return nil } + +func main() { + r := gin.Default() + r.POST("/api/orders/:id", createOrder) + + e := echo.New() + e.GET("/api/orders", listOrders) +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const ginRoute = providers.find((c) => c.contractId === 'http::POST::/api/orders/{param}'); + expect(ginRoute).toBeDefined(); + expect(ginRoute?.symbolName).toBe('createOrder'); + + const echoRoute = providers.find((c) => c.contractId === 'http::GET::/api/orders'); + expect(echoRoute).toBeDefined(); + expect(echoRoute?.symbolName).toBe('listOrders'); + }); + + it('extracts stdlib HandleFunc providers', async () => { + const dir = path.join(tmpDir, 'go-stdlib-provider'); + fs.mkdirSync(path.join(dir, 'cmd'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'cmd', 'server.go'), + ` +package main + +func healthHandler(w http.ResponseWriter, r *http.Request) {} + +func main() { + http.HandleFunc("/api/health", healthHandler) +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const healthRoute = providers.find((c) => c.contractId === 'http::GET::/api/health'); + expect(healthRoute).toBeDefined(); + expect(healthRoute?.symbolName).toBe('healthHandler'); + }); + + it('extracts NestJS controller decorators', async () => { + const dir = path.join(tmpDir, 'nestjs'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'orders.controller.ts'), + ` +import { Controller, Patch } from '@nestjs/common'; + +@Controller('orders') +export class OrdersController { + @Patch(':id') + updateOrder() { + return {}; + } +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const patchRoute = providers.find((c) => c.contractId === 'http::PATCH::/orders/{param}'); + expect(patchRoute).toBeDefined(); + expect(patchRoute?.symbolName).toBe('updateOrder'); + }); }); describe('consumer extraction — fetch patterns', () => { @@ -206,6 +289,91 @@ export const deleteUser = (id: string) => axios.delete(\`/api/users/\${id}\`); consumers.find((c) => c.contractId === 'http::DELETE::/api/users/{param}'), ).toBeDefined(); }); + + it('extracts Python requests calls', async () => { + const dir = path.join(tmpDir, 'python-consumer'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'client.py'), + ` +import requests + +def create_order(): + return requests.post("https://svc.local/api/orders/42", json={"id": 42}) +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect( + consumers.find((c) => c.contractId === 'http::POST::/api/orders/{param}'), + ).toBeDefined(); + }); + + it('extracts Java RestTemplate, WebClient and OkHttp calls', async () => { + const dir = path.join(tmpDir, 'java-consumer'); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'ApiClient.java'), + ` +import org.springframework.http.HttpMethod; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.reactive.function.client.WebClient; +import okhttp3.Request; + +class ApiClient { + void run(RestTemplate restTemplate, WebClient webClient) { + restTemplate.getForObject("/api/users/{id}", String.class, 42); + webClient.method(HttpMethod.PATCH, "/api/users/42"); + new Request.Builder().url("/api/orders/42").build(); + } +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers.find((c) => c.contractId === 'http::GET::/api/users/{param}')).toBeDefined(); + expect( + consumers.find((c) => c.contractId === 'http::PATCH::/api/users/{param}'), + ).toBeDefined(); + expect( + consumers.find((c) => c.contractId === 'http::GET::/api/orders/{param}'), + ).toBeDefined(); + }); + + it('extracts Go stdlib and resty calls', async () => { + const dir = path.join(tmpDir, 'go-consumer'); + fs.mkdirSync(path.join(dir, 'cmd'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'cmd', 'client.go'), + ` +package main + +import ( + "net/http" + + "github.com/go-resty/resty/v2" +) + +func main() { + http.Get("/api/health") + client := resty.New() + client.R().Delete("/api/orders/42") +} +`, + ); + + const contracts = await extractor.extract(null, dir, makeRepo(dir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers.find((c) => c.contractId === 'http::GET::/api/health')).toBeDefined(); + expect( + consumers.find((c) => c.contractId === 'http::DELETE::/api/orders/{param}'), + ).toBeDefined(); + }); }); describe('provider extraction — Laravel', () => { @@ -326,78 +494,6 @@ async def create_user(user: UserCreate): }); }); - describe('interface regex anchoring', () => { - it('skips Feign client interfaces (no @Controller)', async () => { - const dir = path.join(tmpDir, 'feign-skip'); - fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); - fs.writeFileSync( - path.join(dir, 'src/UserClient.java'), - ` -package com.example; -@FeignClient(name = "user-service") -public interface UserClient { - @GetMapping("/users") - List getUsers(); -} -`, - ); - const contracts = await extractor.extract(null, dir, makeRepo(dir)); - expect(contracts.filter((c) => c.role === 'provider')).toHaveLength(0); - }); - - it('does NOT skip when @RestController is present', async () => { - const dir = path.join(tmpDir, 'ctrl-iface'); - fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); - fs.writeFileSync( - path.join(dir, 'src/UserController.java'), - ` -@RestController -@RequestMapping("/api") -public class UserController { - @GetMapping("/users") - public List list() { return null; } -} -`, - ); - const contracts = await extractor.extract(null, dir, makeRepo(dir)); - expect(contracts.filter((c) => c.role === 'provider').length).toBeGreaterThanOrEqual(1); - }); - - it('does NOT false-positive on interface in comments', async () => { - const dir = path.join(tmpDir, 'iface-comment'); - fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); - fs.writeFileSync( - path.join(dir, 'src/Api.java'), - ` -// implements the interface UserApi -public class Api { - @GetMapping("/health") - public String health() { return "ok"; } -} -`, - ); - const contracts = await extractor.extract(null, dir, makeRepo(dir)); - expect(contracts.filter((c) => c.role === 'provider').length).toBeGreaterThanOrEqual(1); - }); - - it('does NOT false-positive on interface in a string', async () => { - const dir = path.join(tmpDir, 'iface-str'); - fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); - fs.writeFileSync( - path.join(dir, 'src/Svc.java'), - ` -public class Svc { - String desc = "implements interface Foo"; - @GetMapping("/status") - public String status() { return desc; } -} -`, - ); - const contracts = await extractor.extract(null, dir, makeRepo(dir)); - expect(contracts.filter((c) => c.role === 'provider').length).toBeGreaterThanOrEqual(1); - }); - }); - describe('path normalization', () => { it('strips trailing slash', async () => { const dir = path.join(tmpDir, 'trailing'); diff --git a/gitnexus/test/unit/group/manifest-extractor.test.ts b/gitnexus/test/unit/group/manifest-extractor.test.ts new file mode 100644 index 000000000..c2c67a33a --- /dev/null +++ b/gitnexus/test/unit/group/manifest-extractor.test.ts @@ -0,0 +1,308 @@ +import { describe, it, expect } from 'vitest'; +import { ManifestExtractor } from '../../../src/core/group/extractors/manifest-extractor.js'; +import type { GroupManifestLink } from '../../../src/core/group/types.js'; + +describe('ManifestExtractor', () => { + const extractor = new ManifestExtractor(); + + it('creates provider + consumer contracts and a cross-link for each manifest link', async () => { + const links: GroupManifestLink[] = [ + { + from: 'hr/payroll/backend', + to: 'hr/hiring/backend', + type: 'topic', + contract: 'employee.hired', + role: 'provider', + }, + ]; + + const result = await extractor.extractFromManifest(links); + + expect(result.contracts).toHaveLength(2); + + const provider = result.contracts.find((c) => c.role === 'provider'); + expect(provider).toBeDefined(); + expect(provider!.contractId).toBe('topic::employee.hired'); + expect(provider!.type).toBe('topic'); + expect(provider!.confidence).toBe(1.0); + + const consumer = result.contracts.find((c) => c.role === 'consumer'); + expect(consumer).toBeDefined(); + expect(consumer!.contractId).toBe('topic::employee.hired'); + + expect(result.crossLinks).toHaveLength(1); + expect(result.crossLinks[0].matchType).toBe('manifest'); + expect(result.crossLinks[0].confidence).toBe(1.0); + expect(result.crossLinks[0].from.repo).toBe('hr/hiring/backend'); + expect(result.crossLinks[0].to.repo).toBe('hr/payroll/backend'); + }); + + it('handles role: consumer (from-repo is consumer)', async () => { + const links: GroupManifestLink[] = [ + { + from: 'sales/admin/bff', + to: 'sales/crm/backend', + type: 'http', + contract: '/api/v2/leads/*', + role: 'consumer', + }, + ]; + + const result = await extractor.extractFromManifest(links); + + const provider = result.contracts.find((c) => c.role === 'provider'); + const consumer = result.contracts.find((c) => c.role === 'consumer'); + + expect(consumer!.contractId).toBe('http::*::/api/v2/leads/*'); + expect(provider!.contractId).toBe('http::*::/api/v2/leads/*'); + + expect(result.crossLinks[0].from.repo).toBe('sales/admin/bff'); + expect(result.crossLinks[0].to.repo).toBe('sales/crm/backend'); + }); + + it('resolves grpc manifest provider by exact method name (no .proto fallback)', async () => { + const links: GroupManifestLink[] = [ + { + from: 'platform/orders', + to: 'platform/auth', + type: 'grpc', + contract: 'auth.AuthService/Login', + role: 'consumer', + }, + ]; + + const dbExecutors = new Map< + string, + (cypher: string, params?: Record) => Promise[]> + >([ + [ + 'platform/auth', + async (_cypher, params) => { + // Exact match on method name. + if (params?.methodName === 'Login') { + return [ + { + uid: 'uid-auth-login', + name: 'Login', + filePath: 'src/auth.proto', + }, + ]; + } + return []; + }, + ], + [ + 'platform/orders', + async (_cypher, params) => { + // No symbol with the exact method name — resolve returns null and + // the consumer contract gets an empty symbolUid, falling back to + // name-based hint at cross-impact time. + if (params?.methodName === 'Login') return []; + return []; + }, + ], + ]); + + const result = await extractor.extractFromManifest(links, dbExecutors); + + const provider = result.contracts.find((c) => c.role === 'provider'); + const consumer = result.contracts.find((c) => c.role === 'consumer'); + + // Provider resolved to the concrete proto symbol. + expect(provider?.symbolUid).toBe('uid-auth-login'); + expect(provider?.symbolRef.filePath).toBe('src/auth.proto'); + + // Consumer falls back to a deterministic synthetic uid + name-based ref. + // The synthetic uid lets the bridge cross-impact query anchor on it + // even when the indexer doesn't expose a matching symbol. + expect(consumer?.symbolUid).toBe('manifest::platform/orders::grpc::auth.AuthService/Login'); + expect(consumer?.symbolRef.name).toBe('auth.AuthService/Login'); + + expect(result.crossLinks[0].to.symbolRef.filePath).toBe('src/auth.proto'); + expect(result.crossLinks[0].from.symbolUid).toBe( + 'manifest::platform/orders::grpc::auth.AuthService/Login', + ); + }); + + it('does NOT resolve grpc manifest to an arbitrary .proto file', async () => { + // Regression test for a previous bug: the extractor had an unconditional + // `OR n.filePath ENDS WITH '.proto'` fallback that returned the first + // proto symbol in the repo, regardless of whether it matched the contract. + const links: GroupManifestLink[] = [ + { + from: 'platform/orders', + to: 'platform/auth', + type: 'grpc', + contract: 'auth.AuthService/Login', + role: 'consumer', + }, + ]; + + const dbExecutors = new Map< + string, + (cypher: string, params?: Record) => Promise[]> + >([ + [ + 'platform/auth', + // Executor returns matches for ANY query (simulates the old buggy + // fallback that returned a random .proto file). The new code must + // only accept a hit when the method/service name matches exactly. + async (_cypher, params) => { + if (params?.methodName === 'Login' || params?.serviceName === 'auth.AuthService') { + return [ + { + uid: 'uid-correct-login', + name: 'Login', + filePath: 'src/auth.proto', + }, + ]; + } + return []; + }, + ], + ['platform/orders', async () => []], + ]); + + const result = await extractor.extractFromManifest(links, dbExecutors); + const provider = result.contracts.find((c) => c.role === 'provider'); + // Must resolve to the correct symbol (not a random proto one). + expect(provider?.symbolUid).toBe('uid-correct-login'); + }); + + it('resolves lib manifest links by exact name only', async () => { + const links: GroupManifestLink[] = [ + { + from: 'platform/web', + to: 'platform/shared-lib', + type: 'lib', + contract: '@platform/contracts', + role: 'consumer', + }, + ]; + + const dbExecutors = new Map< + string, + (cypher: string, params?: Record) => Promise[]> + >([ + [ + 'platform/shared-lib', + async (_cypher, params) => { + if (params?.contract !== '@platform/contracts') return []; + return [ + { + uid: 'uid-lib', + name: '@platform/contracts', + filePath: 'src/index.ts', + }, + ]; + }, + ], + [ + 'platform/web', + async (_cypher, params) => { + if (params?.contract !== '@platform/contracts') return []; + return []; + }, + ], + ]); + + const result = await extractor.extractFromManifest(links, dbExecutors); + + const provider = result.contracts.find((c) => c.role === 'provider'); + const consumer = result.contracts.find((c) => c.role === 'consumer'); + + expect(provider?.symbolUid).toBe('uid-lib'); + // Consumer doesn't have a symbol named exactly '@platform/contracts' — + // exact matching returns null, falling back to the synthetic manifest uid. + expect(consumer?.symbolUid).toBe('manifest::platform/web::lib::@platform/contracts'); + }); + + it('does NOT resolve lib manifest via CONTAINS on name', async () => { + // Regression test: previous CONTAINS fallback would match "react" to + // "react-native" or "@types/react". Exact matching must reject both. + const links: GroupManifestLink[] = [ + { + from: 'web', + to: 'packages/ui', + type: 'lib', + contract: 'react', + role: 'consumer', + }, + ]; + + const dbExecutors = new Map< + string, + (cypher: string, params?: Record) => Promise[]> + >([ + [ + 'packages/ui', + async (_cypher, params) => { + // Executor is called with contract='react'. Only exact matches + // should come back; return only wrong candidates to verify the + // Cypher uses `=` not `CONTAINS`. + if (params?.contract === 'react') { + // Simulated DB returns nothing because it has only "react-native" + // and "@types/react" — neither is an exact match for "react". + return []; + } + return []; + }, + ], + ['web', async () => []], + ]); + + const result = await extractor.extractFromManifest(links, dbExecutors); + const provider = result.contracts.find((c) => c.role === 'provider'); + // No exact match → synthetic manifest uid, not a wrong real one. + expect(provider?.symbolUid).toBe('manifest::packages/ui::lib::react'); + }); + + it('normalizes http contract path for exact Route.name match', async () => { + // Manifest may be written as "/api/orders/" or "api/orders"; both should + // match the canonical "/api/orders" stored in the graph. + const variants = ['/api/orders', '/api/orders/', 'api/orders', '//api//orders']; + for (const raw of variants) { + const links: GroupManifestLink[] = [ + { + from: 'gateway', + to: 'orders-svc', + type: 'http', + contract: raw, + role: 'consumer', + }, + ]; + + let seenParam: string | undefined; + const dbExecutors = new Map< + string, + (cypher: string, params?: Record) => Promise[]> + >([ + [ + 'orders-svc', + async (_cypher, params) => { + seenParam = params?.normalized as string; + return [ + { + uid: 'uid-orders-list', + name: 'listOrders', + filePath: 'src/orders.ts', + }, + ]; + }, + ], + ['gateway', async () => []], + ]); + + const result = await extractor.extractFromManifest(links, dbExecutors); + expect(seenParam).toBe('/api/orders'); + const provider = result.contracts.find((c) => c.role === 'provider'); + expect(provider?.symbolUid).toBe('uid-orders-list'); + } + }); + + it('returns empty for no links', async () => { + const result = await extractor.extractFromManifest([]); + expect(result.contracts).toHaveLength(0); + expect(result.crossLinks).toHaveLength(0); + }); +}); diff --git a/gitnexus/test/unit/group/topic-extractor.test.ts b/gitnexus/test/unit/group/topic-extractor.test.ts index c6a1161a0..bf821de63 100644 --- a/gitnexus/test/unit/group/topic-extractor.test.ts +++ b/gitnexus/test/unit/group/topic-extractor.test.ts @@ -75,8 +75,7 @@ public void handleUserCreated(ConsumerRecord record) { it('test_extract_kafkajs_subscribe_returns_consumer', async () => { writeFile( 'src/consumer.ts', - `await consumer.subscribe({ topic: 'order.placed', fromBeginning: true }); -await consumer.run({ eachMessage: async ({ message }) => {} });`, + `await consumer.subscribe({ topic: 'order.placed', fromBeginning: true });`, ); const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); @@ -101,6 +100,23 @@ await consumer.run({ eachMessage: async ({ message }) => {} });`, }); }); + describe('KafkaJS consumer run', () => { + it('test_extract_kafkajs_consumer_run_eachmessage_returns_consumer', async () => { + writeFile( + 'src/consumer.ts', + `await consumer.subscribe({ topic: 'user.logged-in' }); +await consumer.run({ eachMessage: async () => {} });`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('topic::user.logged-in'); + expect(consumers[0].meta.broker).toBe('kafka'); + }); + }); + describe('RabbitMQ — Java', () => { it('test_extract_rabbit_listener_returns_consumer', async () => { writeFile( @@ -174,6 +190,62 @@ public void processOrder(OrderMessage msg) {}`, }); }); + describe('JetStream', () => { + it('test_extract_jetstream_publish_returns_provider', async () => { + writeFile('src/stream.go', `js.Publish("orders.created", payload)`); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + + expect(producers).toHaveLength(1); + expect(producers[0].contractId).toBe('topic::orders.created'); + expect(producers[0].meta.broker).toBe('nats'); + }); + + it('test_extract_jetstream_subscribe_returns_consumer', async () => { + writeFile('src/stream.go', `js.Subscribe("orders.created", handler)`); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('topic::orders.created'); + expect(consumers[0].meta.broker).toBe('nats'); + }); + }); + + describe('Python NATS', () => { + it('test_extract_python_nats_subscribe_returns_consumer', async () => { + writeFile( + 'src/subscriber.py', + `nc = await nats.connect() +await nc.subscribe("orders.created", cb=handler)`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('topic::orders.created'); + expect(consumers[0].meta.broker).toBe('nats'); + }); + + it('test_extract_python_nats_publish_returns_provider', async () => { + writeFile( + 'src/publisher.py', + `nc = await nats.connect() +await nc.publish("orders.created", payload)`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + + expect(producers).toHaveLength(1); + expect(producers[0].contractId).toBe('topic::orders.created'); + expect(producers[0].meta.broker).toBe('nats'); + }); + }); + describe('NATS', () => { it('test_extract_nats_subscribe_go_returns_consumer', async () => { writeFile( @@ -248,6 +320,96 @@ partConsumer, _ := consumer.ConsumePartition("inventory.update", 0, sarama.Offse expect(consumers[0].contractId).toBe('topic::inventory.update'); expect(consumers[0].meta.broker).toBe('kafka'); }); + + it('test_extract_sarama_sync_producer_returns_provider', async () => { + writeFile( + 'internal/publisher.go', + `package publisher +producer, _ := sarama.NewSyncProducer(brokers, cfg) +producer.SendMessage(&sarama.ProducerMessage{Topic: "inventory.update"})`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + + expect(producers).toHaveLength(1); + expect(producers[0].contractId).toBe('topic::inventory.update'); + expect(producers[0].meta.broker).toBe('kafka'); + }); + + it('test_extract_sarama_async_producer_returns_provider', async () => { + writeFile( + 'internal/publisher.go', + `package publisher +producer, _ := sarama.NewAsyncProducer(brokers, cfg) +producer.Input() <- &sarama.ProducerMessage{Topic: "inventory.update"}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + + expect(producers).toHaveLength(1); + expect(producers[0].contractId).toBe('topic::inventory.update'); + expect(producers[0].meta.broker).toBe('kafka'); + }); + + it('test_extract_sarama_producer_in_loop_captures_all_topics', async () => { + // Regression: a for loop that constructs multiple ProducerMessage + // literals inside a single NewSyncProducer scope. The previous + // regex anchored on NewSyncProducer and captured only the first + // Topic within 300 chars, silently dropping the rest. + writeFile( + 'internal/multi-publisher.go', + `package publisher + +func publishAll(producer sarama.SyncProducer, items []Item) error { + _, _ = sarama.NewSyncProducer(brokers, cfg) + for _, item := range items { + msg1 := &sarama.ProducerMessage{Topic: "order.created"} + msg2 := &sarama.ProducerMessage{Topic: "order.shipped"} + _ = msg1 + _ = msg2 + } + return nil +}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + const topics = producers.map((c) => c.contractId).sort(); + // Both topics must appear (exact set match to catch any duplicates). + expect(topics).toEqual(['topic::order.created', 'topic::order.shipped']); + }); + + it('test_extract_kafka_go_writer_returns_provider', async () => { + writeFile( + 'internal/writer.go', + `package publisher +writer := &kafka.Writer{Topic: "inventory.update"}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const producers = contracts.filter((c) => c.role === 'provider'); + + expect(producers).toHaveLength(1); + expect(producers[0].contractId).toBe('topic::inventory.update'); + expect(producers[0].meta.broker).toBe('kafka'); + }); + + it('test_extract_kafka_go_reader_returns_consumer', async () => { + writeFile( + 'internal/reader.go', + `package consumer +reader := kafka.NewReader(kafka.ReaderConfig{Topic: "inventory.update"})`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('topic::inventory.update'); + expect(consumers[0].meta.broker).toBe('kafka'); + }); }); describe('Kafka — Python', () => { @@ -309,5 +471,16 @@ await consumer.subscribe({ topic: 'order.placed' });`, expect(producers).toHaveLength(2); expect(consumers).toHaveLength(1); }); + + it('test_extract_ignores_go_test_files', async () => { + writeFile( + 'src/orders_test.go', + `consumer.ConsumePartition("fake-topic", 0, sarama.OffsetNewest)`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + + expect(contracts).toEqual([]); + }); }); });