diff --git a/docs/guides/microservices-grpc.md b/docs/guides/microservices-grpc.md index afe6b2499..b09fd55e3 100644 --- a/docs/guides/microservices-grpc.md +++ b/docs/guides/microservices-grpc.md @@ -82,6 +82,9 @@ matching: bm25_threshold: 0.7 embedding_threshold: 0.65 max_candidates_per_step: 3 + # Exclude noisy paths from cross-link matching (contracts are still extracted) + exclude_links_paths: [/ping, /health, /healthcheck] + exclude_links_param_only_paths: true ``` Field notes (schema in [`types.ts`](../../gitnexus/src/core/group/types.ts)): @@ -91,7 +94,9 @@ Field notes (schema in [`types.ts`](../../gitnexus/src/core/group/types.ts)): - `repos` — a mapping from **group path** (a logical name you choose; can be a hierarchy like `backend/orders`) to **registry name** (the name shown by `npx gitnexus list`). Both sides appear throughout the tooling: contract rows use the group path; `@/` routes tools to a single member. - `links` — optional manifest escape hatch, one entry per explicit cross-repo contract. Validated by the parser: `from` and `to` must be known repo paths, `type` must be one of `http | grpc | topic | lib | custom`, and `role` must be `provider | consumer`. - `detect` — toggles per extractor family. Defaults (set in `config-parser.ts`) turn `http`, `grpc`, `topics`, and `shared_libs` on; disable the ones you don't use to speed up sync. -- `matching` — thresholds for the matching cascade. The exact match is always run; other strategies depend on indexer state. +- `matching` — thresholds for the matching cascade. The exact match is always run; other strategies depend on indexer state. Two optional fields reduce false-positive cross-links in large groups: + - `exclude_links_paths` — list of HTTP paths to exclude from cross-link matching (default `[]`). Contracts at these paths are still extracted and visible in the registry, but they don't produce cross-repo links. Useful for health-check endpoints (`/ping`, `/health`) that every service exposes. Trailing slashes are normalized. + - `exclude_links_param_only_paths` — when `true`, exclude routes where every segment is `{param}` (e.g. `/{param}`, `/{param}/{param}`) from cross-link matching (default `false`). Mixed routes like `/users/{param}` are not affected. ### 3. Sync the group diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index bd803981c..cf2141311 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -19,6 +19,8 @@ const DEFAULT_MATCHING = { bm25_threshold: 0.7, embedding_threshold: 0.65, max_candidates_per_step: 3, + exclude_links_paths: [] as string[], + exclude_links_param_only_paths: false, }; export function parseGroupConfig(yamlContent: string): GroupConfig { diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index ec793968b..07f88a61a 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -1,4 +1,4 @@ -import type { StoredContract, CrossLink } from './types.js'; +import type { StoredContract, CrossLink, MatchingConfig } from './types.js'; export interface MatchResult { matched: CrossLink[]; @@ -14,6 +14,43 @@ function isGrpcWildcard(cid: string): boolean { return cid.startsWith('grpc::') && cid.endsWith('/*'); } +/** + * Detect HTTP contracts that are too generic or infrastructure-level to + * produce meaningful cross-repo links. These are still extracted (useful + * for documentation / route maps) but excluded from cross-link matching. + * + * Two categories: + * 1. Health-check / readiness endpoints — every service has one, matching + * them produces N×M false links. + * 2. Param-only paths — routes like `/{param}` or `/{param}/{param}` that + * collapse to a single catch-all after normalization. These match any + * service with a similar shape, producing false positives. + * + * Both are configurable via matching.exclude_links_paths and + * matching.exclude_links_param_only_paths in group.yaml. + */ +function buildNoisyContractFilter( + matchingConfig?: MatchingConfig, +): (contractId: string) => boolean { + const excludePaths = matchingConfig?.exclude_links_paths?.length + ? new Set(matchingConfig.exclude_links_paths.map((p) => p.replace(/\/+$/, ''))) + : new Set(); + const excludeParamOnly = matchingConfig?.exclude_links_param_only_paths === true; + + return function isNoisyHttpContract(contractId: string): boolean { + if (!contractId.startsWith('http::')) return false; + const parts = contractId.split('::'); + if (parts.length < 3) return false; + const pathPart = parts.slice(2).join('::').replace(/\/+$/, ''); + if (excludePaths.has(pathPart)) return true; + if (excludeParamOnly) { + const segments = pathPart.split('/').filter(Boolean); + if (segments.length > 0 && segments.every((s) => s === '{param}')) return true; + } + return false; + }; +} + export function normalizeContractId(id: string): string { const colonIdx = id.indexOf('::'); if (colonIdx === -1) return id; @@ -91,8 +128,12 @@ function findMatchingKeys(contractId: string, index: Map { - const providers = contracts.filter((c) => c.role === 'provider'); +export function buildProviderIndex( + contracts: StoredContract[], + matchingConfig?: MatchingConfig, +): Map { + const isNoisy = buildNoisyContractFilter(matchingConfig); + const providers = contracts.filter((c) => c.role === 'provider' && !isNoisy(c.contractId)); const index = new Map(); for (const p of providers) { const key = normalizeContractId(p.contractId); @@ -106,11 +147,14 @@ export function buildProviderIndex(contracts: StoredContract[]): Map, + matchingConfig?: MatchingConfig, ): MatchResult { - const index = providerIndex ?? buildProviderIndex(contracts); + const isNoisy = buildNoisyContractFilter(matchingConfig); + const index = providerIndex ?? buildProviderIndex(contracts, matchingConfig); - // Skip gRPC wildcard consumers — they go to wildcard pass only - const consumers = contracts.filter((c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId)); + const consumers = contracts.filter( + (c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId) && !isNoisy(c.contractId), + ); const matched: CrossLink[] = []; const matchedConsumerIds = new Set(); @@ -155,6 +199,7 @@ export function runExactMatch( // normalUnmatched: contracts that weren't matched in exact pass const normalUnmatched = contracts.filter((c) => { if (isGrpcWildcard(c.contractId)) return false; // excluded from exact, handled separately + if (isNoisy(c.contractId)) return false; // excluded from matching — don't surface as unmatched const id = `${c.repo}::${c.contractId}`; return c.role === 'provider' ? !matchedProviderIds.has(id) : !matchedConsumerIds.has(id); }); diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index aa6a781a5..5380b9867 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -103,6 +103,8 @@ matching: bm25_threshold: 0.7 embedding_threshold: 0.65 max_candidates_per_step: 3 + # exclude_links_paths: [/ping, /health, /healthcheck] + # exclude_links_param_only_paths: false `; await fsp.writeFile(path.join(groupDir, 'group.yaml'), template, 'utf-8'); return groupDir; diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index af7c3e686..bd2590ecd 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -208,7 +208,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } - const { matched, unmatched } = runExactMatch(autoContracts); + const { matched, unmatched } = runExactMatch(autoContracts, undefined, config.matching); // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a // manifest-declared link can also emit a matchType:'exact' CrossLink with the diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index 793d3d0ad..64ce53143 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -33,6 +33,24 @@ export interface MatchingConfig { bm25_threshold: number; embedding_threshold: number; max_candidates_per_step: number; + /** + * HTTP paths to exclude from cross-link matching. Contracts at these paths + * are still extracted and visible in the registry, but they don't produce + * cross-repo links. Useful for health-check endpoints (`/ping`, `/health`) + * that every service exposes and would otherwise create N×M false links. + * Trailing slashes are normalized before comparison. + * @default [] + */ + exclude_links_paths?: string[]; + /** + * When `true`, exclude HTTP routes where every path segment is `{param}` + * (e.g. `/{param}`, `/{param}/{param}`) from cross-link matching. Mixed + * routes like `/users/{param}` are not affected. These param-only routes + * collapse to a single catch-all after normalization and produce false + * positives across unrelated services. + * @default false + */ + exclude_links_param_only_paths?: boolean; } export interface SymbolRef { diff --git a/gitnexus/test/fixtures/group/group.yaml b/gitnexus/test/fixtures/group/group.yaml index 3a7fa5a31..1fd2ad45c 100644 --- a/gitnexus/test/fixtures/group/group.yaml +++ b/gitnexus/test/fixtures/group/group.yaml @@ -22,3 +22,5 @@ matching: bm25_threshold: 0.7 embedding_threshold: 0.65 max_candidates_per_step: 3 + # exclude_links_paths: [/ping, /health, /healthcheck] + # exclude_links_param_only_paths: false diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index 50878ce7d..7bfc2cf8f 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -60,6 +60,8 @@ repos: expect(config.packages).toEqual({}); expect(config.detect.http).toBe(true); expect(config.matching.bm25_threshold).toBe(0.7); + expect(config.matching.exclude_links_paths).toEqual([]); + expect(config.matching.exclude_links_param_only_paths).toBe(false); }); it('throws on missing required fields', () => { diff --git a/gitnexus/test/unit/group/matching.test.ts b/gitnexus/test/unit/group/matching.test.ts index c5713d909..5c29bb4a6 100644 --- a/gitnexus/test/unit/group/matching.test.ts +++ b/gitnexus/test/unit/group/matching.test.ts @@ -5,7 +5,7 @@ import { buildProviderIndex, runWildcardMatch, } from '../../../src/core/group/matching.js'; -import type { StoredContract } from '../../../src/core/group/types.js'; +import type { StoredContract, MatchingConfig } from '../../../src/core/group/types.js'; describe('normalizeContractId', () => { it('lowercases HTTP method', () => { @@ -403,3 +403,227 @@ describe('runWildcardMatch', () => { expect(matched[0].contractId).toBe('grpc::com.example.UserService/*'); }); }); + +describe('buildNoisyContractFilter (via runExactMatch)', () => { + const makeContract = ( + id: string, + role: 'provider' | 'consumer', + repo: string, + ): StoredContract => ({ + contractId: id, + type: 'http', + role, + symbolUid: `uid-${repo}-${id}`, + symbolRef: { filePath: `src/${repo}.ts`, name: `fn-${id}` }, + symbolName: `fn-${id}`, + confidence: 0.8, + meta: {}, + repo, + }); + + it('exclude_links_paths prevents cross-links for configured paths', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/ping'], + exclude_links_param_only_paths: false, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/ping', 'provider', 'backend'), + makeContract('http::GET::/ping', 'consumer', 'frontend'), + makeContract('http::GET::/api/users', 'provider', 'backend'), + makeContract('http::GET::/api/users', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched, unmatched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(1); + expect(matched[0].contractId).toBe('http::GET::/api/users'); + }); + + it('excluded providers do not appear in matched', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/health'], + exclude_links_param_only_paths: false, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/health', 'provider', 'backend'), + makeContract('http::GET::/health', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(0); + }); + + it('excluded contracts do not appear in unmatched', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/ping'], + exclude_links_param_only_paths: false, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/ping', 'provider', 'backend'), + makeContract('http::GET::/ping', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched, unmatched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(0); + expect(unmatched).toHaveLength(0); + }); + + it('exclude_links_param_only_paths filters /{param} and /{param}/{param}', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: [], + exclude_links_param_only_paths: true, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/{param}', 'provider', 'backend'), + makeContract('http::GET::/{param}', 'consumer', 'frontend'), + makeContract('http::GET::/{param}/{param}', 'provider', 'backend'), + makeContract('http::GET::/{param}/{param}', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched, unmatched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(0); + expect(unmatched).toHaveLength(0); + }); + + it('mixed routes like /users/{param} are NOT excluded by param_only', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: [], + exclude_links_param_only_paths: true, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/users/{param}', 'provider', 'backend'), + makeContract('http::GET::/users/{param}', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(1); + expect(matched[0].contractId).toBe('http::GET::/users/{param}'); + }); + + it('default config (no exclusions) produces no filtering', () => { + const contracts: StoredContract[] = [ + makeContract('http::GET::/ping', 'provider', 'backend'), + makeContract('http::GET::/ping', 'consumer', 'frontend'), + makeContract('http::GET::/{param}', 'provider', 'backend'), + makeContract('http::GET::/{param}', 'consumer', 'frontend'), + ]; + + const { matched } = runExactMatch(contracts); + + expect(matched).toHaveLength(2); + }); + + it('trailing slash on contractId still matches configured exclusion', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/ping'], + exclude_links_param_only_paths: false, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/ping/', 'provider', 'backend'), + makeContract('http::GET::/ping/', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched, unmatched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(0); + expect(unmatched).toHaveLength(0); + }); + + it('root path exclusion ["/"] suppresses http::GET::/ contracts', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/'], + exclude_links_param_only_paths: false, + }; + + const contracts: StoredContract[] = [ + makeContract('http::GET::/', 'provider', 'backend'), + makeContract('http::GET::/', 'consumer', 'frontend'), + makeContract('http::GET::/api/users', 'provider', 'backend'), + makeContract('http::GET::/api/users', 'consumer', 'frontend'), + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched, unmatched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(1); + expect(matched[0].contractId).toBe('http::GET::/api/users'); + expect(unmatched).toHaveLength(0); + }); + + it('non-HTTP contracts are never filtered', () => { + const matchingConfig: MatchingConfig = { + bm25_threshold: 0.7, + embedding_threshold: 0.65, + max_candidates_per_step: 3, + exclude_links_paths: ['/ping'], + exclude_links_param_only_paths: true, + }; + + const contracts: StoredContract[] = [ + { + contractId: 'topic::events.ping', + type: 'topic', + role: 'provider', + symbolUid: 'uid-backend-topic', + symbolRef: { filePath: 'src/backend.ts', name: 'fn-topic' }, + symbolName: 'fn-topic', + confidence: 0.8, + meta: {}, + repo: 'backend', + }, + { + contractId: 'topic::events.ping', + type: 'topic', + role: 'consumer', + symbolUid: 'uid-frontend-topic', + symbolRef: { filePath: 'src/frontend.ts', name: 'fn-topic' }, + symbolName: 'fn-topic', + confidence: 0.8, + meta: {}, + repo: 'frontend', + }, + ]; + + const providerIndex = buildProviderIndex(contracts, matchingConfig); + const { matched } = runExactMatch(contracts, providerIndex, matchingConfig); + + expect(matched).toHaveLength(1); + }); +});