From 474789dd99cec97a5f13c3635ccc216594cadbb5 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 15 Apr 2026 08:16:52 +0100 Subject: [PATCH] perf(group/manifest): parallelize and memoize symbol resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous implementation ran 2N sequential Cypher round-trips per manifest (one for provider side, one for consumer, awaited in-order per link). For manifests with tens of links this dominated syncGroup latency in groups with many declared cross-repo contracts. Changes: - Resolve provider + consumer in parallel per link (Promise.all). - Resolve all links in parallel (outer Promise.all over links.map). Each repo's executor pool is independent, so cross-repo fan-out scales with the number of distinct repos in the manifest. - Memoize by (repo, type, contract). Manifests frequently declare the same contract from both directions or across sibling groups, so duplicate triples now hit the DB once instead of 2× per link. Correctness: - resolveSymbol is a pure LIMIT 1 read, so caching + concurrent invocation is safe. - Iteration order over links is preserved in the final contracts / crossLinks arrays — result shape is identical. Test: - New test asserts that two links sharing (repo, type, contract) produce exactly one DB call per distinct repo-tuple. --- .../group/extractors/manifest-extractor.ts | 49 ++++++++++++++++--- .../unit/group/manifest-extractor.test.ts | 30 ++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 4c0d737b7..83f5cab5e 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -79,17 +79,50 @@ export class ManifestExtractor { links: GroupManifestLink[], dbExecutors?: Map, ): Promise { + // Resolve all (repo, link) pairs in parallel. The previous sequential + // await-per-link produced 2N round-trips; parallel resolution uses the + // per-repo executor pool directly and scales linearly with manifest size. + // + // Memoization: a manifest can list the same contract multiple times + // (e.g. a consumer and provider declaration, or cross-referenced groups). + // Key on (repo, type, contract) — the canonical input to the Cypher + // query — so duplicate links resolve to one DB hit. + type ResolvedSymbol = { filePath: string; name: string; uid: string } | null; + const resolveCache = new Map>(); + const resolveOnce = (repo: string, link: GroupManifestLink): Promise => { + const key = `${repo}\u0000${link.type}\u0000${link.contract}`; + let pending = resolveCache.get(key); + if (!pending) { + pending = this.resolveSymbol(repo, link, dbExecutors); + resolveCache.set(key, pending); + } + return pending; + }; + + const perLink = await Promise.all( + links.map(async (link) => { + const contractId = this.buildContractId(link.type, link.contract); + const providerRepo = link.role === 'provider' ? link.from : link.to; + const consumerRepo = link.role === 'provider' ? link.to : link.from; + const [providerSymbol, consumerSymbol] = await Promise.all([ + resolveOnce(providerRepo, link), + resolveOnce(consumerRepo, link), + ]); + return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol }; + }), + ); + const contracts: StoredContract[] = []; const crossLinks: CrossLink[] = []; - for (const link of links) { - const contractId = this.buildContractId(link.type, link.contract); - - const providerRepo = link.role === 'provider' ? link.from : link.to; - const consumerRepo = link.role === 'provider' ? link.to : link.from; - - const providerSymbol = await this.resolveSymbol(providerRepo, link, dbExecutors); - const consumerSymbol = await this.resolveSymbol(consumerRepo, link, dbExecutors); + for (const { + link, + contractId, + providerRepo, + consumerRepo, + providerSymbol, + consumerSymbol, + } of perLink) { const providerRef = providerSymbol || { filePath: '', name: link.contract }; const consumerRef = consumerSymbol || { filePath: '', name: link.contract }; // When the resolver finds a real graph symbol we keep its uid, otherwise diff --git a/gitnexus/test/unit/group/manifest-extractor.test.ts b/gitnexus/test/unit/group/manifest-extractor.test.ts index 42ed86b84..59725dc86 100644 --- a/gitnexus/test/unit/group/manifest-extractor.test.ts +++ b/gitnexus/test/unit/group/manifest-extractor.test.ts @@ -583,4 +583,34 @@ describe('ManifestExtractor', () => { expect(result.contracts).toHaveLength(0); expect(result.crossLinks).toHaveLength(0); }); + + it('memoizes repeated (repo, type, contract) resolutions so each tuple hits the DB once', async () => { + const calls: Array<{ repo: string; cypher: string }> = []; + const execFor = (repo: string) => async (cypher: string) => { + calls.push({ repo, cypher }); + return [{ uid: `uid::${repo}`, name: 'handler', filePath: 'src/h.ts' }]; + }; + + const dbExecutors = new Map Promise[]>>([ + ['svc/a', execFor('svc/a')], + ['svc/b', execFor('svc/b')], + ]); + + // Two links declare the same (repo, type, contract) triple on each side, + // so naive sequential resolution would run 4 queries; memoization collapses + // to 2 (one per distinct repo tuple). + const link: GroupManifestLink = { + from: 'svc/b', + to: 'svc/a', + type: 'http', + contract: 'GET::/api/orders', + role: 'consumer', + }; + + await extractor.extractFromManifest([link, { ...link }], dbExecutors); + + // One resolution per distinct (repo, type, contract) — not per (link × side). + expect(calls).toHaveLength(2); + expect(new Set(calls.map((c) => c.repo))).toEqual(new Set(['svc/a', 'svc/b'])); + }); });