From d1971cf95330fb57837a4f27335b71027c91a32c Mon Sep 17 00:00:00 2001 From: articultur <547692615@qq.com> Date: Sat, 3 Oct 2026 01:25:35 -0700 Subject: [PATCH] fix(communities): omit memberships for filtered singleton communities (#3447) --- gitnexus/src/cli/skill-gen.ts | 39 ++-- .../src/core/ingestion/community-processor.ts | 18 +- .../test/unit/community-processor.test.ts | 69 +++++++ gitnexus/test/unit/skill-gen.test.ts | 195 ++++++++++++++++++ 4 files changed, 301 insertions(+), 20 deletions(-) diff --git a/gitnexus/src/cli/skill-gen.ts b/gitnexus/src/cli/skill-gen.ts index f1dd45c3b..ed7f58232 100644 --- a/gitnexus/src/cli/skill-gen.ts +++ b/gitnexus/src/cli/skill-gen.ts @@ -114,21 +114,21 @@ export const generateSkillFiles = async ( } } - if (!communityResult || !communityResult.memberships.length) { + const memberships = communityResult?.rawMemberships ?? communityResult?.memberships ?? []; + if (!communityResult || !memberships.length) { console.log('\n Skills: no communities detected, skipping skill generation'); return { skills: [], outputPath: outputDir }; } console.log('\n Generating repo-specific skills...'); - // Step 1: Build communities from memberships (not the filtered communities array). - // The community processor skips singletons from its communities array but memberships - // include ALL assignments. For repos with sparse CALLS edges, the communities array - // can be empty while memberships still has useful groupings. + // Step 1: Use raw assignments for the fallback when all communities were + // filtered as singletons. Same-folder aggregation can still produce skills + // for these sparse graphs without emitting dangling MEMBER_OF edges. const communities = communityResult.communities.length > 0 ? communityResult.communities - : buildCommunitiesFromMemberships(communityResult.memberships, graph, repoPath); + : buildCommunitiesFromMemberships(memberships, graph, repoPath); const aggregated = aggregateCommunities(communities); @@ -145,11 +145,8 @@ export const generateSkillFiles = async ( } // Step 3: Build lookup maps - const membershipsByComm = buildMembershipMap(communityResult.memberships); - const nodeIdToCommunityLabel = buildNodeCommunityLabelMap( - communityResult.memberships, - communities, - ); + const membershipsByComm = buildMembershipMap(memberships); + const nodeIdToCommunityLabel = buildNodeCommunityLabelMap(memberships, communities); // Step 4: Ensure the shared project-skill root exists. Never clear it: it // also contains user-authored and standard GitNexus skills. @@ -185,7 +182,7 @@ export const generateSkillFiles = async ( const entryPoints = gatherEntryPoints(members); // Gather execution flows - const flows = gatherFlows(community.rawIds, processResult?.processes || []); + const flows = gatherFlows(community.rawIds, members, processResult?.processes || []); // Gather cross-community connections const connections = gatherCrossConnections( @@ -529,14 +526,26 @@ const gatherEntryPoints = (members: MemberSymbol[]): MemberSymbol[] => { /** * @brief Gather execution flows touching this community * @param {string[]} rawIds - Raw community IDs for this aggregated community + * @param {MemberSymbol[]} members - Member symbols, including raw singleton assignments * @param {ProcessNode[]} processes - All detected processes - * @returns {ProcessNode[]} Processes whose communities intersect rawIds, sorted by stepCount + * @returns {ProcessNode[]} Processes matching the community IDs or member symbols, sorted by stepCount */ -const gatherFlows = (rawIds: string[], processes: ProcessNode[]): ProcessNode[] => { +const gatherFlows = ( + rawIds: string[], + members: MemberSymbol[], + processes: ProcessNode[], +): ProcessNode[] => { const rawIdSet = new Set(rawIds); + const memberIds = new Set(members.map((member) => member.id)); return processes - .filter((proc) => proc.communities.some((cid) => rawIdSet.has(cid))) + .filter( + (proc) => + proc.communities.some((cid) => rawIdSet.has(cid)) || + // Filtered singleton communities are absent from process metadata, + // but their symbols still participate in detected execution traces. + proc.trace.some((nodeId) => memberIds.has(nodeId)), + ) .sort((a, b) => b.stepCount - a.stepCount); }; diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 77b66d73b..233326e03 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -176,7 +176,10 @@ export interface CommunityMembership { export interface CommunityDetectionResult { communities: CommunityNode[]; + /** Assignments to retained communities, safe to emit as MEMBER_OF edges. */ memberships: CommunityMembership[]; + /** All Leiden assignments, including filtered singletons. Optional for legacy producers. */ + rawMemberships?: CommunityMembership[]; stats: { totalCommunities: number; modularity: number; @@ -235,6 +238,7 @@ export const processCommunities = async ( return { communities: [], memberships: [], + rawMemberships: [], stats: { totalCommunities: 0, modularity: 0, @@ -267,13 +271,16 @@ export const processCommunities = async ( onProgress?.('Creating membership edges...', 80); - // Step 4: Create membership mappings + // Step 4: Preserve all assignments for skill generation, but only emit + // memberships for retained communities with a corresponding graph node. + const retainedCommunityIds = new Set(communityNodes.map((community) => community.id)); const memberships: CommunityMembership[] = []; + const rawMemberships: CommunityMembership[] = []; Object.entries(details.communities).forEach(([nodeId, communityNum]) => { - memberships.push({ - nodeId, - communityId: `comm_${communityNum}`, - }); + const communityId = `comm_${communityNum}`; + const membership = { nodeId, communityId }; + rawMemberships.push(membership); + if (retainedCommunityIds.has(communityId)) memberships.push(membership); }); onProgress?.('Community detection complete!', 100); @@ -281,6 +288,7 @@ export const processCommunities = async ( return { communities: communityNodes, memberships, + rawMemberships, stats: { totalCommunities: details.count, modularity: details.modularity, diff --git a/gitnexus/test/unit/community-processor.test.ts b/gitnexus/test/unit/community-processor.test.ts index 2e207a22a..e9b3d07aa 100644 --- a/gitnexus/test/unit/community-processor.test.ts +++ b/gitnexus/test/unit/community-processor.test.ts @@ -448,7 +448,76 @@ module.exports = { const second = await processCommunities(graph); expect(second.memberships).toEqual(first.memberships); + expect(second.rawMemberships).toEqual(first.rawMemberships); expect(second.stats.modularity).toBe(first.stats.modularity); }); }); }); + +describe('community membership integrity', () => { + it('returns empty raw and retained memberships for an empty graph', async () => { + const result = await processCommunities(createKnowledgeGraph()); + + expect(result.communities).toEqual([]); + expect(result.memberships).toEqual([]); + expect(result.rawMemberships).toEqual([]); + expect(result.stats).toMatchObject({ totalCommunities: 0, modularity: 0, nodesProcessed: 0 }); + }); + + it('retains connected members without emitting memberships for a filtered singleton', async () => { + const graph = createKnowledgeGraph(); + for (const id of ['fn:a', 'fn:b', 'fn:c', 'fn:singleton']) { + graph.addNode(makeNode(id, id.slice(3))); + } + graph.addNode(makeNode('file:target', 'target', 'File')); + graph.addRelationship(makeRel('rel:ab', 'fn:a', 'fn:b')); + graph.addRelationship(makeRel('rel:bc', 'fn:b', 'fn:c')); + graph.addRelationship(makeRel('rel:ca', 'fn:c', 'fn:a')); + // The symbol enters the projection, but its non-symbol target does not. + // Leiden therefore partitions it into a singleton, which is not emitted. + graph.addRelationship(makeRel('rel:singleton', 'fn:singleton', 'file:target')); + + const result = await processCommunities(graph, undefined, { engine: 'graphology' }); + const communityIds = new Set(result.communities.map((community) => community.id)); + + expect(result.communities).toHaveLength(1); + expect(result.communities[0].symbolCount).toBe(3); + expect(result.stats).toMatchObject({ totalCommunities: 2, nodesProcessed: 4 }); + expect(result.memberships.map((membership) => membership.nodeId).sort()).toEqual([ + 'fn:a', + 'fn:b', + 'fn:c', + ]); + expect(result.memberships.every((membership) => communityIds.has(membership.communityId))).toBe( + true, + ); + expect(result.rawMemberships?.map((membership) => membership.nodeId)).toEqual([ + 'fn:a', + 'fn:b', + 'fn:c', + 'fn:singleton', + ]); + expect( + result.rawMemberships?.filter((membership) => communityIds.has(membership.communityId)), + ).toEqual(result.memberships); + }); + + it('returns no memberships when every detected community is a filtered singleton', async () => { + const graph = createKnowledgeGraph(); + graph.addNode(makeNode('file:target', 'target', 'File')); + for (const id of ['fn:a', 'fn:b']) { + graph.addNode(makeNode(id, id.slice(3))); + graph.addRelationship(makeRel(`rel:${id}`, id, 'file:target')); + } + + const result = await processCommunities(graph, undefined, { engine: 'graphology' }); + + expect(result.communities).toEqual([]); + expect(result.memberships).toEqual([]); + expect(result.rawMemberships).toEqual([ + { nodeId: 'fn:a', communityId: 'comm_0' }, + { nodeId: 'fn:b', communityId: 'comm_1' }, + ]); + expect(result.stats).toMatchObject({ totalCommunities: 2, nodesProcessed: 2 }); + }); +}); diff --git a/gitnexus/test/unit/skill-gen.test.ts b/gitnexus/test/unit/skill-gen.test.ts index 0ffb60ec0..57167ce20 100644 --- a/gitnexus/test/unit/skill-gen.test.ts +++ b/gitnexus/test/unit/skill-gen.test.ts @@ -22,6 +22,9 @@ import type { ProcessDetectionResult, } from '../../src/core/ingestion/process-processor.js'; import type { PipelineResult } from '../../src/types/pipeline.js'; +import { communitiesPhase } from '../../src/core/ingestion/pipeline-phases/communities.js'; +import { processesPhase } from '../../src/core/ingestion/pipeline-phases/processes.js'; +import type { PhaseResult } from '../../src/core/ingestion/pipeline-phases/types.js'; // ============================================================================ // FIXTURE HELPERS @@ -135,6 +138,25 @@ function buildPipelineResult(opts: { }; } +/** Run the real community phase, including graph node and membership edge emission. */ +async function detectCommunities(graph: KnowledgeGraph, repoPath: string): Promise { + const { communityResult } = await communitiesPhase.execute( + { graph, repoPath, onProgress: () => {}, pipelineStart: Date.now() }, + new Map([['structure', { phaseName: 'structure', output: { totalFiles: 0 }, durationMs: 0 }]]), + ); + return { + graph, + repoPath, + totalFileCount: 0, + communityResult, + resolutionOutcomes: [], + usedWorkerPool: false, + reparsedFileCount: 0, + scopeExtractionFailures: [], + unavailableScopeLanguageFiles: 0, + }; +} + // ============================================================================ // TESTS — RETURN VALUES // ============================================================================ @@ -407,6 +429,179 @@ describe('generateSkillFiles — return values', () => { expect(result.skills[0].label).toBe('Auth'); }); + it('generates a folder skill from real singleton assignments without dangling membership edges', async () => { + const graph = createKnowledgeGraph(); + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + for (const name of ['gamma', 'alpha', 'beta']) { + graph.addNode( + makeNode(`fn:${name}`, name, 'Function', `${tmpDir}/src/auth/${name}.ts`, 1, true), + ); + // The File target admits the symbol to the projection, then is excluded + // itself, leaving a singleton in the real Leiden result. + graph.addRelationship(makeRel(`rel:${name}`, `fn:${name}`, 'file:target', 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ + name: 'gitnexus-area-auth', + label: 'Auth', + symbolCount: 3, + fileCount: 3, + }); + expect(pipeline.communityResult?.communities).toEqual([]); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toEqual([ + { nodeId: 'fn:alpha', communityId: 'comm_0' }, + { nodeId: 'fn:beta', communityId: 'comm_1' }, + { nodeId: 'fn:gamma', communityId: 'comm_2' }, + ]); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + for (const name of ['alpha', 'beta', 'gamma']) { + expect(content).toContain(name); + expect(content).toContain(`src/auth/${name}.ts`); + } + }); + + it('preserves execution flows for real singleton fallback skills', async () => { + const graph = createKnowledgeGraph(); + // Large-graph projection prunes the degree-one endpoints of each chain, + // leaving only its middle function as a singleton community. + for (let i = 0; i < 10_001; i++) { + graph.addNode(makeNode(`fn:unused${i}`, `unused${i}`, 'Function', 'src/unused.ts', 1, false)); + } + for (let i = 0; i < 4; i++) { + const folder = i < 3 ? 'auth' : 'billing'; + for (const role of ['handle', 'middle', 'end']) { + const name = `${role}${i}`; + graph.addNode( + makeNode( + `fn:${name}`, + name, + 'Function', + `${tmpDir}/src/${folder}/${name}.ts`, + 1, + role === 'handle', + ), + ); + } + graph.addRelationship(makeRel(`rel:handle${i}`, `fn:handle${i}`, `fn:middle${i}`, 'CALLS')); + graph.addRelationship(makeRel(`rel:middle${i}`, `fn:middle${i}`, `fn:end${i}`, 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const { processResult } = await processesPhase.execute( + { graph, repoPath: tmpDir, onProgress: () => {}, pipelineStart: Date.now() }, + new Map>([ + ['structure', { phaseName: 'structure', output: { totalFiles: 0 }, durationMs: 0 }], + [ + 'communities', + { + phaseName: 'communities', + output: { communityResult: pipeline.communityResult }, + durationMs: 0, + }, + ], + ['routes', { phaseName: 'routes', output: { routeRegistry: new Map() }, durationMs: 0 }], + ['tools', { phaseName: 'tools', output: { toolDefs: [] }, durationMs: 0 }], + ]), + ); + pipeline.processResult = processResult; + + expect(pipeline.communityResult?.communities).toEqual([]); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(4); + expect(processResult.processes).toHaveLength(4); + expect(processResult.processes.every((process) => process.communities.length === 0)).toBe(true); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ label: 'Auth', symbolCount: 3 }); + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + expect(content).toContain('## Execution Flows'); + for (const process of processResult.processes) { + if (process.entryPointId === 'fn:handle3') { + expect(content).not.toContain(process.heuristicLabel); + } else { + expect(content).toContain(process.heuristicLabel); + } + } + }); + + it.each([0, 2])('skips real singleton fallback below threshold (%i symbols)', async (count) => { + const graph = createKnowledgeGraph(); + if (count > 0) { + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + } + for (let i = 0; i < count; i++) { + graph.addNode( + makeNode(`fn:n${i}`, `n${i}`, 'Function', `${tmpDir}/src/auth/f${i}.ts`, 1, true), + ); + graph.addRelationship(makeRel(`rel:${i}`, `fn:n${i}`, 'file:target', 'CALLS')); + } + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toEqual([]); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(count); + expect(pipeline.communityResult?.memberships).toEqual([]); + expect([...graph.iterRelationships()].filter((rel) => rel.type === 'MEMBER_OF')).toEqual([]); + }); + + it('keeps real retained skills and membership edges separate from filtered singletons', async () => { + const graph = createKnowledgeGraph(); + for (const name of ['alpha', 'beta', 'gamma', 'singleton']) { + graph.addNode( + makeNode(`fn:${name}`, name, 'Function', `${tmpDir}/src/auth/${name}.ts`, 1, true), + ); + } + graph.addNode(makeNode('file:target', 'target', 'File', `${tmpDir}/target.ts`, 1, false)); + graph.addRelationship(makeRel('rel:ab', 'fn:alpha', 'fn:beta', 'CALLS')); + graph.addRelationship(makeRel('rel:bc', 'fn:beta', 'fn:gamma', 'CALLS')); + graph.addRelationship(makeRel('rel:ca', 'fn:gamma', 'fn:alpha', 'CALLS')); + graph.addRelationship(makeRel('rel:singleton', 'fn:singleton', 'file:target', 'CALLS')); + + const pipeline = await detectCommunities(graph, tmpDir); + const result = await generateSkillFiles(tmpDir, 'TestProject', pipeline); + + expect(result.skills).toHaveLength(1); + expect(result.skills[0]).toMatchObject({ label: 'Auth', symbolCount: 3, fileCount: 3 }); + expect(pipeline.communityResult?.rawMemberships).toHaveLength(4); + expect(pipeline.communityResult?.memberships.map((membership) => membership.nodeId)).toEqual([ + 'fn:alpha', + 'fn:beta', + 'fn:gamma', + ]); + const membershipEdges = [...graph.iterRelationships()].filter( + (rel) => rel.type === 'MEMBER_OF', + ); + expect(membershipEdges).toHaveLength(3); + for (const edge of membershipEdges) { + expect(graph.getNode(edge.sourceId)).toBeDefined(); + expect(graph.getNode(edge.targetId)?.label).toBe('Community'); + expect(edge.sourceId).not.toBe('fn:singleton'); + } + const content = await fs.readFile( + path.join(result.outputPath, result.skills[0].name, 'SKILL.md'), + 'utf-8', + ); + expect(content).not.toContain('singleton'); + for (const name of ['alpha', 'beta', 'gamma']) { + expect(content).toContain(`src/auth/${name}.ts`); + } + }); + /** * When processResult is undefined, the generator should still work * without crashing — it simply has no execution flows.