From d8b14cfa7f30a9f5fafce74c721b295f1c36eb0e Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Mon, 8 Jun 2026 19:37:48 +0000 Subject: [PATCH] =?UTF-8?q?feat(ingestion):=20U4=20=E2=80=94=20emit=20Basi?= =?UTF-8?q?cBlock=20+=20CFG=20within=20scope-resolution=20(#2081)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emit persisted BasicBlock nodes + CFG edges from each ParsedFile's worker-built cfgSideChannel, INSIDE scope-resolution's Phase-4 graph emission — the last point where the worker-built CFGs are loaded (emitParsedFiles carries the channel; the disk store is cleared right after the orchestrator returns). This is the architecture the doc-review corrected to: a standalone post-`mro` phase (the issue's literal subtask) provably reads empty data (KTD1). - cfg/emit.ts (new): pure emitFileCfgs(graph, cfgs, maxEdgesPerFunction, onWarn). BasicBlock id = `BasicBlock:::` (KTD3 — funcStart disambiguates blocks across functions in one file; no `name` column). CFG edge = CodeRelation type 'CFG' with the edge KIND (seq/cond-true/…) in `reason` (kinds can't be their own edge type). Per- function edge cap stops at the cap and warns with the dropped count — no silent truncation (R6/KTD6). - run.ts: pdg-gated emit pass over emitParsedFiles after emitPostResolutionEdges (store still live); RunScopeResolutionInput gains pdg + pdgMaxEdgesPerFunction. - phase.ts: thread ctx.options.pdg / pdgMaxEdgesPerFunction into the call. - pipeline.ts: PipelineOptions.pdgMaxEdgesPerFunction. 6 tests: node/edge shape (KTD3 id, no name, type='CFG', kind in reason), cross-function id uniqueness, AC2 reachability-from-ENTRY property, the edge cap's no-silent-truncation contract, and empty-input no-op. Flag-off byte-identity + full runPipelineFromRepo round-trip land in U7. Build clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- gitnexus/src/core/ingestion/cfg/emit.ts | 106 +++++++++++ gitnexus/src/core/ingestion/pipeline.ts | 6 + .../scope-resolution/pipeline/phase.ts | 3 + .../scope-resolution/pipeline/run.ts | 41 +++++ .../test/integration/cfg/cfg-emit.test.ts | 166 ++++++++++++++++++ 5 files changed, 322 insertions(+) create mode 100644 gitnexus/src/core/ingestion/cfg/emit.ts create mode 100644 gitnexus/test/integration/cfg/cfg-emit.test.ts diff --git a/gitnexus/src/core/ingestion/cfg/emit.ts b/gitnexus/src/core/ingestion/cfg/emit.ts new file mode 100644 index 000000000..5a25ae9ef --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/emit.ts @@ -0,0 +1,106 @@ +/** + * cfg/emit.ts (issue #2081, M1) — serialized side-channel → graph. + * + * Pure helper: given a file's per-function CFGs (off `ParsedFile.cfgSideChannel`, + * produced by the worker in U3), emit one persisted `BasicBlock` node per block + * and one `CFG` edge per edge into the {@link KnowledgeGraph}. Invoked from + * scope-resolution (run.ts Phase 4) while the disk-backed ParsedFile store is + * still live — the only window where the worker-built CFGs are loaded (KTD1/ + * KTD5). Default (`--pdg` off) runs never call this, so the emitted graph stays + * byte-identical to a pre-#2081 run. + * + * BasicBlock id: `BasicBlock:::` + * (KTD3). The `functionStartLine` segment disambiguates blocks across multiple + * functions in one file (each function's block indices restart at 0); blocks + * carry no `name` (the BasicBlock table has no such column). The edge KIND + * (`seq`/`cond-true`/…) rides in the relationship `reason` — CFG edges are + * values of the single `CodeRelation` table's `type` column (`'CFG'`), so the + * kind cannot be its own edge type and is queried via `reason`. + */ +import type { KnowledgeGraph } from '../../graph/types.js'; +import { generateId } from '../../../lib/utils.js'; +import type { FunctionCfg } from './types.js'; + +/** + * Default per-function CFG edge cap. A pathological generated function could + * otherwise emit an unbounded edge set; the cap bounds graph growth and is + * overridable via `--pdg` options. `0` (in options) means "use this default". + */ +export const DEFAULT_MAX_CFG_EDGES_PER_FUNCTION = 5000; + +export interface CfgEmitResult { + blocks: number; + edges: number; + /** Edges dropped because a function's edge count exceeded the cap. */ + droppedEdges: number; + /** Number of functions that hit the cap. */ + cappedFunctions: number; +} + +const basicBlockId = (filePath: string, functionStartLine: number, blockIndex: number): string => + `BasicBlock:${filePath}:${functionStartLine}:${blockIndex}`; + +/** + * Emit BasicBlock nodes + CFG edges for every function CFG in `cfgs`. + * + * `maxEdgesPerFunction` caps edges per function. On overflow we stop emitting + * that function's remaining edges and call `onWarn` naming the dropped count — + * no silent truncation (KTD6/R6). Block nodes are always fully emitted (their + * count is bounded by the function's statement count); only edges are capped. + */ +export function emitFileCfgs( + graph: KnowledgeGraph, + cfgs: readonly FunctionCfg[], + maxEdgesPerFunction: number = DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + onWarn?: (message: string) => void, +): CfgEmitResult { + const result: CfgEmitResult = { blocks: 0, edges: 0, droppedEdges: 0, cappedFunctions: 0 }; + const cap = maxEdgesPerFunction > 0 ? maxEdgesPerFunction : Infinity; + + for (const cfg of cfgs) { + const { filePath, functionStartLine } = cfg; + + for (const b of cfg.blocks) { + graph.addNode({ + id: basicBlockId(filePath, functionStartLine, b.index), + label: 'BasicBlock', + properties: { + name: '', // BasicBlock has no name column; identified by id + span + filePath, + startLine: b.startLine, + endLine: b.endLine, + text: b.text, + }, + }); + result.blocks++; + } + + let emittedForFn = 0; + for (const e of cfg.edges) { + if (emittedForFn >= cap) { + const dropped = cfg.edges.length - emittedForFn; + result.droppedEdges += dropped; + result.cappedFunctions++; + onWarn?.( + `[cfg] ${filePath}:${functionStartLine}: per-function CFG edge cap ` + + `(${maxEdgesPerFunction}) reached — dropped ${dropped} of ${cfg.edges.length} edges`, + ); + break; + } + const sourceId = basicBlockId(filePath, functionStartLine, e.from); + const targetId = basicBlockId(filePath, functionStartLine, e.to); + graph.addRelationship({ + id: generateId('CFG', `${sourceId}->${targetId}:${e.kind}`), + type: 'CFG', + sourceId, + targetId, + confidence: 1.0, + reason: e.kind, // CfgEdgeKind (seq/cond-true/loop-back/…) — queryable + }); + result.edges++; + emittedForFn++; + } + } + + return result; +} diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 7898b4b18..26cedac65 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -64,6 +64,12 @@ export interface PipelineOptions { * over-cap functions are skipped (no CFG emitted for them). */ pdgMaxFunctionLines?: number; + /** + * Per-function CFG edge cap for the scope-resolution emit step + * (`undefined`/0 ⇒ the emit default). Over-cap functions stop at the cap and + * log a structured drop warning (no silent truncation). + */ + pdgMaxEdgesPerFunction?: number; /** * Request parsing with the worker pool disabled. The sequential parser was * removed — the worker pool is the sole parse path — so setting this now diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index 3c3f688ec..22afc6778 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -340,6 +340,9 @@ export const scopeResolutionPhase: PipelinePhase = { prebuiltNodeLookup: sharedNodeLookup, preExtractedParsedFiles: preExtractedByPath, scopeIndexStorePath: parsedFileStorePath, + // CFG/PDG emission (#2081 M1) — opt-in; off ⇒ byte-identical graph. + pdg: ctx.options?.pdg === true, + pdgMaxEdgesPerFunction: ctx.options?.pdgMaxEdgesPerFunction, recordResolutionOutcome: (outcome) => { resolutionOutcomes.push(outcome); }, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index 88c9f5f70..da274c0e6 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -34,6 +34,8 @@ import { extractParsedFile } from '../../scope-extractor-bridge.js'; import { finalizeScopeModel } from '../../finalize-orchestrator.js'; import { resolveReferenceSites, type ResolveStats } from '../../resolve-references.js'; import { buildGraphNodeLookup } from '../graph-bridge/node-lookup.js'; +import { emitFileCfgs, DEFAULT_MAX_CFG_EDGES_PER_FUNCTION } from '../../cfg/emit.js'; +import type { FunctionCfg } from '../../cfg/types.js'; import { resolveDefGraphId } from '../graph-bridge/ids.js'; import { buildPopulatedMethodDispatch } from '../graph-bridge/method-dispatch.js'; import { propagateImportedReturnTypes } from '../passes/imported-return-types.js'; @@ -252,6 +254,15 @@ interface RunScopeResolutionInput { * cache miss is safe (the provider re-parses). */ readonly treeCache?: { get(filePath: string): unknown }; + /** + * CFG/PDG opt-in (#2081 M1). When true, emit BasicBlock nodes + CFG edges + * from each ParsedFile's worker-built `cfgSideChannel` during Phase-4 graph + * emission (while the disk store is still live). Default/false ⇒ no CFG + * nodes or edges and a byte-identical graph. + */ + readonly pdg?: boolean; + /** Per-function CFG edge cap (0/undefined ⇒ {@link DEFAULT_MAX_CFG_EDGES_PER_FUNCTION}). */ + readonly pdgMaxEdgesPerFunction?: number; /** * Optional graph-node lookup built ONCE by the caller and shared across * every language pass. `buildGraphNodeLookup` scans the whole graph and is @@ -679,6 +690,36 @@ export function runScopeResolution( }); } + // ── CFG/PDG emission (#2081 M1, opt-in via `--pdg`) ────────────────────── + // Emit BasicBlock nodes + CFG edges from each ParsedFile's worker-built + // `cfgSideChannel`, HERE — the last point inside scope-resolution where the + // ParsedFiles are still loaded (`emitParsedFiles` carries the channel; the + // disk store is cleared right after this orchestrator returns, see phase.ts). + // A post-`mro` phase would read empty data (KTD1). Off by default ⇒ zero + // BasicBlock/CFG nodes/edges and a byte-identical graph. + if (input.pdg === true) { + let cfgBlocks = 0; + let cfgEdges = 0; + for (const pf of emitParsedFiles) { + const cfgs = pf.cfgSideChannel as readonly FunctionCfg[] | undefined; + if (cfgs === undefined || cfgs.length === 0) continue; + const emitted = emitFileCfgs( + graph, + cfgs, + input.pdgMaxEdgesPerFunction ?? DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + input.onWarn, + ); + cfgBlocks += emitted.blocks; + cfgEdges += emitted.edges; + } + if (cfgBlocks > 0) { + logger.debug( + `[scope-resolution] CFG emit (lang=${provider.language}): ` + + `${cfgBlocks} BasicBlock nodes, ${cfgEdges} CFG edges`, + ); + } + } + if (PROF) { const tEnd = process.hrtime.bigint(); const ns = (a: bigint, b: bigint): number => Number(b - a) / 1_000_000; diff --git a/gitnexus/test/integration/cfg/cfg-emit.test.ts b/gitnexus/test/integration/cfg/cfg-emit.test.ts new file mode 100644 index 000000000..4c4bc25cb --- /dev/null +++ b/gitnexus/test/integration/cfg/cfg-emit.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, vi } from 'vitest'; +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { collectFunctionCfgs } from '../../../src/core/ingestion/cfg/collect.js'; +import { emitFileCfgs } from '../../../src/core/ingestion/cfg/emit.js'; +import { getProvider } from '../../../src/core/ingestion/languages/index.js'; +import { SupportedLanguages } from '../../../src/config/supported-languages.js'; +import type { CfgVisitor, FunctionCfg } from '../../../src/core/ingestion/cfg/types.js'; +import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js'; +import type { KnowledgeGraph } from '../../../src/core/graph/types.js'; + +// U4 — emit BasicBlock nodes + CFG edges from the worker-built side-channel +// (R5, R6). Tests the pure emit helper against a recording graph: id shape +// (KTD3), edge `type`/`reason`, the AC2 reachability property, and the +// per-function edge cap's no-silent-truncation contract. The flag-gated +// run.ts wiring + full runPipelineFromRepo round-trip are covered in U7. + +interface RecordedNode { + id: string; + label: string; + properties: Record; +} +interface RecordedRel { + id: string; + type: string; + sourceId: string; + targetId: string; + reason: string; +} + +function recordingGraph(): { graph: KnowledgeGraph; nodes: RecordedNode[]; rels: RecordedRel[] } { + const nodes: RecordedNode[] = []; + const rels: RecordedRel[] = []; + const graph = { + addNode: (n: RecordedNode) => nodes.push(n), + addRelationship: (r: RecordedRel) => rels.push(r), + } as unknown as KnowledgeGraph; + return { graph, nodes, rels }; +} + +function tsRoot(code: string): SyntaxNode { + const parser = new Parser(); + parser.setLanguage(TypeScript.typescript); + return parser.parse(code).rootNode; +} + +const visitor = (): CfgVisitor => { + const v = getProvider(SupportedLanguages.TypeScript).cfgVisitor; + if (!v) throw new Error('no cfgVisitor'); + return v; +}; + +const cfgsOf = (code: string, filePath = 'f.ts'): readonly FunctionCfg[] => + collectFunctionCfgs(tsRoot(code), visitor(), filePath).cfgs; + +describe('U4 — emitFileCfgs node/edge shape', () => { + it('emits BasicBlock nodes (KTD3 id, no name) + CFG edges carrying the kind in reason', () => { + const cfgs = cfgsOf(`function f(x: number) { if (x) { a(); } else { b(); } }`, 'src/f.ts'); + const { graph, nodes, rels } = recordingGraph(); + const r = emitFileCfgs(graph, cfgs); + + expect(r.blocks).toBe(nodes.length); + expect(r.edges).toBe(rels.length); + expect(nodes.length).toBeGreaterThan(0); + + // every node is a BasicBlock with the KTD3 id `BasicBlock:::` + for (const n of nodes) { + expect(n.label).toBe('BasicBlock'); + expect(n.id).toMatch(/^BasicBlock:src\/f\.ts:\d+:\d+$/); + expect(n.properties.filePath).toBe('src/f.ts'); + expect(n.properties.name).toBe(''); // no name column + } + // every edge is type 'CFG' and its reason is a CfgEdgeKind + const kinds = new Set(rels.map((e) => e.reason)); + expect(rels.every((e) => e.type === 'CFG')).toBe(true); + expect(kinds.has('cond-true')).toBe(true); + expect(kinds.has('cond-false')).toBe(true); + }); + + it('block ids are unique across two functions in the same file (funcStart disambiguates)', () => { + const cfgs = cfgsOf(`function a() { x(); }\nfunction b() { y(); }`, 'm.ts'); + const { graph, nodes } = recordingGraph(); + emitFileCfgs(graph, cfgs); + const ids = nodes.map((n) => n.id); + expect(new Set(ids).size).toBe(ids.length); // no collisions + }); +}); + +describe('U4 — AC2: every BasicBlock is reachable from its function ENTRY', () => { + // Fixtures deliberately contain no dead code, so the reachability closure + // from each function's ENTRY (block index 0) must cover all of its blocks. + const FIXTURE = ` + function branch(x: number) { if (x) { a(); } else { b(); } c(); } + function loop(xs: number[]) { for (const y of xs) { use(y); } done(); } + function multi(x: number) { + switch (x) { case 1: one(); break; default: other(); } + tail(); + } + `; + + it('reachability closure from ENTRY covers every emitted block per function', () => { + const cfgs = cfgsOf(FIXTURE, 'r.ts'); + const { graph, nodes, rels } = recordingGraph(); + emitFileCfgs(graph, cfgs); + + const adj = new Map(); + for (const e of rels) + (adj.get(e.sourceId) ?? adj.set(e.sourceId, []).get(e.sourceId)!).push(e.targetId); + + for (const cfg of cfgs) { + const entryId = `BasicBlock:r.ts:${cfg.functionStartLine}:${cfg.entryIndex}`; + const fnNodeIds = nodes + .map((n) => n.id) + .filter((id) => id.startsWith(`BasicBlock:r.ts:${cfg.functionStartLine}:`)); + // BFS from ENTRY + const seen = new Set([entryId]); + const stack = [entryId]; + while (stack.length) { + const n = stack.pop() as string; + for (const nx of adj.get(n) ?? []) if (!seen.has(nx)) (seen.add(nx), stack.push(nx)); + } + for (const id of fnNodeIds) { + expect(seen.has(id), `${id} unreachable from ENTRY`).toBe(true); + } + } + }); +}); + +describe('U4 — per-function edge cap (R6, no silent truncation)', () => { + it('stops at the cap, records the dropped count, and warns', () => { + const cfgs = cfgsOf(`function f(x: number) { if (x) { a(); } else { b(); } c(); }`); + const total = cfgs[0].edges.length; + expect(total).toBeGreaterThan(2); + + const { graph, rels } = recordingGraph(); + const onWarn = vi.fn(); + const r = emitFileCfgs(graph, cfgs, 2, onWarn); + + expect(rels.length).toBe(2); // emitted exactly the cap + expect(r.droppedEdges).toBe(total - 2); + expect(r.cappedFunctions).toBe(1); + expect(onWarn).toHaveBeenCalledTimes(1); + expect(onWarn.mock.calls[0][0]).toContain(`dropped ${total - 2} of ${total}`); + }); + + it('cap of 0 means unlimited (emits every edge, no warning)', () => { + const cfgs = cfgsOf(`function f(x: number) { if (x) { a(); } else { b(); } }`); + const { graph, rels } = recordingGraph(); + const onWarn = vi.fn(); + const r = emitFileCfgs(graph, cfgs, 0, onWarn); + expect(rels.length).toBe(cfgs[0].edges.length); + expect(r.droppedEdges).toBe(0); + expect(onWarn).not.toHaveBeenCalled(); + }); +}); + +describe('U4 — flag-off / empty input emits nothing', () => { + it('no functions ⇒ zero nodes and edges', () => { + const { graph, nodes, rels } = recordingGraph(); + const r = emitFileCfgs(graph, []); + expect(nodes).toHaveLength(0); + expect(rels).toHaveLength(0); + expect(r.blocks).toBe(0); + expect(r.edges).toBe(0); + }); +});