feat(ingestion): U3 — worker CFG build + cfgSideChannel + cache coherence (#2081)

Run the CFG visitor in the parse worker (where the AST lives), serialize the
per-function CFG onto a new ParsedFile.cfgSideChannel, and keep it coherent
across the disk-backed store and the warm/durable parse cache (R3, R4).

- gitnexus-shared parsed-file.ts: add `cfgSideChannel?: unknown` as a DISTINCT
  field from captureSideChannel (different producer/consumer/lifecycle; plain
  JSON data — blocks/edges deliberately lack the `nodeId` the store's interning
  reviver keys on, so no mis-interning).
- cfg/types.ts + visitors/typescript.ts: add CfgVisitor.isFunction so the worker
  enumerates functions (and applies the line budget) by a cheap node-type test.
- cfg/collect.ts (new): collectFunctionCfgs walks the tree, builds one CFG per
  function (nested included), applies maxFunctionLines (over-cap = skipped).
- language-provider.ts: add `cfgVisitor?: CfgVisitor<SyntaxNode>` hook;
  typescript.ts attaches it to both the TS and JS providers (shared grammar).
- parse-worker.ts: read pdg + pdgMaxFunctionLines from workerData (read once at
  init — the worker never sees PipelineOptions), gate the build, attach
  cfgSideChannel alongside captureSideChannel.
- parse-cache.ts: bump SCHEMA_BUMP 4→5 (ParsedFile shape changed) and fold the
  pdg flag into computeChunkHash so a pdg-off cached chunk is NOT reused on a
  --pdg run (the #2038-class warm-cache trap). Default path keeps its keys.
- worker-pool.ts + parse-impl.ts + pipeline.ts: thread pdg/pdgMaxFunctionLines
  PipelineOptions → WorkerPoolOptions → workerData, and into the chunk-hash key.

9 boundary tests: collect contract, JSON round-trip identity (no AST leakage),
the pdg cache-key guard, the line-cap skip, and the no-visitor gate. Full CFG
suite (U1+U2+U3) green; build clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gergo Magyar 2026-06-08 19:26:58 +00:00
parent d33c69f799
commit 15bb02abbc
12 changed files with 302 additions and 9 deletions

View file

@ -98,4 +98,26 @@ export interface ParsedFile {
* side effects — the contract default) leave this undefined.
*/
readonly captureSideChannel?: unknown;
/**
* Per-function control-flow graphs for this file (#2081 M1, PDG/taint
* substrate). A DISTINCT field from {@link captureSideChannel} — different
* producer, consumer, and lifecycle: the worker builds it from the
* tree-sitter AST via `LanguageProvider.cfgVisitor` (only on a `--pdg` run),
* and scope-resolution emits BasicBlock nodes + CFG edges from it while the
* disk-backed ParsedFile store is still live (it is NOT a capture-time
* marker the resolver restores into module maps). Kept separate so a future
* change to either channel's shape invalidates independently.
*
* Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md.
* Concretely it is a `readonly FunctionCfg[]` (see
* `core/ingestion/cfg/types.ts`) — plain JSON-serializable data (no AST
* refs, no class instances) so it round-trips through the parse cache and
* the `parsedfile-store` (whose interning reviver keys on `nodeId`, which
* these blocks/edges deliberately lack).
*
* Optional: `undefined` on non-`--pdg` runs and for languages with no
* `cfgVisitor` — the default for every run today.
*/
readonly cfgSideChannel?: unknown;
}

View file

@ -0,0 +1,55 @@
/**
* collectFunctionCfgs (issue #2081, M1).
*
* Walks a parsed file's tree-sitter tree and builds one {@link FunctionCfg} per
* CFG-bearing function via the language's {@link CfgVisitor}. Runs IN THE PARSE
* WORKER (where the AST lives — KTD1/KTD7); the result rides on
* `ParsedFile.cfgSideChannel` across the worker→main boundary.
*
* Nested functions are enumerated independently — each gets its own CFG, and
* appears as an opaque straight-line block in its enclosing function's CFG (the
* visitor does not descend into nested function bodies). `maxFunctionLines`
* bounds per-function cost: a function whose source span exceeds the cap is
* skipped (and counted) rather than walked, so a pathological mega-function
* cannot blow up worker time/memory. A cap of `0` means no limit.
*/
import type { SyntaxNode } from '../utils/ast-helpers.js';
import type { CfgVisitor, FunctionCfg } from './types.js';
export interface CollectedCfgs {
readonly cfgs: readonly FunctionCfg[];
/** Functions skipped for exceeding `maxFunctionLines` (0 ⇒ none skipped). */
readonly skipped: number;
}
export function collectFunctionCfgs(
root: SyntaxNode,
visitor: CfgVisitor<SyntaxNode>,
filePath: string,
maxFunctionLines = 0,
): CollectedCfgs {
const cfgs: FunctionCfg[] = [];
let skipped = 0;
const stack: SyntaxNode[] = [root];
while (stack.length) {
const node = stack.pop() as SyntaxNode;
if (visitor.isFunction(node)) {
const lines = node.endPosition.row - node.startPosition.row + 1;
if (maxFunctionLines > 0 && lines > maxFunctionLines) {
skipped++;
} else {
const cfg = visitor.buildFunctionCfg(node, filePath);
if (cfg) cfgs.push(cfg);
}
}
// Descend regardless (a skipped mega-function may still contain small
// nested functions that are worth a CFG of their own).
for (let i = node.namedChildCount - 1; i >= 0; i--) {
const child = node.namedChild(i);
if (child) stack.push(child);
}
}
return { cfgs, skipped };
}

View file

@ -61,4 +61,13 @@ export interface FunctionCfg {
*/
export interface CfgVisitor<TNode = unknown> {
buildFunctionCfg(fnNode: TNode, filePath: string): FunctionCfg | undefined;
/**
* Whether `node` is a CFG-bearing function this visitor handles. Lets the
* worker enumerate functions (and apply the per-function line budget) by a
* cheap node-type test, instead of attempting to build a CFG for every AST
* node. `buildFunctionCfg` still re-checks, so this is purely an optimization
* + the seam the line-budget hooks into.
*/
isFunction(node: TNode): boolean;
}

View file

@ -512,9 +512,14 @@ function buildFunctionCfg(fnNode: SyntaxNode, filePath: string): FunctionCfg | u
return builder.finish();
}
/** Whether a node is a TS/JS function this visitor builds a CFG for. */
function isFunction(node: SyntaxNode): boolean {
return TS_FUNCTION_TYPES.has(node.type);
}
/** The TS/JS CFG visitor (shared by TypeScript and JavaScript). */
export function createTypeScriptCfgVisitor(): CfgVisitor<SyntaxNode> {
return { buildFunctionCfg };
return { buildFunctionCfg, isFunction };
}
export { TS_FUNCTION_TYPES };

View file

@ -35,6 +35,7 @@ import type { MethodExtractor } from './method-types.js';
import type { VariableExtractor } from './variable-types.js';
import type { ImportResolverFn } from './import-resolvers/types.js';
import type { SyntaxNode } from './utils/ast-helpers.js';
import type { CfgVisitor } from './cfg/types.js';
import type { NodeLabel } from 'gitnexus-shared';
// ── Shared type aliases ────────────────────────────────────────────────────
@ -332,6 +333,17 @@ interface LanguageProviderConfig {
*/
readonly collectCaptureSideChannel?: (filePath: string) => unknown;
/**
* Per-language control-flow-graph builder (#2081 M1, PDG/taint substrate).
* Invoked IN THE PARSE WORKER (where the AST lives) for each function node,
* gated on the `--pdg` opt-in; the resulting per-function CFGs are serialized
* onto `ParsedFile.cfgSideChannel` and emitted as BasicBlock nodes + CFG
* edges during scope-resolution. `TNode` is `SyntaxNode` for the tree-sitter
* languages. Default: undefined (language has no CFG support yet — TS/JS are
* the M1 set).
*/
readonly cfgVisitor?: CfgVisitor<SyntaxNode>;
/**
* Interpret a raw `@import.statement` capture group into a `ParsedImport`.
* The central finalize algorithm resolves `ParsedImport.targetRaw` to a

View file

@ -16,6 +16,7 @@ import {
javascriptClassConfig,
} from '../class-extractors/configs/typescript-javascript.js';
import type { SyntaxNode } from '../utils/ast-helpers.js';
import { createTypeScriptCfgVisitor } from '../cfg/visitors/typescript.js';
import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js';
import { tsExportChecker } from '../export-detection.js';
import { createImportResolver } from '../import-resolvers/resolver-factory.js';
@ -351,6 +352,8 @@ export const typescriptProvider = defineLanguage({
// canonical capture vocabulary in ./typescript/query.ts
// (TYPESCRIPT_SCOPE_QUERY constant).
emitScopeCaptures: emitTsScopeCaptures,
// CFG/PDG substrate (#2081 M1) — runs in the worker on a --pdg run.
cfgVisitor: createTypeScriptCfgVisitor(),
interpretImport: interpretTsImport,
interpretTypeBinding: interpretTsTypeBinding,
bindingScopeFor: tsBindingScopeFor,
@ -412,6 +415,8 @@ export const javascriptProvider = defineLanguage({
// JSDoc type bindings) live in ./javascript/captures.ts.
// See ./javascript/index.ts for the full per-module rationale.
emitScopeCaptures: emitJsScopeCaptures,
// CFG/PDG substrate (#2081 M1) — TS and JS share the same grammar family.
cfgVisitor: createTypeScriptCfgVisitor(),
interpretImport: interpretJsImport,
interpretTypeBinding: interpretJsTypeBinding,
bindingScopeFor: jsBindingScopeFor,

View file

@ -448,6 +448,10 @@ export async function runChunkedParseAndResolve(
// Initialized below before the chunk loop (same deferred-init pattern
// as `parsedFileStorePath`); this closure only runs from the loop.
durableParsedFileStoragePath: durableParsedFileDir,
// CFG/PDG opt-in (#2081 M1) — baked into each worker's workerData so the
// worker builds + attaches cfgSideChannel. Off by default.
pdg: options?.pdg === true,
pdgMaxFunctionLines: options?.pdgMaxFunctionLines,
// Fan each chunk across the whole pool (#worker-idle): without this a
// chunk smaller than the 8 MB sub-batch cap became a single job on a
// single worker. Honors an explicit `subBatchMaxBytes` / env override.
@ -724,7 +728,7 @@ export async function runChunkedParseAndResolve(
filePath: f.path,
contentHash: fileContentHash(f.content),
}));
chunkHash = computeChunkHash(entries);
chunkHash = computeChunkHash(entries, options?.pdg === true);
}
const cachedRaw =

View file

@ -50,6 +50,20 @@ export interface PipelineOptions {
* to retain those nodes under `skipGraphPhases`.
*/
skipGraphPhases?: boolean;
/**
* Build the control-flow-graph / PDG substrate (#2081 M1, opt-in via `--pdg`).
* Off by default: workers skip all CFG work and emit no `cfgSideChannel`, and
* scope-resolution emits no BasicBlock nodes or CFG edges — so the default
* graph is byte-identical to a pre-#2081 run. Folded into the parse-cache key
* so a pdg-off warm cache is not reused on a `--pdg` run.
*/
pdg?: boolean;
/**
* Per-function source-line cap for worker-side CFG construction
* (`undefined`/0 ⇒ no cap). Bounds the cost of a pathological mega-function;
* over-cap functions are skipped (no CFG emitted for them).
*/
pdgMaxFunctionLines?: number;
/**
* Request parsing with the worker pool disabled. The sequential parser was
* removed — the worker pool is the sole parse path — so setting this now

View file

@ -109,6 +109,7 @@ import {
persistDurableParsedFileShardSync,
} from '../../../storage/parsedfile-store.js';
import { extractLaravelRoutes, type ExtractedRoute } from '../route-extractors/laravel.js';
import { collectFunctionCfgs } from '../cfg/collect.js';
import { logger } from '../../logger.js';
export type { ExtractedRoute } from '../route-extractors/laravel.js';
@ -134,6 +135,18 @@ const DURABLE_PARSED_FILE_STORAGE_PATH: string | undefined = (
)?.durableParsedFileStoragePath;
let shardSeq = 0;
// ── PDG/CFG opt-in (#2081 M1) ───────────────────────────────────────────────
// Read ONCE at worker init from `workerData` (the worker never sees
// PipelineOptions — config arrives via the pool factory's `workerData`, see
// KTD7 / U5). When `pdg` is set, the worker builds a per-function control-flow
// graph from the tree-sitter AST (where it lives) and serializes it onto
// `ParsedFile.cfgSideChannel`. Off ⇒ no CFG work and no field — the default for
// every run today. `pdgMaxFunctionLines` bounds per-function CFG cost
// (0/undefined ⇒ no cap; see collectFunctionCfgs).
const PDG_ENABLED: boolean = (workerData as { pdg?: boolean } | undefined)?.pdg === true;
const PDG_MAX_FUNCTION_LINES: number =
(workerData as { pdgMaxFunctionLines?: number } | undefined)?.pdgMaxFunctionLines ?? 0;
// ── Bootstrap-stage diagnostics (#1741) ────────────────────────────────────
// When GITNEXUS_WORKER_BOOTSTRAP=1 (or --verbose sets GITNEXUS_VERBOSE), each
// worker reports its startup stage timings to stderr — which the pool tees
@ -1201,9 +1214,26 @@ const processFileGroup = (
// copy — scopes/defs are carried by reference) to attach the field rather
// than mutate the frozen object.
const sideChannel = provider.collectCaptureSideChannel?.(file.path);
result.parsedFiles.push(
sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile,
);
let withChannels =
sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile;
// CFG side-channel (#2081 M1): build the per-function control-flow graph
// here, where the tree-sitter AST is still in hand, and attach it as plain
// serializable data. Only on a --pdg run and only for languages with a
// cfgVisitor (TS/JS in M1). The same disk-store/warm-cache machinery that
// carries captureSideChannel carries this — its coherence rests on the
// SCHEMA_BUMP + the pdg-folded chunk-hash key (see parse-cache.ts).
if (PDG_ENABLED && provider.cfgVisitor) {
const { cfgs } = collectFunctionCfgs(
tree.rootNode,
provider.cfgVisitor,
file.path,
PDG_MAX_FUNCTION_LINES,
);
if (cfgs.length) withChannels = { ...withChannels, cfgSideChannel: cfgs };
}
result.parsedFiles.push(withChannels);
}
// Build per-file type environment + constructor bindings in a single AST walk.

View file

@ -232,6 +232,15 @@ export interface WorkerPoolOptions {
* `undefined` ⇒ no durable write.
*/
durableParsedFileStoragePath?: string;
/**
* CFG/PDG opt-in (#2081 M1). Baked into every spawned worker's `workerData`
* (like the store paths above); when `true`, workers build a per-function
* control-flow graph from the tree-sitter AST and attach it to
* `ParsedFile.cfgSideChannel`. `undefined`/`false` ⇒ no CFG work.
*/
pdg?: boolean;
/** Per-function source-line cap for worker-side CFG construction (0 ⇒ no cap). */
pdgMaxFunctionLines?: number;
}
export class WorkerPoolDispatchError extends Error {
@ -884,9 +893,12 @@ export const createWorkerPool = (
// signature is unchanged so the zero-arg test factories keep working.
const parsedFileStoreStoragePath = options?.parsedFileStoreStoragePath;
const durableParsedFileStoragePath = options?.durableParsedFileStoragePath;
// CFG/PDG opt-in (#2081 M1) — carried in workerData alongside the store paths.
const pdg = options?.pdg === true;
const pdgMaxFunctionLines = options?.pdgMaxFunctionLines;
const workerStoreData =
parsedFileStoreStoragePath || durableParsedFileStoragePath
? { parsedFileStoreStoragePath, durableParsedFileStoragePath }
parsedFileStoreStoragePath || durableParsedFileStoragePath || pdg
? { parsedFileStoreStoragePath, durableParsedFileStoragePath, pdg, pdgMaxFunctionLines }
: undefined;
const spawnWorker =
options?.workerFactory ??

View file

@ -55,7 +55,7 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j
// the main thread (the #1983 OOM). Because the two stores share this version,
// any future change to the `ParsedFile` serialization shape MUST bump
// SCHEMA_BUMP so both invalidate in lockstep.
const SCHEMA_BUMP = 4;
const SCHEMA_BUMP = 5; // #2081 M1: ParsedFile gained `cfgSideChannel`
const GITNEXUS_PKG_VERSION = (() => {
try {
// package.json sits at gitnexus/package.json — two levels up from
@ -143,10 +143,16 @@ export const fileContentHash = (content: Buffer | string): string => sha256Hex(c
*/
export const computeChunkHash = (
entries: Array<{ filePath: string; contentHash: string }>,
pdg = false,
): string => {
const sorted = [...entries].sort((a, b) => (a.filePath < b.filePath ? -1 : 1));
const joined = sorted.map((e) => `${e.filePath}:${e.contentHash}`).join('\n');
return sha256Hex(joined);
// Fold the `--pdg` opt-in into the key (#2081 M1) so a chunk cached WITHOUT a
// CFG (`cfgSideChannel`) is NOT reused on a `--pdg` run, and vice-versa — the
// #2038-class warm-cache trap where an option-blind key silently serves
// field-less shards. Only prefixed when `pdg` is on, so the default (pdg-off)
// path keeps its existing keys and warm caches survive this change.
return sha256Hex(pdg ? `pdg\n${joined}` : joined);
};
/**

View file

@ -0,0 +1,119 @@
import { describe, it, expect } from 'vitest';
import Parser from 'tree-sitter';
import TypeScript from 'tree-sitter-typescript';
import { collectFunctionCfgs } from '../../../src/core/ingestion/cfg/collect.js';
import { computeChunkHash, mapReplacer, mapReviver } from '../../../src/storage/parse-cache.js';
import { getProvider } from '../../../src/core/ingestion/languages/index.js';
import { SupportedLanguages } from '../../../src/config/supported-languages.js';
import type { CfgVisitor } from '../../../src/core/ingestion/cfg/types.js';
import type { SyntaxNode } from '../../../src/core/ingestion/utils/ast-helpers.js';
// U3 — the worker→main boundary + cache coherence for the CFG side-channel.
// These pin the contracts that make the disk-store + warm/durable parse cache
// carry the CFG intact across the --pdg flag (R3, R4) WITHOUT spinning a real
// worker pool: the worker simply calls collectFunctionCfgs (tested here) and
// attaches the result as plain data, and the parse-cache key folds the flag.
function tsRoot(code: string): SyntaxNode {
const parser = new Parser();
parser.setLanguage(TypeScript.typescript);
return parser.parse(code).rootNode;
}
const tsVisitor = (): CfgVisitor<SyntaxNode> => {
const v = getProvider(SupportedLanguages.TypeScript).cfgVisitor;
if (!v) throw new Error('typescript provider has no cfgVisitor');
return v;
};
describe('U3 — TS/JS provider exposes a cfgVisitor; others do not (worker gate)', () => {
it('TS and JS providers carry a cfgVisitor', () => {
expect(getProvider(SupportedLanguages.TypeScript).cfgVisitor).toBeDefined();
expect(getProvider(SupportedLanguages.JavaScript).cfgVisitor).toBeDefined();
});
it('a non-CFG language (Python) has no cfgVisitor ⇒ worker emits no cfgSideChannel', () => {
// `provider.cfgVisitor &&` short-circuits in the worker → no CFG, no field.
expect(getProvider(SupportedLanguages.Python).cfgVisitor).toBeUndefined();
});
});
describe('U3 — collectFunctionCfgs', () => {
it('produces one CFG per function with the expected branch edges', () => {
const root = tsRoot(`
function a(x: number) { if (x) { p(); } else { q(); } }
function b() { return 1; }
`);
const { cfgs, skipped } = collectFunctionCfgs(root, tsVisitor(), 'a.ts');
expect(skipped).toBe(0);
expect(cfgs).toHaveLength(2);
const a = cfgs.find((c) => c.blocks.some((bl) => bl.text.includes('p();')));
expect(a).toBeDefined();
const kinds = new Set(a!.edges.map((e) => e.kind));
expect(kinds.has('cond-true')).toBe(true);
expect(kinds.has('cond-false')).toBe(true);
// every block belongs to its declaring file
for (const c of cfgs) expect(c.filePath).toBe('a.ts');
});
it('a file with no functions yields an empty CFG set (no error)', () => {
const { cfgs, skipped } = collectFunctionCfgs(
tsRoot(`const x = 1; export {};`),
tsVisitor(),
'x.ts',
);
expect(cfgs).toHaveLength(0);
expect(skipped).toBe(0);
});
it('maxFunctionLines skips an over-cap function and counts the skip', () => {
const big = `function big() {\n${' step();\n'.repeat(20)}}`;
const root = tsRoot(`${big}\nfunction small() { ok(); }`);
const { cfgs, skipped } = collectFunctionCfgs(root, tsVisitor(), 'f.ts', 5);
expect(skipped).toBe(1); // big() exceeds the 5-line cap
// small() is still built
expect(cfgs.some((c) => c.blocks.some((bl) => bl.text.includes('ok();')))).toBe(true);
expect(cfgs.some((c) => c.blocks.some((bl) => bl.text.includes('step();')))).toBe(false);
});
});
describe('U3 — CFG side-channel JSON round-trip (no AST leakage, no field loss)', () => {
it('serialize → JSON → deserialize yields an identical CFG', () => {
const root = tsRoot(`function f(xs: number[]) {
for (const x of xs) { if (x > 0) { use(x); } else { break; } }
done();
}`);
const { cfgs } = collectFunctionCfgs(root, tsVisitor(), 'rt.ts');
expect(cfgs.length).toBeGreaterThan(0);
// The worker serializes ParsedFile via mapReplacer; the store revives via
// mapReviver. The CFG is plain data, so it must survive byte-for-byte.
const round = JSON.parse(JSON.stringify(cfgs, mapReplacer), mapReviver);
expect(round).toEqual(cfgs);
// No tree-sitter nodes leaked: every value is a primitive/array/plain object.
for (const c of round) {
for (const b of c.blocks) expect(typeof b.text).toBe('string');
for (const e of c.edges) expect(typeof e.from).toBe('number');
}
});
});
describe('U3 — parse-cache key folds the --pdg flag (R4, #2038-class guard)', () => {
const entries = [
{ filePath: 'b.ts', contentHash: 'h2' },
{ filePath: 'a.ts', contentHash: 'h1' },
];
it('pdg-on and pdg-off produce DIFFERENT chunk keys', () => {
expect(computeChunkHash(entries, false)).not.toBe(computeChunkHash(entries, true));
});
it('the same flag value is stable and order-independent', () => {
const reordered = [...entries].reverse();
expect(computeChunkHash(entries, true)).toBe(computeChunkHash(reordered, true));
expect(computeChunkHash(entries, false)).toBe(computeChunkHash(reordered, false));
});
it('default (no flag arg) equals the explicit pdg-off key — warm caches survive the change', () => {
expect(computeChunkHash(entries)).toBe(computeChunkHash(entries, false));
});
});