mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-28 06:22:22 +00:00
perf(ingestion): prune inert local value symbols (#2065)
This commit is contained in:
parent
cf9ecda477
commit
1215d5fa9b
14 changed files with 567 additions and 22 deletions
|
|
@ -77,11 +77,11 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`).
|
|||
|
||||
## Pipeline Phase DAG
|
||||
|
||||
12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output.
|
||||
14 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output.
|
||||
|
||||
```
|
||||
scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
|
||||
→ crossFile → mro → communities → processes
|
||||
→ crossFile → scopeResolution → pruneLocalSymbols → mro → communities → processes
|
||||
```
|
||||
|
||||
| Phase | File | Deps | Output |
|
||||
|
|
@ -95,9 +95,11 @@ scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
|
|||
| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges |
|
||||
| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) |
|
||||
| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order |
|
||||
| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges |
|
||||
| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) |
|
||||
| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges |
|
||||
| `scopeResolution` | `scope-resolution/pipeline/phase.ts` | `parse`, `crossFile`, `structure` | Binding/reference + inheritance edges; disposes BindingAccumulator |
|
||||
| `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution |
|
||||
| `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges |
|
||||
| `communities` | `communities.ts` | `mro`, `pruneLocalSymbols`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) |
|
||||
| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `pruneLocalSymbols`, `structure` | Process nodes + STEP_IN_PROCESS edges |
|
||||
|
||||
**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`.
|
||||
|
||||
|
|
@ -119,7 +121,8 @@ scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
|
|||
- **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output.
|
||||
- **Typed phase access** — `getPhaseOutput<T>(deps, 'name')` for type-safe upstream results.
|
||||
- **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership.
|
||||
- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing.
|
||||
- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests); `pruneLocalSymbols` still runs (it is graph cleanup, not analysis). `skipWorkers` forces sequential parsing.
|
||||
- **Local-symbol pruning** — `pruneLocalSymbols` removes inert block-local value symbols after scope resolution has consumed them. Opt out per-call with `PipelineOptions.keepLocalValueSymbols` or globally with the `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` env var.
|
||||
|
||||
### How to add a new phase
|
||||
|
||||
|
|
|
|||
|
|
@ -423,6 +423,16 @@ Three env vars expose the pool's resilience layers (respawn budget, cumulative-t
|
|||
| `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. |
|
||||
| `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. |
|
||||
|
||||
### Graph cleanup tuning
|
||||
|
||||
After scope resolution, analyze prunes inert block-local value symbols (a function-local `const`/`let`/`var` that ends up with only its structural `File→DEFINES` edge) to keep the graph focused on cross-symbol relationships. Module/file-scope symbols, class members, and any local with a real edge are always kept.
|
||||
|
||||
| Variable | Default | Effect |
|
||||
| ------------------------------------ | ------- | ------------------------------------------------------------------------------------------------------- |
|
||||
| `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` | unset | Set to `1`/`true` to keep inert block-local value symbols instead of pruning them. |
|
||||
|
||||
Programmatic callers can pass `keepLocalValueSymbols: true` in `PipelineOptions` instead of setting the env var.
|
||||
|
||||
## Privacy
|
||||
|
||||
- All processing happens locally on your machine
|
||||
|
|
|
|||
82
gitnexus/src/core/ingestion/local-symbol-pruner.ts
Normal file
82
gitnexus/src/core/ingestion/local-symbol-pruner.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import type { GraphNode, GraphRelationship, NodeLabel } from 'gitnexus-shared';
|
||||
import type { KnowledgeGraph } from '../graph/types.js';
|
||||
import { parseTruthyEnv } from './utils/env.js';
|
||||
|
||||
const LOCAL_VALUE_LABELS = new Set<NodeLabel>(['Const', 'Variable', 'Static']);
|
||||
const KEEP_LOCAL_VALUE_SYMBOLS_ENV = 'GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS';
|
||||
|
||||
export interface LocalSymbolPruneStats {
|
||||
candidateNodes: number;
|
||||
prunedNodes: number;
|
||||
keptWithSemanticEdges: number;
|
||||
skippedByEnv: boolean;
|
||||
}
|
||||
|
||||
export const shouldKeepLocalValueSymbols = (): boolean =>
|
||||
parseTruthyEnv(process.env[KEEP_LOCAL_VALUE_SYMBOLS_ENV]);
|
||||
|
||||
const emptyStats = (skippedByEnv: boolean): LocalSymbolPruneStats => ({
|
||||
candidateNodes: 0,
|
||||
prunedNodes: 0,
|
||||
keptWithSemanticEdges: 0,
|
||||
skippedByEnv,
|
||||
});
|
||||
|
||||
const isLocalValueCandidate = (node: GraphNode): boolean => {
|
||||
if (!LOCAL_VALUE_LABELS.has(node.label)) return false;
|
||||
return node.properties.scope === 'block';
|
||||
};
|
||||
|
||||
// True when `rel` is the structural `File -> DEFINES -> candidate` edge. Callers
|
||||
// guard on the candidate already being the edge target, so only the source label
|
||||
// needs checking here.
|
||||
const isFileDefinesEdge = (graph: KnowledgeGraph, rel: GraphRelationship): boolean => {
|
||||
if (rel.type !== 'DEFINES') return false;
|
||||
return graph.getNode(rel.sourceId)?.label === 'File';
|
||||
};
|
||||
|
||||
export const pruneLocalValueSymbols = (
|
||||
graph: KnowledgeGraph,
|
||||
options: { keepLocalValueSymbols?: boolean } = {},
|
||||
): LocalSymbolPruneStats => {
|
||||
if (options.keepLocalValueSymbols ?? shouldKeepLocalValueSymbols()) {
|
||||
return emptyStats(true);
|
||||
}
|
||||
|
||||
const candidateIds = new Set<string>();
|
||||
for (const node of graph.iterNodes()) {
|
||||
if (isLocalValueCandidate(node)) candidateIds.add(node.id);
|
||||
}
|
||||
|
||||
if (candidateIds.size === 0) return emptyStats(false);
|
||||
|
||||
const candidatesWithSemanticEdges = new Set<string>();
|
||||
for (const rel of graph.iterRelationships()) {
|
||||
// Any outgoing edge from a candidate is a semantic edge: the only structural
|
||||
// edge a block-local value symbol carries is the incoming File -> DEFINES, on
|
||||
// which the candidate is the target, never the source.
|
||||
if (candidateIds.has(rel.sourceId)) {
|
||||
candidatesWithSemanticEdges.add(rel.sourceId);
|
||||
}
|
||||
|
||||
// An incoming edge is semantic unless it is the structural File -> DEFINES.
|
||||
if (candidateIds.has(rel.targetId)) {
|
||||
if (!isFileDefinesEdge(graph, rel)) {
|
||||
candidatesWithSemanticEdges.add(rel.targetId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let prunedNodes = 0;
|
||||
for (const candidateId of candidateIds) {
|
||||
if (candidatesWithSemanticEdges.has(candidateId)) continue;
|
||||
if (graph.removeNode(candidateId)) prunedNodes++;
|
||||
}
|
||||
|
||||
return {
|
||||
candidateNodes: candidateIds.size,
|
||||
prunedNodes,
|
||||
keptWithSemanticEdges: candidatesWithSemanticEdges.size,
|
||||
skippedByEnv: false,
|
||||
};
|
||||
};
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
* Detects code communities via Leiden algorithm and creates
|
||||
* Community nodes + MEMBER_OF edges.
|
||||
*
|
||||
* @deps mro
|
||||
* @deps mro, pruneLocalSymbols
|
||||
* @reads graph (all nodes and relationships)
|
||||
* @writes graph (Community nodes, MEMBER_OF edges)
|
||||
*/
|
||||
|
|
@ -22,7 +22,10 @@ export interface CommunitiesOutput {
|
|||
|
||||
export const communitiesPhase: PipelinePhase<CommunitiesOutput> = {
|
||||
name: 'communities',
|
||||
deps: ['mro', 'structure'],
|
||||
// `pruneLocalSymbols` is declared explicitly (not just transitively via `mro`)
|
||||
// so community detection always reads the trimmed graph even if a future option
|
||||
// drops `mro` from the phase list.
|
||||
deps: ['mro', 'pruneLocalSymbols', 'structure'],
|
||||
|
||||
async execute(
|
||||
ctx: PipelineContext,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export {
|
|||
scopeResolutionPhase,
|
||||
type ScopeResolutionOutput,
|
||||
} from '../scope-resolution/pipeline/phase.js';
|
||||
export { pruneLocalSymbolsPhase, type PruneLocalSymbolsOutput } from './prune-local-symbols.js';
|
||||
export { mroPhase, type MROOutput } from './mro.js';
|
||||
export { communitiesPhase, type CommunitiesOutput } from './communities.js';
|
||||
export { processesPhase, type ProcessesOutput } from './processes.js';
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* Computes Method Resolution Order (MRO) and creates METHOD_OVERRIDES
|
||||
* and METHOD_IMPLEMENTS edges.
|
||||
*
|
||||
* @deps crossFile, scopeResolution
|
||||
* @deps crossFile, scopeResolution, pruneLocalSymbols
|
||||
* @reads graph (all nodes and relationships)
|
||||
* @writes graph (METHOD_OVERRIDES, METHOD_IMPLEMENTS edges)
|
||||
*/
|
||||
|
|
@ -25,7 +25,7 @@ export interface MROOutput {
|
|||
|
||||
export const mroPhase: PipelinePhase<MROOutput> = {
|
||||
name: 'mro',
|
||||
deps: ['crossFile', 'scopeResolution', 'structure'],
|
||||
deps: ['crossFile', 'scopeResolution', 'pruneLocalSymbols', 'structure'],
|
||||
|
||||
async execute(
|
||||
ctx: PipelineContext,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
* Detects execution flows (processes) and creates Process nodes +
|
||||
* STEP_IN_PROCESS edges. Also links Route/Tool nodes to processes.
|
||||
*
|
||||
* @deps communities, routes, tools
|
||||
* @deps communities, routes, tools, pruneLocalSymbols
|
||||
* @reads graph (all nodes and relationships), communityResult, routeRegistry, toolDefs
|
||||
* @writes graph (Process nodes, STEP_IN_PROCESS edges, ENTRY_POINT_OF edges)
|
||||
*/
|
||||
|
|
@ -27,8 +27,10 @@ export interface ProcessesOutput {
|
|||
export const processesPhase: PipelinePhase<ProcessesOutput> = {
|
||||
name: 'processes',
|
||||
// `structure` supplies `totalFiles` (progress counter) without the spurious
|
||||
// structural data dependency on `parse`.
|
||||
deps: ['communities', 'routes', 'tools', 'structure'],
|
||||
// structural data dependency on `parse`. `pruneLocalSymbols` is declared
|
||||
// explicitly so process extraction always reads the trimmed graph even if a
|
||||
// future option drops the intervening `mro`/`communities` phases.
|
||||
deps: ['communities', 'routes', 'tools', 'pruneLocalSymbols', 'structure'],
|
||||
|
||||
async execute(
|
||||
ctx: PipelineContext,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
/**
|
||||
* Phase: pruneLocalSymbols
|
||||
*
|
||||
* Drops inert function/block-local value symbols after scope resolution has
|
||||
* already used them for binding and call resolution.
|
||||
*
|
||||
* @deps scopeResolution
|
||||
* @reads graph (nodes and relationships)
|
||||
* @writes graph (removes unreferenced local Const/Variable/Static nodes)
|
||||
*/
|
||||
|
||||
import type { PipelinePhase, PipelineContext } from './types.js';
|
||||
import { pruneLocalValueSymbols, type LocalSymbolPruneStats } from '../local-symbol-pruner.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
export type PruneLocalSymbolsOutput = LocalSymbolPruneStats;
|
||||
|
||||
export const pruneLocalSymbolsPhase: PipelinePhase<PruneLocalSymbolsOutput> = {
|
||||
name: 'pruneLocalSymbols',
|
||||
deps: ['scopeResolution'],
|
||||
|
||||
async execute(ctx: PipelineContext): Promise<PruneLocalSymbolsOutput> {
|
||||
const stats = pruneLocalValueSymbols(ctx.graph, {
|
||||
keepLocalValueSymbols: ctx.options?.keepLocalValueSymbols,
|
||||
});
|
||||
|
||||
if (isDev && !stats.skippedByEnv && stats.prunedNodes > 0) {
|
||||
logger.info(`Pruned ${stats.prunedNodes}/${stats.candidateNodes} inert local value symbols`);
|
||||
}
|
||||
|
||||
return stats;
|
||||
},
|
||||
};
|
||||
|
|
@ -31,6 +31,7 @@ import {
|
|||
ormPhase,
|
||||
crossFilePhase,
|
||||
scopeResolutionPhase,
|
||||
pruneLocalSymbolsPhase,
|
||||
mroPhase,
|
||||
communitiesPhase,
|
||||
processesPhase,
|
||||
|
|
@ -41,7 +42,12 @@ import {
|
|||
} from './pipeline-phases/index.js';
|
||||
|
||||
export interface PipelineOptions {
|
||||
/** Skip MRO, community detection, and process extraction for faster test runs. */
|
||||
/**
|
||||
* Skip MRO, community detection, and process extraction for faster test runs.
|
||||
* The `pruneLocalSymbols` phase still runs — it is graph construction (it cleans
|
||||
* up inert local symbols), not graph analysis — so set `keepLocalValueSymbols`
|
||||
* to retain those nodes under `skipGraphPhases`.
|
||||
*/
|
||||
skipGraphPhases?: boolean;
|
||||
/**
|
||||
* Request parsing with the worker pool disabled. The sequential parser was
|
||||
|
|
@ -114,6 +120,14 @@ export interface PipelineOptions {
|
|||
* without leaking `process.env` state across invocations.
|
||||
*/
|
||||
chunkByteBudget?: number;
|
||||
/**
|
||||
* Keep inert block-local value symbols (Const/Variable/Static) that the
|
||||
* `pruneLocalSymbols` phase would otherwise drop. Mirrors the
|
||||
* `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` env var, but threaded per-call so
|
||||
* long-running hosts (eval-server, MCP daemon) can opt out without leaking
|
||||
* `process.env` state across invocations. When undefined, the env var decides.
|
||||
*/
|
||||
keepLocalValueSymbols?: boolean;
|
||||
}
|
||||
|
||||
// ── Phase registry ─────────────────────────────────────────────────────────
|
||||
|
|
@ -124,7 +138,8 @@ export interface PipelineOptions {
|
|||
* Phase dependency graph:
|
||||
*
|
||||
* scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
|
||||
* → crossFile → mro → communities → processes
|
||||
* → crossFile → scopeResolution → pruneLocalSymbols
|
||||
* → mro → communities → processes
|
||||
*
|
||||
* To add a new phase: create a file in pipeline-phases/, export the phase
|
||||
* object, and add it to the appropriate position in this array.
|
||||
|
|
@ -141,6 +156,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] {
|
|||
ormPhase,
|
||||
crossFilePhase,
|
||||
scopeResolutionPhase,
|
||||
pruneLocalSymbolsPhase,
|
||||
];
|
||||
|
||||
if (!options?.skipGraphPhases) {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,26 @@ import type {
|
|||
VariableScope,
|
||||
} from '../variable-types.js';
|
||||
|
||||
/**
|
||||
* Type-declaration node types whose body can be a bare `block`. tree-sitter-python
|
||||
* models a class body as a `block` node — the same node type used for function and
|
||||
* control-flow bodies — so a class attribute would otherwise look block-scoped. Most
|
||||
* other grammars give class bodies dedicated node types (`class_body`,
|
||||
* `declaration_list`, `body_statement`), which are not in the block-scope list below,
|
||||
* so this guard is a no-op for them but keeps the rule language-agnostic.
|
||||
*/
|
||||
const CLASS_LIKE_CONTAINERS = new Set<string>([
|
||||
'class_definition',
|
||||
'class_declaration',
|
||||
'class_specifier',
|
||||
'struct_item',
|
||||
'impl_item',
|
||||
'trait_item',
|
||||
'interface_declaration',
|
||||
'enum_declaration',
|
||||
'object_declaration',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Create a VariableExtractor from a declarative config.
|
||||
*/
|
||||
|
|
@ -48,7 +68,7 @@ export function createVariableExtractor(config: VariableExtractionConfig): Varia
|
|||
) {
|
||||
return 'module';
|
||||
}
|
||||
// Function/method/block boundaries indicate block scope
|
||||
// Function/method boundaries indicate block scope
|
||||
if (
|
||||
t === 'function_declaration' ||
|
||||
t === 'function_definition' ||
|
||||
|
|
@ -58,12 +78,18 @@ export function createVariableExtractor(config: VariableExtractionConfig): Varia
|
|||
t === 'arrow_function' ||
|
||||
t === 'function_expression' ||
|
||||
t === 'lambda' ||
|
||||
t === 'block' ||
|
||||
t === 'function_body' ||
|
||||
t === 'compound_statement'
|
||||
) {
|
||||
return 'block';
|
||||
}
|
||||
// A bare `block` is block scope UNLESS it is a class body. A class member
|
||||
// (e.g. Python `class C: MAX = 100`) is not an inert function-local — keep
|
||||
// walking so it resolves to its true enclosing scope ('module' for a
|
||||
// top-level class) instead of being misclassified and pruned.
|
||||
if (t === 'block' && !(current.parent && CLASS_LIKE_CONTAINERS.has(current.parent.type))) {
|
||||
return 'block';
|
||||
}
|
||||
current = current.parent;
|
||||
}
|
||||
return 'file';
|
||||
|
|
|
|||
|
|
@ -2,13 +2,12 @@
|
|||
"capture": "initial capture (U8, post-U1–U7)",
|
||||
"fixture": "mini-repo",
|
||||
"totalFileCount": 7,
|
||||
"symbols": 37,
|
||||
"relationships": 73,
|
||||
"symbols": 33,
|
||||
"relationships": 69,
|
||||
"processes": 4,
|
||||
"byType": {
|
||||
"Class": 1,
|
||||
"Community": 4,
|
||||
"Const": 4,
|
||||
"File": 7,
|
||||
"Folder": 1,
|
||||
"Function": 12,
|
||||
|
|
@ -19,11 +18,11 @@
|
|||
"byRelType": {
|
||||
"CALLS": 9,
|
||||
"CONTAINS": 7,
|
||||
"DEFINES": 20,
|
||||
"DEFINES": 16,
|
||||
"HAS_METHOD": 1,
|
||||
"IMPORTS": 12,
|
||||
"MEMBER_OF": 12,
|
||||
"STEP_IN_PROCESS": 12
|
||||
},
|
||||
"edgeDigest": "196198eb9a900721aec892400d141189aa1e874722917843c7f1206805d817f1"
|
||||
"edgeDigest": "1e80aba78cb1784276d387e9debd006ae4e82e60ce33c59e338aac4886d98f61"
|
||||
}
|
||||
|
|
|
|||
107
gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts
Normal file
107
gitnexus/test/integration/local-symbol-pruner-pipeline.test.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js';
|
||||
import { DIST_WORKER_URL, distWorkerExists } from '../helpers/worker-parse.js';
|
||||
|
||||
const describeIfWorkerBuilt = distWorkerExists() ? describe : describe.skip;
|
||||
|
||||
let tmpDirs: string[] = [];
|
||||
|
||||
const makeRepo = (source: string, filename = 'sample.ts'): string => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-local-prune-'));
|
||||
tmpDirs.push(dir);
|
||||
fs.writeFileSync(path.join(dir, filename), source, 'utf-8');
|
||||
return dir;
|
||||
};
|
||||
|
||||
const findNode = (
|
||||
result: Awaited<ReturnType<typeof runPipelineFromRepo>>,
|
||||
label: string,
|
||||
name: string,
|
||||
) => result.graph.nodes.find((node) => node.label === label && node.properties.name === name);
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tmpDirs) fs.rmSync(dir, { recursive: true, force: true });
|
||||
tmpDirs = [];
|
||||
});
|
||||
|
||||
describeIfWorkerBuilt('local value symbol pruning pipeline', () => {
|
||||
it('prunes inert locals inside exported functions without losing resolved calls', async () => {
|
||||
const repo = makeRepo(`
|
||||
const MODULE_CONST = 1;
|
||||
export const exportedHandler = () => MODULE_CONST;
|
||||
|
||||
export function run() {
|
||||
const boring = 1;
|
||||
const handler = () => boring;
|
||||
const client = new Client();
|
||||
client.send();
|
||||
return handler();
|
||||
}
|
||||
|
||||
class Client {
|
||||
send() {}
|
||||
}
|
||||
`);
|
||||
|
||||
const result = await runPipelineFromRepo(repo, () => {}, {
|
||||
skipGraphPhases: true,
|
||||
workerPoolSize: 1,
|
||||
workerUrlForTest: DIST_WORKER_URL,
|
||||
});
|
||||
|
||||
expect(findNode(result, 'Const', 'boring')).toBeUndefined();
|
||||
expect(findNode(result, 'Const', 'client')).toBeUndefined();
|
||||
expect(findNode(result, 'Const', 'handler')).toBeUndefined();
|
||||
|
||||
expect(findNode(result, 'Const', 'MODULE_CONST')).toBeDefined();
|
||||
expect(findNode(result, 'Const', 'exportedHandler')).toBeDefined();
|
||||
expect(findNode(result, 'Function', 'handler')).toBeDefined();
|
||||
|
||||
const keepsResolvedClientCall = result.graph.relationships.some((rel) => {
|
||||
if (rel.type !== 'CALLS') return false;
|
||||
const source = result.graph.getNode(rel.sourceId);
|
||||
const target = result.graph.getNode(rel.targetId);
|
||||
return source?.properties.name === 'run' && target?.properties.name === 'send';
|
||||
});
|
||||
expect(keepsResolvedClientCall).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps Python class-level constants while pruning function-locals', async () => {
|
||||
// Regression: tree-sitter-python models the class body as a `block` node, so
|
||||
// `determineScope` classified an untyped class attribute as block-scope. Python
|
||||
// emits such assignments as `Variable` (the `@definition.variable` capture), and
|
||||
// value labels get no owner edge (needsOwner excludes them) — only File->DEFINES.
|
||||
// The prune pass would therefore silently delete unreferenced class attributes.
|
||||
// A class-level symbol is NOT a function-local and must survive.
|
||||
const repo = makeRepo(
|
||||
`MODULE_CONST = 1
|
||||
|
||||
|
||||
class Settings:
|
||||
MAX_SIZE = 100
|
||||
|
||||
|
||||
def run():
|
||||
boring = 1
|
||||
return boring
|
||||
`,
|
||||
'sample.py',
|
||||
);
|
||||
|
||||
const result = await runPipelineFromRepo(repo, () => {}, {
|
||||
skipGraphPhases: true,
|
||||
workerPoolSize: 1,
|
||||
workerUrlForTest: DIST_WORKER_URL,
|
||||
});
|
||||
|
||||
// Unreferenced class-level attribute must survive the prune.
|
||||
expect(findNode(result, 'Variable', 'MAX_SIZE')).toBeDefined();
|
||||
// Module-level symbol is preserved as before.
|
||||
expect(findNode(result, 'Variable', 'MODULE_CONST')).toBeDefined();
|
||||
// Genuine function-local is still pruned.
|
||||
expect(findNode(result, 'Variable', 'boring')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
239
gitnexus/test/unit/local-symbol-pruner.test.ts
Normal file
239
gitnexus/test/unit/local-symbol-pruner.test.ts
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { GraphNode, GraphRelationship, NodeLabel, RelationshipType } from 'gitnexus-shared';
|
||||
import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
|
||||
import {
|
||||
pruneLocalValueSymbols,
|
||||
shouldKeepLocalValueSymbols,
|
||||
} from '../../src/core/ingestion/local-symbol-pruner.js';
|
||||
import { pruneLocalSymbolsPhase } from '../../src/core/ingestion/pipeline-phases/prune-local-symbols.js';
|
||||
|
||||
const fileNode = (): GraphNode => ({
|
||||
id: 'file:src/app.ts',
|
||||
label: 'File',
|
||||
properties: {
|
||||
name: 'app.ts',
|
||||
filePath: 'src/app.ts',
|
||||
},
|
||||
});
|
||||
|
||||
const node = (
|
||||
id: string,
|
||||
label: NodeLabel,
|
||||
properties: Partial<GraphNode['properties']> = {},
|
||||
): GraphNode => ({
|
||||
id,
|
||||
label,
|
||||
properties: {
|
||||
name: id,
|
||||
filePath: 'src/app.ts',
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
...properties,
|
||||
},
|
||||
});
|
||||
|
||||
const rel = (
|
||||
id: string,
|
||||
sourceId: string,
|
||||
targetId: string,
|
||||
type: RelationshipType = 'DEFINES',
|
||||
): GraphRelationship => ({
|
||||
id,
|
||||
sourceId,
|
||||
targetId,
|
||||
type,
|
||||
confidence: 1,
|
||||
reason: 'test',
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe('pruneLocalValueSymbols', () => {
|
||||
it.each(['Const', 'Variable', 'Static'] as const)(
|
||||
'prunes inert block-scope %s nodes after scope resolution',
|
||||
(label) => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node(`${label}:tmp`, label, { scope: 'block' }));
|
||||
graph.addRelationship(rel(`rel:${label}`, 'file:src/app.ts', `${label}:tmp`));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats).toEqual({
|
||||
candidateNodes: 1,
|
||||
prunedNodes: 1,
|
||||
keptWithSemanticEdges: 0,
|
||||
skippedByEnv: false,
|
||||
});
|
||||
expect(graph.getNode(`${label}:tmp`)).toBeUndefined();
|
||||
expect(graph.relationshipCount).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['module', 'file'] as const)('keeps %s-scope value symbols', (scope) => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node(`Const:${scope}`, 'Const', { scope }));
|
||||
graph.addRelationship(rel(`rel:${scope}`, 'file:src/app.ts', `Const:${scope}`));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(stats.candidateNodes).toBe(0);
|
||||
expect(graph.getNode(`Const:${scope}`)).toBeDefined();
|
||||
expect(graph.relationshipCount).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps block-scope value symbols with semantic edges', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Function:run', 'Function'));
|
||||
graph.addNode(node('Const:client', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:client'));
|
||||
graph.addRelationship(rel('rel:access', 'Function:run', 'Const:client', 'ACCESSES'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats).toMatchObject({
|
||||
candidateNodes: 1,
|
||||
prunedNodes: 0,
|
||||
keptWithSemanticEdges: 1,
|
||||
skippedByEnv: false,
|
||||
});
|
||||
expect(graph.getNode('Const:client')).toBeDefined();
|
||||
expect(graph.relationshipCount).toBe(2);
|
||||
});
|
||||
|
||||
it('keeps block-scope value symbols that are the source of an outgoing edge', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Element:thing', 'CodeElement'));
|
||||
graph.addNode(node('Const:config', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:file-def', 'file:src/app.ts', 'Const:config'));
|
||||
// Candidate is the SOURCE of an outgoing DEFINES edge — any outgoing edge is
|
||||
// semantic, so the node must be kept (guards the source-branch simplification).
|
||||
graph.addRelationship(rel('rel:out', 'Const:config', 'Element:thing', 'DEFINES'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(stats.keptWithSemanticEdges).toBe(1);
|
||||
expect(graph.getNode('Const:config')).toBeDefined();
|
||||
});
|
||||
|
||||
it('does not treat value symbols without a scope property as candidates', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Const:noScope', 'Const'));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:noScope'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.candidateNodes).toBe(0);
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(graph.getNode('Const:noScope')).toBeDefined();
|
||||
});
|
||||
|
||||
it('prunes block-scope value symbols even when parser metadata marks them exported', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Const:tmp', 'Const', { scope: 'block', isExported: true }));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:tmp'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.prunedNodes).toBe(1);
|
||||
expect(graph.getNode('Const:tmp')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps block-scope value symbols defined by explicit scope graph nodes', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Scope:function:run', 'CodeElement'));
|
||||
graph.addNode(node('Const:client', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:file-def', 'file:src/app.ts', 'Const:client'));
|
||||
graph.addRelationship(rel('rel:scope-def', 'Scope:function:run', 'Const:client'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(stats.keptWithSemanticEdges).toBe(1);
|
||||
expect(graph.getNode('Const:client')).toBeDefined();
|
||||
});
|
||||
|
||||
it('does not prune function-like local symbols', () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Function:inner', 'Function', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:function', 'file:src/app.ts', 'Function:inner'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(stats.candidateNodes).toBe(0);
|
||||
expect(graph.getNode('Function:inner')).toBeDefined();
|
||||
});
|
||||
|
||||
it('can be disabled with GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS', () => {
|
||||
vi.stubEnv('GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS', '1');
|
||||
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Const:tmp', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:tmp'));
|
||||
|
||||
const stats = pruneLocalValueSymbols(graph);
|
||||
|
||||
expect(shouldKeepLocalValueSymbols()).toBe(true);
|
||||
expect(stats.skippedByEnv).toBe(true);
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(graph.getNode('Const:tmp')).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pruneLocalSymbolsPhase', () => {
|
||||
it('runs after scope resolution and returns prune stats', async () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Const:tmp', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:tmp'));
|
||||
|
||||
const stats = await pruneLocalSymbolsPhase.execute(
|
||||
{
|
||||
repoPath: '/repo',
|
||||
graph,
|
||||
onProgress: () => {},
|
||||
pipelineStart: Date.now(),
|
||||
},
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(pruneLocalSymbolsPhase.name).toBe('pruneLocalSymbols');
|
||||
expect(pruneLocalSymbolsPhase.deps).toEqual(['scopeResolution']);
|
||||
expect(stats.prunedNodes).toBe(1);
|
||||
expect(graph.getNode('Const:tmp')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('honors the keepLocalValueSymbols option without reading process.env', async () => {
|
||||
const graph = createKnowledgeGraph();
|
||||
graph.addNode(fileNode());
|
||||
graph.addNode(node('Const:tmp', 'Const', { scope: 'block' }));
|
||||
graph.addRelationship(rel('rel:def', 'file:src/app.ts', 'Const:tmp'));
|
||||
|
||||
const stats = await pruneLocalSymbolsPhase.execute(
|
||||
{
|
||||
repoPath: '/repo',
|
||||
graph,
|
||||
onProgress: () => {},
|
||||
pipelineStart: Date.now(),
|
||||
options: { keepLocalValueSymbols: true },
|
||||
},
|
||||
new Map(),
|
||||
);
|
||||
|
||||
expect(stats.skippedByEnv).toBe(true);
|
||||
expect(stats.prunedNodes).toBe(0);
|
||||
expect(graph.getNode('Const:tmp')).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -686,6 +686,29 @@ describe('VariableExtractor — block-scoped declarations', () => {
|
|||
expect(info!.scope).toBe('block');
|
||||
});
|
||||
|
||||
it('Python: class-body attribute is not block-scoped (class body is a `block` node)', () => {
|
||||
// Regression: tree-sitter-python models the class body as a `block` node, the
|
||||
// same node type as a function body. A class attribute must NOT be classified
|
||||
// as a function-local block, or the pruner would silently drop it.
|
||||
const extractor = createVariableExtractor(pythonVariableConfig);
|
||||
const ctx: VariableExtractorContext = {
|
||||
filePath: 'test.py',
|
||||
language: SupportedLanguages.Python,
|
||||
};
|
||||
parser.setLanguage(Python);
|
||||
const tree = parser.parse('class Settings:\n MAX_SIZE = 100');
|
||||
// module > class_definition > block > expression_statement > assignment
|
||||
const classDef = tree.rootNode.namedChildren.find((c) => c.type === 'class_definition')!;
|
||||
const body = classDef.childForFieldName('body')!;
|
||||
const exprStmt = body.namedChildren.find((c) => c.type === 'expression_statement');
|
||||
expect(exprStmt).toBeDefined();
|
||||
const info = extractor.extract(exprStmt!, ctx);
|
||||
expect(info).not.toBeNull();
|
||||
expect(info!.name).toBe('MAX_SIZE');
|
||||
expect(info!.scope).not.toBe('block');
|
||||
expect(info!.scope).toBe('module');
|
||||
});
|
||||
|
||||
it('Python: rejects non-assignment expression statements (e.g. function calls)', () => {
|
||||
const extractor = createVariableExtractor(pythonVariableConfig);
|
||||
const ctx: VariableExtractorContext = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue