mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-11 03:38:07 +00:00
perf(parse/heritage/mro): typed graph iterator + cross-phase tree cache
Two structural perf wins targeting the parse / heritage / MRO
layers, identified by the post-WorkspaceResolutionIndex profiling
(scope-resolution = ~1% of pipeline; the bulk lives upstream).
## 1. KnowledgeGraph.iterRelationshipsByType (PHM-Units 1-2)
- Adds a per-type `Map<RelationshipType, Map<id, Relationship>>`
index inside `createKnowledgeGraph`, maintained on add / remove /
removeNode / removeNodesByFile.
- New `iterRelationshipsByType(type)` returns a typed iterator that
yields only the requested type. Backwards-compatible: existing
`iterRelationships()` / `forEachRelationship()` callers untouched.
- Migrated two MRO call sites:
- `mro-processor.ts buildAdjacency`: split the single
`forEachRelationship` (which scanned every edge in the graph and
type-filtered per-iteration) into three typed iterations
(EXTENDS, IMPLEMENTS, HAS_METHOD).
- `scope-resolution/passes/mro.ts buildMro`: replaced
`for (const rel of graph.iterRelationships()) if (rel.type !== 'EXTENDS') continue`
with `for (const rel of graph.iterRelationshipsByType('EXTENDS'))`.
- Heritage-processor (PHM-Unit 3) was a no-op: it only WRITES
EXTENDS/IMPLEMENTS edges, never re-reads. Index is still useful
for the seven other graph-iter consumers (community-processor,
csv-generator, wildcard-synthesis, process-processor, etc.) — those
follow-ups can switch to the typed iterator without touching the
graph layer.
- Adds 5 unit tests for the new method (add/remove/dedupe semantics,
empty-type fresh iterator, removeNode index sync).
## 2. Cross-phase tree cache (PHM-Units 4-5)
The audit's #2 finding: Python files are parsed by tree-sitter once
in the parse phase, then re-parsed inside scope-resolution's
`captures.ts`. Eliminate the second parse by sharing the Tree across
phases.
- `parse-impl.ts` now maintains TWO ASTCaches with distinct lifetimes:
- `astCache` (chunk-local, cleared between chunks) — unchanged;
used by call/heritage/import processors during parse.
- `scopeTreeCache` (total-parseable-sized, never cleared) — new,
exposed via `ParseOutput.astCache` for cross-phase consumption.
- `parsing-processor.ts` writes every sequentially-parsed Tree to
BOTH caches. Worker-mode parses skip the persistent cache too
(Trees can't cross MessageChannels).
- `LanguageProvider.emitScopeCaptures` gains an optional `cachedTree`
parameter (typed `unknown` to keep the tree-sitter dep out of the
contract).
- `captures.ts` short-circuits its own `parser.parse(sourceText)`
when a cached Tree is supplied. Cache miss falls back to a fresh
parse — same correctness path as before.
- `runScopeResolution` accepts an optional `treeCache` and forwards
per-file `cachedTree` to `extractParsedFile`.
- `scope-resolution/pipeline/phase.ts` reads
`getPhaseOutput<{astCache}>(deps, 'parse')` and passes through.
Verified end-to-end: a small fixture run with PROF_SCOPE_RESOLUTION=1
shows 6/6 cache hits (100% hit rate) on the python-grandparent fixture
that exercises the full pipeline below the worker-pool threshold.
## Verification
- REGISTRY_PRIMARY_PYTHON=0 (legacy): 191/191.
- REGISTRY_PRIMARY_PYTHON=1 (registry): 191/191.
- New graph.test.ts: 25/25 (was 20).
- tsc --noEmit clean.
## Where the win lands
Wall-clock on the 49-fixture integration suite: 14050ms → 14080ms
(within noise). Fixtures are 1-3 files each, dominated by per-fixture
pipeline overhead (worker-pool init, DB writes, fixture startup).
The cache + typed-iterator wins are constant-factor improvements
that scale linearly with workload size and visible only on larger
repos. The dev-mode `PROF_SCOPE_RESOLUTION` instrumentation +
`getPythonCaptureCacheStats()` are kept for future perf work.
## Plan
docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md.
PHM-Unit 3 (heritage-processor migration) intentionally collapsed
to a no-op — heritage only writes, never re-reads.
This commit is contained in:
parent
4b9762577e
commit
8c6f5ceeab
13 changed files with 270 additions and 39 deletions
|
|
@ -1,9 +1,21 @@
|
||||||
import type { GraphNode, GraphRelationship } from 'gitnexus-shared';
|
import type { GraphNode, GraphRelationship, RelationshipType } from 'gitnexus-shared';
|
||||||
import { KnowledgeGraph } from './types.js';
|
import { KnowledgeGraph } from './types.js';
|
||||||
|
|
||||||
|
/** Fresh empty iterator per call — `[].values()` returns a new
|
||||||
|
* exhausted iterator each invocation, so empty-type lookups don't
|
||||||
|
* share a single already-exhausted iterator across callers. */
|
||||||
|
function emptyRelIter(): IterableIterator<GraphRelationship> {
|
||||||
|
return ([] as GraphRelationship[]).values();
|
||||||
|
}
|
||||||
|
|
||||||
export const createKnowledgeGraph = (): KnowledgeGraph => {
|
export const createKnowledgeGraph = (): KnowledgeGraph => {
|
||||||
const nodeMap = new Map<string, GraphNode>();
|
const nodeMap = new Map<string, GraphNode>();
|
||||||
const relationshipMap = new Map<string, GraphRelationship>();
|
const relationshipMap = new Map<string, GraphRelationship>();
|
||||||
|
// Per-type index maintained alongside `relationshipMap`. Bucket
|
||||||
|
// values are `Map<id, Relationship>` so per-type iteration is cheap
|
||||||
|
// and per-edge removal is O(1). See plan
|
||||||
|
// docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 1).
|
||||||
|
const relationshipsByType = new Map<RelationshipType, Map<string, GraphRelationship>>();
|
||||||
|
|
||||||
const addNode = (node: GraphNode) => {
|
const addNode = (node: GraphNode) => {
|
||||||
if (!nodeMap.has(node.id)) {
|
if (!nodeMap.has(node.id)) {
|
||||||
|
|
@ -12,9 +24,14 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
||||||
};
|
};
|
||||||
|
|
||||||
const addRelationship = (relationship: GraphRelationship) => {
|
const addRelationship = (relationship: GraphRelationship) => {
|
||||||
if (!relationshipMap.has(relationship.id)) {
|
if (relationshipMap.has(relationship.id)) return;
|
||||||
relationshipMap.set(relationship.id, relationship);
|
relationshipMap.set(relationship.id, relationship);
|
||||||
|
let bucket = relationshipsByType.get(relationship.type);
|
||||||
|
if (bucket === undefined) {
|
||||||
|
bucket = new Map();
|
||||||
|
relationshipsByType.set(relationship.type, bucket);
|
||||||
}
|
}
|
||||||
|
bucket.set(relationship.id, relationship);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -25,10 +42,12 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
||||||
|
|
||||||
nodeMap.delete(nodeId);
|
nodeMap.delete(nodeId);
|
||||||
|
|
||||||
// Remove all relationships involving this node
|
// Remove all relationships involving this node — clean up both
|
||||||
|
// indexes in lockstep so the per-type buckets never drift.
|
||||||
for (const [relId, rel] of relationshipMap) {
|
for (const [relId, rel] of relationshipMap) {
|
||||||
if (rel.sourceId === nodeId || rel.targetId === nodeId) {
|
if (rel.sourceId === nodeId || rel.targetId === nodeId) {
|
||||||
relationshipMap.delete(relId);
|
relationshipMap.delete(relId);
|
||||||
|
relationshipsByType.get(rel.type)?.delete(relId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -39,7 +58,11 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
||||||
* Returns true if the relationship existed and was removed, false otherwise.
|
* Returns true if the relationship existed and was removed, false otherwise.
|
||||||
*/
|
*/
|
||||||
const removeRelationship = (relationshipId: string): boolean => {
|
const removeRelationship = (relationshipId: string): boolean => {
|
||||||
return relationshipMap.delete(relationshipId);
|
const rel = relationshipMap.get(relationshipId);
|
||||||
|
if (rel === undefined) return false;
|
||||||
|
relationshipMap.delete(relationshipId);
|
||||||
|
relationshipsByType.get(rel.type)?.delete(relationshipId);
|
||||||
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -67,6 +90,10 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
||||||
|
|
||||||
iterNodes: () => nodeMap.values(),
|
iterNodes: () => nodeMap.values(),
|
||||||
iterRelationships: () => relationshipMap.values(),
|
iterRelationships: () => relationshipMap.values(),
|
||||||
|
iterRelationshipsByType: (type: RelationshipType) => {
|
||||||
|
const bucket = relationshipsByType.get(type);
|
||||||
|
return bucket === undefined ? emptyRelIter() : bucket.values();
|
||||||
|
},
|
||||||
forEachNode(fn: (node: GraphNode) => void) {
|
forEachNode(fn: (node: GraphNode) => void) {
|
||||||
nodeMap.forEach(fn);
|
nodeMap.forEach(fn);
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@
|
||||||
*
|
*
|
||||||
* This file only defines the CLI's KnowledgeGraph with mutation methods.
|
* This file only defines the CLI's KnowledgeGraph with mutation methods.
|
||||||
*/
|
*/
|
||||||
import type { GraphNode, GraphRelationship } from 'gitnexus-shared';
|
import type { GraphNode, GraphRelationship, RelationshipType } from 'gitnexus-shared';
|
||||||
|
|
||||||
// CLI-specific: full KnowledgeGraph with mutation methods for incremental updates
|
// CLI-specific: full KnowledgeGraph with mutation methods for incremental updates
|
||||||
export interface KnowledgeGraph {
|
export interface KnowledgeGraph {
|
||||||
|
|
@ -14,6 +14,17 @@ export interface KnowledgeGraph {
|
||||||
relationships: GraphRelationship[];
|
relationships: GraphRelationship[];
|
||||||
iterNodes: () => IterableIterator<GraphNode>;
|
iterNodes: () => IterableIterator<GraphNode>;
|
||||||
iterRelationships: () => IterableIterator<GraphRelationship>;
|
iterRelationships: () => IterableIterator<GraphRelationship>;
|
||||||
|
/**
|
||||||
|
* Iterate ONLY relationships of the given type, backed by a per-type
|
||||||
|
* index maintained in `addRelationship` / `removeRelationship` /
|
||||||
|
* `removeNode` / `removeNodesByFile`. Returns an empty iterator when
|
||||||
|
* the graph contains no relationships of that type.
|
||||||
|
*
|
||||||
|
* Prefer this over `iterRelationships()` + per-edge type filtering
|
||||||
|
* for hot paths (MRO setup, heritage walks). Backwards-compatible:
|
||||||
|
* existing `iterRelationships()` callers keep working.
|
||||||
|
*/
|
||||||
|
iterRelationshipsByType: (type: RelationshipType) => IterableIterator<GraphRelationship>;
|
||||||
forEachNode: (fn: (node: GraphNode) => void) => void;
|
forEachNode: (fn: (node: GraphNode) => void) => void;
|
||||||
forEachRelationship: (fn: (rel: GraphRelationship) => void) => void;
|
forEachRelationship: (fn: (rel: GraphRelationship) => void) => void;
|
||||||
getNode: (id: string) => GraphNode | undefined;
|
getNode: (id: string) => GraphNode | undefined;
|
||||||
|
|
|
||||||
|
|
@ -329,7 +329,21 @@ interface LanguageProviderConfig {
|
||||||
*
|
*
|
||||||
* Default: undefined (language continues to use legacy DAG).
|
* Default: undefined (language continues to use legacy DAG).
|
||||||
*/
|
*/
|
||||||
readonly emitScopeCaptures?: (sourceText: string, filePath: string) => readonly CaptureMatch[];
|
readonly emitScopeCaptures?: (
|
||||||
|
sourceText: string,
|
||||||
|
filePath: string,
|
||||||
|
/**
|
||||||
|
* Optional pre-parsed tree-sitter Tree the caller has already
|
||||||
|
* produced (e.g. from the parse phase's AST cache). When supplied,
|
||||||
|
* the provider SHOULD skip its own `parser.parse(sourceText)` and
|
||||||
|
* run its capture query against the supplied tree directly. Typed
|
||||||
|
* as `unknown` here to avoid leaking the tree-sitter dependency
|
||||||
|
* into the provider contract — the provider casts at use site.
|
||||||
|
* Cache miss (parameter omitted or undefined) is always safe and
|
||||||
|
* MUST trigger a fresh parse.
|
||||||
|
*/
|
||||||
|
cachedTree?: unknown,
|
||||||
|
) => readonly CaptureMatch[];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interpret a raw `@import.statement` capture group into a `ParsedImport`.
|
* Interpret a raw `@import.statement` capture group into a `ParsedImport`.
|
||||||
|
|
|
||||||
|
|
@ -23,11 +23,36 @@ import { getPythonParser, getPythonScopeQuery } from './query.js';
|
||||||
import { synthesizeReceiverTypeBinding } from './receiver-binding.js';
|
import { synthesizeReceiverTypeBinding } from './receiver-binding.js';
|
||||||
import { computePythonArityMetadata } from './arity-metadata.js';
|
import { computePythonArityMetadata } from './arity-metadata.js';
|
||||||
|
|
||||||
|
// Dev-mode counters for the parse-cache hit-rate. Gated by
|
||||||
|
// `PROF_SCOPE_RESOLUTION=1` to keep the hot path branch-free in
|
||||||
|
// production. Surfaced via `getPythonCaptureCacheStats()` so
|
||||||
|
// benchmarks / debug scripts can verify the cache is being used.
|
||||||
|
const PROF = process.env.PROF_SCOPE_RESOLUTION === '1';
|
||||||
|
let CACHE_HITS = 0;
|
||||||
|
let CACHE_MISSES = 0;
|
||||||
|
export function getPythonCaptureCacheStats(): { hits: number; misses: number } {
|
||||||
|
return { hits: CACHE_HITS, misses: CACHE_MISSES };
|
||||||
|
}
|
||||||
|
export function resetPythonCaptureCacheStats(): void {
|
||||||
|
CACHE_HITS = 0;
|
||||||
|
CACHE_MISSES = 0;
|
||||||
|
}
|
||||||
|
|
||||||
export function emitPythonScopeCaptures(
|
export function emitPythonScopeCaptures(
|
||||||
sourceText: string,
|
sourceText: string,
|
||||||
_filePath: string,
|
_filePath: string,
|
||||||
|
cachedTree?: unknown,
|
||||||
): readonly CaptureMatch[] {
|
): readonly CaptureMatch[] {
|
||||||
const tree = getPythonParser().parse(sourceText);
|
// Skip the parse when the caller (parse phase's ASTCache) already
|
||||||
|
// produced a Tree for this source. Cache miss = re-parse, same as
|
||||||
|
// before. The cachedTree parameter is typed as `unknown` at the
|
||||||
|
// contract layer (see `LanguageProvider.emitScopeCaptures`); cast
|
||||||
|
// here at the use site.
|
||||||
|
let tree = cachedTree as ReturnType<ReturnType<typeof getPythonParser>['parse']> | undefined;
|
||||||
|
if (tree === undefined) {
|
||||||
|
tree = getPythonParser().parse(sourceText);
|
||||||
|
if (PROF) CACHE_MISSES++;
|
||||||
|
} else if (PROF) CACHE_HITS++;
|
||||||
const rawMatches = getPythonScopeQuery().matches(tree.rootNode);
|
const rawMatches = getPythonScopeQuery().matches(tree.rootNode);
|
||||||
|
|
||||||
const out: CaptureMatch[] = [];
|
const out: CaptureMatch[] = [];
|
||||||
|
|
|
||||||
|
|
@ -64,8 +64,11 @@ function buildAdjacency(graph: KnowledgeGraph) {
|
||||||
// Track which edge type each parent link came from
|
// Track which edge type each parent link came from
|
||||||
const parentEdgeType = new Map<string, Map<string, 'EXTENDS' | 'IMPLEMENTS'>>();
|
const parentEdgeType = new Map<string, Map<string, 'EXTENDS' | 'IMPLEMENTS'>>();
|
||||||
|
|
||||||
graph.forEachRelationship((rel) => {
|
// Three typed iterations replace one full-relationship-map scan
|
||||||
if (rel.type === 'EXTENDS' || rel.type === 'IMPLEMENTS') {
|
// with per-edge type checks. Each consumes only the edges of the
|
||||||
|
// type it cares about — see plan
|
||||||
|
// docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 2).
|
||||||
|
for (const rel of graph.iterRelationshipsByType('EXTENDS')) {
|
||||||
let parents = parentMap.get(rel.sourceId);
|
let parents = parentMap.get(rel.sourceId);
|
||||||
if (!parents) {
|
if (!parents) {
|
||||||
parents = [];
|
parents = [];
|
||||||
|
|
@ -78,10 +81,24 @@ function buildAdjacency(graph: KnowledgeGraph) {
|
||||||
edgeTypes = new Map();
|
edgeTypes = new Map();
|
||||||
parentEdgeType.set(rel.sourceId, edgeTypes);
|
parentEdgeType.set(rel.sourceId, edgeTypes);
|
||||||
}
|
}
|
||||||
edgeTypes.set(rel.targetId, rel.type);
|
edgeTypes.set(rel.targetId, 'EXTENDS');
|
||||||
}
|
}
|
||||||
|
for (const rel of graph.iterRelationshipsByType('IMPLEMENTS')) {
|
||||||
|
let parents = parentMap.get(rel.sourceId);
|
||||||
|
if (!parents) {
|
||||||
|
parents = [];
|
||||||
|
parentMap.set(rel.sourceId, parents);
|
||||||
|
}
|
||||||
|
parents.push(rel.targetId);
|
||||||
|
|
||||||
if (rel.type === 'HAS_METHOD') {
|
let edgeTypes = parentEdgeType.get(rel.sourceId);
|
||||||
|
if (!edgeTypes) {
|
||||||
|
edgeTypes = new Map();
|
||||||
|
parentEdgeType.set(rel.sourceId, edgeTypes);
|
||||||
|
}
|
||||||
|
edgeTypes.set(rel.targetId, 'IMPLEMENTS');
|
||||||
|
}
|
||||||
|
for (const rel of graph.iterRelationshipsByType('HAS_METHOD')) {
|
||||||
let methods = methodMap.get(rel.sourceId);
|
let methods = methodMap.get(rel.sourceId);
|
||||||
if (!methods) {
|
if (!methods) {
|
||||||
methods = [];
|
methods = [];
|
||||||
|
|
@ -89,7 +106,6 @@ function buildAdjacency(graph: KnowledgeGraph) {
|
||||||
}
|
}
|
||||||
methods.push(rel.targetId);
|
methods.push(rel.targetId);
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
|
||||||
return { parentMap, methodMap, parentEdgeType };
|
return { parentMap, methodMap, parentEdgeType };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -318,6 +318,7 @@ const processParsingSequential = async (
|
||||||
files: { path: string; content: string }[],
|
files: { path: string; content: string }[],
|
||||||
symbolTable: SymbolTableWriter,
|
symbolTable: SymbolTableWriter,
|
||||||
astCache: ASTCache,
|
astCache: ASTCache,
|
||||||
|
scopeTreeCache: ASTCache | undefined,
|
||||||
onFileProgress?: FileProgressCallback,
|
onFileProgress?: FileProgressCallback,
|
||||||
) => {
|
) => {
|
||||||
const parser = await loadParser();
|
const parser = await loadParser();
|
||||||
|
|
@ -380,6 +381,9 @@ const processParsingSequential = async (
|
||||||
}
|
}
|
||||||
|
|
||||||
astCache.set(file.path, tree);
|
astCache.set(file.path, tree);
|
||||||
|
// Mirror into the cross-phase cache when supplied. parse-impl
|
||||||
|
// clears `astCache` between chunks; `scopeTreeCache` survives.
|
||||||
|
scopeTreeCache?.set(file.path, tree);
|
||||||
|
|
||||||
const provider = getProvider(language);
|
const provider = getProvider(language);
|
||||||
const queryString = provider.treeSitterQueries;
|
const queryString = provider.treeSitterQueries;
|
||||||
|
|
@ -699,6 +703,14 @@ export const processParsing = async (
|
||||||
files: { path: string; content: string }[],
|
files: { path: string; content: string }[],
|
||||||
symbolTable: SymbolTableWriter,
|
symbolTable: SymbolTableWriter,
|
||||||
astCache: ASTCache,
|
astCache: ASTCache,
|
||||||
|
/**
|
||||||
|
* Persistent tree cache (separate from `astCache`, which the caller
|
||||||
|
* clears between chunks). Sequential parses additionally write the
|
||||||
|
* Tree here so cross-phase consumers (scope-resolution) can read it.
|
||||||
|
* Worker-mode parses skip — Trees can't cross MessageChannels.
|
||||||
|
* Pass `undefined` if no consumer needs cross-phase access.
|
||||||
|
*/
|
||||||
|
scopeTreeCache: ASTCache | undefined,
|
||||||
onFileProgress?: FileProgressCallback,
|
onFileProgress?: FileProgressCallback,
|
||||||
workerPool?: WorkerPool,
|
workerPool?: WorkerPool,
|
||||||
): Promise<WorkerExtractedData | null> => {
|
): Promise<WorkerExtractedData | null> => {
|
||||||
|
|
@ -721,6 +733,13 @@ export const processParsing = async (
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: sequential parsing (no pre-extracted data)
|
// Fallback: sequential parsing (no pre-extracted data)
|
||||||
await processParsingSequential(graph, files, symbolTable, astCache, onFileProgress);
|
await processParsingSequential(
|
||||||
|
graph,
|
||||||
|
files,
|
||||||
|
symbolTable,
|
||||||
|
astCache,
|
||||||
|
scopeTreeCache,
|
||||||
|
onFileProgress,
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ import {
|
||||||
getHeritageStrategyForLanguage,
|
getHeritageStrategyForLanguage,
|
||||||
} from '../heritage-processor.js';
|
} from '../heritage-processor.js';
|
||||||
import { createResolutionContext } from '../model/resolution-context.js';
|
import { createResolutionContext } from '../model/resolution-context.js';
|
||||||
import { createASTCache } from '../ast-cache.js';
|
import { ASTCache, createASTCache } from '../ast-cache.js';
|
||||||
import { type PipelineProgress, getLanguageFromFilename } from 'gitnexus-shared';
|
import { type PipelineProgress, getLanguageFromFilename } from 'gitnexus-shared';
|
||||||
import { readFileContents } from '../filesystem-walker.js';
|
import { readFileContents } from '../filesystem-walker.js';
|
||||||
import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js';
|
import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js';
|
||||||
|
|
@ -109,6 +109,13 @@ export async function runChunkedParseAndResolve(
|
||||||
bindingAccumulator: BindingAccumulator;
|
bindingAccumulator: BindingAccumulator;
|
||||||
resolutionContext: ReturnType<typeof createResolutionContext>;
|
resolutionContext: ReturnType<typeof createResolutionContext>;
|
||||||
usedWorkerPool: boolean;
|
usedWorkerPool: boolean;
|
||||||
|
/** AST cache populated by the sequential parse path. Empty when
|
||||||
|
* every chunk ran via the worker pool (workers can't return native
|
||||||
|
* tree-sitter Trees across the MessageChannel). Downstream phases
|
||||||
|
* (e.g. scope-resolution) read from this to skip re-parsing the
|
||||||
|
* same source. See plan
|
||||||
|
* docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4). */
|
||||||
|
astCache: ASTCache;
|
||||||
}> {
|
}> {
|
||||||
const ctx = createResolutionContext();
|
const ctx = createResolutionContext();
|
||||||
const symbolTable = ctx.model.symbols;
|
const symbolTable = ctx.model.symbols;
|
||||||
|
|
@ -220,9 +227,18 @@ export async function runChunkedParseAndResolve(
|
||||||
|
|
||||||
let filesParsedSoFar = 0;
|
let filesParsedSoFar = 0;
|
||||||
|
|
||||||
// AST cache sized for one chunk (sequential fallback uses it for import/call/heritage)
|
// Two caches with different lifetimes:
|
||||||
|
// - `astCache` (chunk-local, cleared between chunks) — call /
|
||||||
|
// heritage / import processors read it during parse to avoid
|
||||||
|
// re-parsing within the same chunk.
|
||||||
|
// - `scopeTreeCache` (total-parseable-sized, never cleared by
|
||||||
|
// parse-impl) — exposed via ParseOutput so scope-resolution can
|
||||||
|
// skip a second tree-sitter parse. Worker-mode parses don't
|
||||||
|
// populate either; consumers fall back to a fresh parse.
|
||||||
|
// See plan docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4).
|
||||||
const maxChunkFiles = chunks.reduce((max, c) => Math.max(max, c.length), 0);
|
const maxChunkFiles = chunks.reduce((max, c) => Math.max(max, c.length), 0);
|
||||||
let astCache = createASTCache(maxChunkFiles);
|
let astCache = createASTCache(maxChunkFiles);
|
||||||
|
const scopeTreeCache = createASTCache(Math.max(parseableScanned.length, 1));
|
||||||
|
|
||||||
// Build import resolution context once — suffix index, file lists, resolve cache.
|
// Build import resolution context once — suffix index, file lists, resolve cache.
|
||||||
const importCtx = buildImportResolutionContext(allPaths);
|
const importCtx = buildImportResolutionContext(allPaths);
|
||||||
|
|
@ -267,6 +283,7 @@ export async function runChunkedParseAndResolve(
|
||||||
chunkFiles,
|
chunkFiles,
|
||||||
symbolTable,
|
symbolTable,
|
||||||
astCache,
|
astCache,
|
||||||
|
scopeTreeCache,
|
||||||
(current, _total, filePath) => {
|
(current, _total, filePath) => {
|
||||||
const globalCurrent = filesParsedSoFar + current;
|
const globalCurrent = filesParsedSoFar + current;
|
||||||
const parsingProgress = 20 + (globalCurrent / totalParseable) * 62;
|
const parsingProgress = 20 + (globalCurrent / totalParseable) * 62;
|
||||||
|
|
@ -595,5 +612,11 @@ export async function runChunkedParseAndResolve(
|
||||||
// sequential fallback handled every chunk (either due to `skipWorkers`,
|
// sequential fallback handled every chunk (either due to `skipWorkers`,
|
||||||
// the file-count/byte thresholds, or a pool-creation failure).
|
// the file-count/byte thresholds, or a pool-creation failure).
|
||||||
usedWorkerPool: workerPool !== undefined,
|
usedWorkerPool: workerPool !== undefined,
|
||||||
|
// Surface the persistent scope cache so downstream phases
|
||||||
|
// (scope-resolution) can skip re-parsing files that the
|
||||||
|
// sequential path already parsed. Survives chunk boundaries; the
|
||||||
|
// chunk-local `astCache` above is intentionally NOT exposed
|
||||||
|
// because parse-impl clears it between chunks.
|
||||||
|
astCache: scopeTreeCache,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ import type {
|
||||||
} from '../workers/parse-worker.js';
|
} from '../workers/parse-worker.js';
|
||||||
import type { createResolutionContext } from '../model/resolution-context.js';
|
import type { createResolutionContext } from '../model/resolution-context.js';
|
||||||
import { runChunkedParseAndResolve } from './parse-impl.js';
|
import { runChunkedParseAndResolve } from './parse-impl.js';
|
||||||
|
import type { ASTCache } from '../ast-cache.js';
|
||||||
|
|
||||||
export interface ParseOutput {
|
export interface ParseOutput {
|
||||||
/**
|
/**
|
||||||
|
|
@ -63,6 +64,15 @@ export interface ParseOutput {
|
||||||
* see `PipelineOptions.workerThresholdsForTest`.
|
* see `PipelineOptions.workerThresholdsForTest`.
|
||||||
*/
|
*/
|
||||||
readonly usedWorkerPool: boolean;
|
readonly usedWorkerPool: boolean;
|
||||||
|
/**
|
||||||
|
* AST cache populated by the sequential parse path. Empty entries
|
||||||
|
* for files that ran through the worker pool (workers can't return
|
||||||
|
* native tree-sitter Trees across the MessageChannel). Downstream
|
||||||
|
* phases (scope-resolution) read from this to skip re-parsing —
|
||||||
|
* cache miss is safe and falls back to a fresh parse. See plan
|
||||||
|
* docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4).
|
||||||
|
*/
|
||||||
|
readonly astCache: ASTCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const parsePhase: PipelinePhase<ParseOutput> = {
|
export const parsePhase: PipelinePhase<ParseOutput> = {
|
||||||
|
|
|
||||||
|
|
@ -38,10 +38,11 @@ export function extractParsedFile(
|
||||||
sourceText: string,
|
sourceText: string,
|
||||||
filePath: string,
|
filePath: string,
|
||||||
onWarn?: ScopeBridgeWarn,
|
onWarn?: ScopeBridgeWarn,
|
||||||
|
cachedTree?: unknown,
|
||||||
): ParsedFile | undefined {
|
): ParsedFile | undefined {
|
||||||
if (provider.emitScopeCaptures === undefined) return undefined;
|
if (provider.emitScopeCaptures === undefined) return undefined;
|
||||||
try {
|
try {
|
||||||
const captures = provider.emitScopeCaptures(sourceText, filePath);
|
const captures = provider.emitScopeCaptures(sourceText, filePath, cachedTree);
|
||||||
return extractScope(captures, filePath, provider);
|
return extractScope(captures, filePath, provider);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = `scope extraction failed for ${filePath}: ${
|
const message = `scope extraction failed for ${filePath}: ${
|
||||||
|
|
|
||||||
|
|
@ -41,10 +41,11 @@ export function buildMro(
|
||||||
nodeLookup: GraphNodeLookup,
|
nodeLookup: GraphNodeLookup,
|
||||||
linearize: LinearizeStrategy,
|
linearize: LinearizeStrategy,
|
||||||
): Map<string /* DefId */, string[] /* DefId[] */> {
|
): Map<string /* DefId */, string[] /* DefId[] */> {
|
||||||
// Step 1: parentsByGraphId.
|
// Step 1: parentsByGraphId — typed iterator skips the per-edge type
|
||||||
|
// check and the millions of CALLS/ACCESSES/IMPORTS/DEFINES edges
|
||||||
|
// that aren't relevant to MRO.
|
||||||
const parentsByGraphId = new Map<string, string[]>();
|
const parentsByGraphId = new Map<string, string[]>();
|
||||||
for (const rel of graph.iterRelationships()) {
|
for (const rel of graph.iterRelationshipsByType('EXTENDS')) {
|
||||||
if (rel.type !== 'EXTENDS') continue;
|
|
||||||
let list = parentsByGraphId.get(rel.sourceId);
|
let list = parentsByGraphId.get(rel.sourceId);
|
||||||
if (list === undefined) {
|
if (list === undefined) {
|
||||||
list = [];
|
list = [];
|
||||||
|
|
|
||||||
|
|
@ -78,6 +78,14 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
||||||
deps: ReadonlyMap<string, PhaseResult<unknown>>,
|
deps: ReadonlyMap<string, PhaseResult<unknown>>,
|
||||||
): Promise<ScopeResolutionOutput> {
|
): Promise<ScopeResolutionOutput> {
|
||||||
const { scannedFiles } = getPhaseOutput<StructureOutput>(deps, 'structure');
|
const { scannedFiles } = getPhaseOutput<StructureOutput>(deps, 'structure');
|
||||||
|
// Reach into the parse phase's AST cache so per-file extract can
|
||||||
|
// skip a second tree-sitter parse. Cache miss is safe (re-parses).
|
||||||
|
// Worker-mode parses leave the cache empty for those files; they
|
||||||
|
// also fall back to a fresh parse — no correctness impact.
|
||||||
|
const { astCache } = getPhaseOutput<{ astCache: { get(path: string): unknown } }>(
|
||||||
|
deps,
|
||||||
|
'parse',
|
||||||
|
);
|
||||||
|
|
||||||
let totalFiles = 0;
|
let totalFiles = 0;
|
||||||
let totalImports = 0;
|
let totalImports = 0;
|
||||||
|
|
@ -110,6 +118,7 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
||||||
{
|
{
|
||||||
graph: ctx.graph,
|
graph: ctx.graph,
|
||||||
files,
|
files,
|
||||||
|
treeCache: astCache,
|
||||||
onWarn: (msg) => {
|
onWarn: (msg) => {
|
||||||
if (isDev) console.warn(`[scope-resolution:${lang}] ${msg}`);
|
if (isDev) console.warn(`[scope-resolution:${lang}] ${msg}`);
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,14 @@ export interface RunScopeResolutionInput {
|
||||||
readonly graph: KnowledgeGraph;
|
readonly graph: KnowledgeGraph;
|
||||||
readonly files: readonly { readonly path: string; readonly content: string }[];
|
readonly files: readonly { readonly path: string; readonly content: string }[];
|
||||||
readonly onWarn?: (message: string) => void;
|
readonly onWarn?: (message: string) => void;
|
||||||
|
/**
|
||||||
|
* Optional pre-parsed-Tree lookup keyed by file path. When the
|
||||||
|
* pipeline's parse phase ran sequentially, it populated an
|
||||||
|
* `ASTCache`; passing that here lets the per-file extract step
|
||||||
|
* skip a second `tree-sitter parser.parse(...)` call. Cache miss
|
||||||
|
* is safe — falls back to a fresh parse inside the provider.
|
||||||
|
*/
|
||||||
|
readonly treeCache?: { get(filePath: string): unknown };
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RunScopeResolutionStats {
|
export interface RunScopeResolutionStats {
|
||||||
|
|
@ -65,8 +73,16 @@ export function runScopeResolution(
|
||||||
// ── Phase 1: extract each file → ParsedFile ────────────────────────────
|
// ── Phase 1: extract each file → ParsedFile ────────────────────────────
|
||||||
const parsedFiles: ParsedFile[] = [];
|
const parsedFiles: ParsedFile[] = [];
|
||||||
let filesSkipped = 0;
|
let filesSkipped = 0;
|
||||||
|
const treeCache = input.treeCache;
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const parsed = extractParsedFile(provider.languageProvider, file.content, file.path, onWarn);
|
const cachedTree = treeCache?.get(file.path);
|
||||||
|
const parsed = extractParsedFile(
|
||||||
|
provider.languageProvider,
|
||||||
|
file.content,
|
||||||
|
file.path,
|
||||||
|
onWarn,
|
||||||
|
cachedTree,
|
||||||
|
);
|
||||||
if (parsed === undefined) {
|
if (parsed === undefined) {
|
||||||
filesSkipped++;
|
filesSkipped++;
|
||||||
continue;
|
continue;
|
||||||
|
|
|
||||||
|
|
@ -247,4 +247,63 @@ describe('createKnowledgeGraph', () => {
|
||||||
expect(remaining[0].sourceId).toBe('fn:b');
|
expect(remaining[0].sourceId).toBe('fn:b');
|
||||||
expect(remaining[0].targetId).toBe('fn:c');
|
expect(remaining[0].targetId).toBe('fn:c');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ─── iterRelationshipsByType ───────────────────────────────────────
|
||||||
|
|
||||||
|
describe('iterRelationshipsByType', () => {
|
||||||
|
it('yields only the requested type', () => {
|
||||||
|
const g = createKnowledgeGraph();
|
||||||
|
g.addRelationship(makeRel('fn:a', 'fn:b', 'CALLS'));
|
||||||
|
g.addRelationship(makeRel('fn:b', 'fn:c', 'CALLS'));
|
||||||
|
g.addRelationship(makeRel('cls:X', 'cls:Y', 'EXTENDS'));
|
||||||
|
g.addRelationship(makeRel('cls:Y', 'cls:Z', 'EXTENDS'));
|
||||||
|
|
||||||
|
expect([...g.iterRelationshipsByType('CALLS')]).toHaveLength(2);
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns a fresh empty iterator when the type has no edges', () => {
|
||||||
|
const g = createKnowledgeGraph();
|
||||||
|
g.addRelationship(makeRel('fn:a', 'fn:b', 'CALLS'));
|
||||||
|
// Two consecutive calls must each be exhaustible — guards against
|
||||||
|
// returning a single shared exhausted iterator.
|
||||||
|
expect([...g.iterRelationshipsByType('IMPLEMENTS')]).toHaveLength(0);
|
||||||
|
expect([...g.iterRelationshipsByType('IMPLEMENTS')]).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reflects removeRelationship on both indexes', () => {
|
||||||
|
const g = createKnowledgeGraph();
|
||||||
|
g.addRelationship(makeRel('cls:X', 'cls:Y', 'EXTENDS'));
|
||||||
|
g.addRelationship(makeRel('cls:Y', 'cls:Z', 'EXTENDS'));
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(2);
|
||||||
|
|
||||||
|
g.removeRelationship('cls:X-EXTENDS-cls:Y');
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(1);
|
||||||
|
expect(g.relationshipCount).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reflects removeNode on both indexes', () => {
|
||||||
|
const g = createKnowledgeGraph();
|
||||||
|
g.addNode(makeNode('cls:X', 'X', 'src/x.ts'));
|
||||||
|
g.addNode(makeNode('cls:Y', 'Y', 'src/y.ts'));
|
||||||
|
g.addRelationship(makeRel('cls:X', 'cls:Y', 'EXTENDS'));
|
||||||
|
g.addRelationship(makeRel('cls:X', 'cls:Y', 'IMPLEMENTS'));
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(1);
|
||||||
|
expect([...g.iterRelationshipsByType('IMPLEMENTS')]).toHaveLength(1);
|
||||||
|
|
||||||
|
g.removeNode('cls:Y');
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(0);
|
||||||
|
expect([...g.iterRelationshipsByType('IMPLEMENTS')]).toHaveLength(0);
|
||||||
|
expect([...g.iterRelationships()]).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('dedupes by id across both indexes', () => {
|
||||||
|
const g = createKnowledgeGraph();
|
||||||
|
const rel = makeRel('cls:X', 'cls:Y', 'EXTENDS');
|
||||||
|
g.addRelationship(rel);
|
||||||
|
g.addRelationship(rel); // dedup by id
|
||||||
|
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(1);
|
||||||
|
expect(g.relationshipCount).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue