mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-16 23:43:12 +00:00
perf(scope-resolution): bound tree-cache lifetime + gate population
Address P1 residuals from ce:review of 8c6f5cee:
- Dispose scopeTreeCache at end of scopeResolutionPhase via
astCache.clear(). Trees were previously retained for the full
pipeline (10-100x memory regression on large repos). Downstream
phases (mro, community, csv-generator) never read them.
- Gate scopeTreeCache.set on provider.emitScopeCaptures !== undefined.
Polyglot repos no longer retain Trees for languages with no
scope-resolution consumer.
- PROF_SCOPE_RESOLUTION=1 now warns when workers engage, since
Trees can't cross MessageChannels so the cache will be empty for
worker-parsed files — prevents a silent perf cliff once a repo
crosses the worker-pool threshold.
Tests: 26/26 graph unit, 299/299 scope-resolution unit, 191/191
python integration both flag paths.
This commit is contained in:
parent
8c6f5ceeab
commit
d4fd96fd79
3 changed files with 49 additions and 9 deletions
|
|
@ -381,11 +381,15 @@ const processParsingSequential = async (
|
|||
}
|
||||
|
||||
astCache.set(file.path, tree);
|
||||
// Mirror into the cross-phase cache when supplied. parse-impl
|
||||
// clears `astCache` between chunks; `scopeTreeCache` survives.
|
||||
scopeTreeCache?.set(file.path, tree);
|
||||
|
||||
const provider = getProvider(language);
|
||||
// Mirror into the cross-phase cache only when the language has a
|
||||
// scope-resolution consumer — otherwise we retain Trees no one
|
||||
// reads. parse-impl clears `astCache` between chunks;
|
||||
// `scopeTreeCache` survives until scope-resolution disposes it.
|
||||
if (provider.emitScopeCaptures !== undefined) {
|
||||
scopeTreeCache?.set(file.path, tree);
|
||||
}
|
||||
const queryString = provider.treeSitterQueries;
|
||||
if (!queryString) {
|
||||
continue;
|
||||
|
|
@ -715,6 +719,15 @@ export const processParsing = async (
|
|||
workerPool?: WorkerPool,
|
||||
): Promise<WorkerExtractedData | null> => {
|
||||
if (workerPool) {
|
||||
if (scopeTreeCache !== undefined && process.env.PROF_SCOPE_RESOLUTION === '1') {
|
||||
// Trees can't cross MessageChannels, so worker-parsed files land
|
||||
// in scope-resolution with an empty cache and get re-parsed.
|
||||
// Surfacing this in PROF mode prevents silent perf cliffs when
|
||||
// a repo crosses the worker-pool threshold.
|
||||
console.warn(
|
||||
`[scope-resolution prof] worker pool engaged for ${files.length} files — cross-phase tree cache will be empty; scope-resolution re-parses.`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await processParsingWithWorkers(
|
||||
graph,
|
||||
|
|
|
|||
|
|
@ -82,10 +82,9 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
// skip a second tree-sitter parse. Cache miss is safe (re-parses).
|
||||
// Worker-mode parses leave the cache empty for those files; they
|
||||
// also fall back to a fresh parse — no correctness impact.
|
||||
const { astCache } = getPhaseOutput<{ astCache: { get(path: string): unknown } }>(
|
||||
deps,
|
||||
'parse',
|
||||
);
|
||||
const { astCache } = getPhaseOutput<{
|
||||
astCache: { get(path: string): unknown; clear(): void };
|
||||
}>(deps, 'parse');
|
||||
|
||||
let totalFiles = 0;
|
||||
let totalImports = 0;
|
||||
|
|
@ -143,6 +142,14 @@ export const scopeResolutionPhase: PipelinePhase<ScopeResolutionOutput> = {
|
|||
}
|
||||
}
|
||||
|
||||
// Dispose the cross-phase Tree cache — scope-resolution is the
|
||||
// only consumer. Holding Trees past this point is pure memory
|
||||
// pressure: downstream phases (mro, community, csv-generator)
|
||||
// never read them, and tree-sitter Trees hold native-heap memory
|
||||
// under WASM runtimes. ASTCache.clear() fires the LRU dispose
|
||||
// handler which calls tree.delete?.() on each retained Tree.
|
||||
astCache.clear();
|
||||
|
||||
if (!anyRan) return NOOP_OUTPUT;
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -258,8 +258,28 @@ describe('createKnowledgeGraph', () => {
|
|||
g.addRelationship(makeRel('cls:X', 'cls:Y', 'EXTENDS'));
|
||||
g.addRelationship(makeRel('cls:Y', 'cls:Z', 'EXTENDS'));
|
||||
|
||||
expect([...g.iterRelationshipsByType('CALLS')]).toHaveLength(2);
|
||||
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(2);
|
||||
const calls = [...g.iterRelationshipsByType('CALLS')];
|
||||
const extends_ = [...g.iterRelationshipsByType('EXTENDS')];
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(extends_).toHaveLength(2);
|
||||
// Identity assertions guard against a bucket-key swap bug that
|
||||
// would return the wrong edges with the right count.
|
||||
expect(calls.every((r) => r.type === 'CALLS')).toBe(true);
|
||||
expect(extends_.every((r) => r.type === 'EXTENDS')).toBe(true);
|
||||
expect(new Set(calls.map((r) => r.sourceId))).toEqual(new Set(['fn:a', 'fn:b']));
|
||||
});
|
||||
|
||||
it('retains an empty bucket after last edge removed and reuses it on re-add', () => {
|
||||
const g = createKnowledgeGraph();
|
||||
g.addRelationship(makeRel('cls:X', 'cls:Y', 'EXTENDS'));
|
||||
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(1);
|
||||
g.removeRelationship('cls:X-EXTENDS-cls:Y');
|
||||
expect([...g.iterRelationshipsByType('EXTENDS')]).toHaveLength(0);
|
||||
// Re-add the same type — bucket must still be live.
|
||||
g.addRelationship(makeRel('cls:A', 'cls:B', 'EXTENDS'));
|
||||
const again = [...g.iterRelationshipsByType('EXTENDS')];
|
||||
expect(again).toHaveLength(1);
|
||||
expect(again[0].sourceId).toBe('cls:A');
|
||||
});
|
||||
|
||||
it('returns a fresh empty iterator when the type has no edges', () => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue