diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 1161a0772..01ef73452 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -135,6 +135,11 @@ export async function runChunkedParseAndResolve( * source. See plan * docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4). */ scopeTreeCache: ASTCache; + /** Worker-produced ParsedFile artifacts aggregated across chunks. + * Threaded into scope-resolution as a re-extract cache so the warm- + * cache analyze run can skip the dominant `extractParsedFile` cost + * (otherwise ~58s on a 1000-file repo). */ + parsedFiles: import('gitnexus-shared').ParsedFile[]; }> { const ctx = createResolutionContext(); const symbolTable = ctx.model.symbols; @@ -158,6 +163,14 @@ export async function runChunkedParseAndResolve( ); } + // Sort by path so chunk membership is stable across runs even when + // the filesystem returns scan order non-deterministically. Without + // this, the parse cache misses on every run because chunk boundaries + // shift even when no source file content has changed. Sort is + // ascending alphabetical — the comparator works for both POSIX and + // Windows path separators since both are `string` in JS. + parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)); + const totalParseable = parseableScanned.length; if (totalParseable === 0) { @@ -287,6 +300,12 @@ export async function runChunkedParseAndResolve( const deferredWorkerHeritage: ExtractedHeritage[] = []; const deferredConstructorBindings: FileConstructorBindings[] = []; const deferredAssignments: ExtractedAssignment[] = []; + // Aggregated per-file ParsedFile artifacts produced by workers' calls + // to `extractParsedFile`. Threaded through to the scope-resolution + // phase so it can SKIP its own re-extraction on cache hits — this is + // the second-half of the parse-cache speedup since scope-resolution's + // re-parse otherwise dominates the warm-cache wall-clock time. + const allParsedFiles: import('gitnexus-shared').ParsedFile[] = []; // Incremental parse cache (Option B): chunk-level content-addressed. // When the chunk's (filePath, content-hash) signature matches a prior @@ -428,6 +447,12 @@ export async function runChunkedParseAndResolve( for (const item of chunkWorkerData.heritage) deferredWorkerHeritage.push(item); for (const item of chunkWorkerData.constructorBindings) deferredConstructorBindings.push(item); + // Aggregate worker-produced ParsedFile artifacts so scope- + // resolution can use them as a re-extraction cache (skips its + // own tree-sitter re-parse on warm runs). + if (chunkWorkerData.parsedFiles?.length) { + for (const item of chunkWorkerData.parsedFiles) allParsedFiles.push(item); + } if (chunkWorkerData.assignments?.length) { for (const item of chunkWorkerData.assignments) deferredAssignments.push(item); } @@ -700,5 +725,12 @@ export async function runChunkedParseAndResolve( // chunk-local `astCache` above is intentionally NOT exposed // because parse-impl clears it between chunks. scopeTreeCache, + // Per-file ParsedFile artifacts produced by workers' calls to + // `extractParsedFile`. Empty when only the sequential path ran + // (sequential doesn't go through the worker, and extracts ParsedFile + // inline rather than emitting it). Consumed by scope-resolution as + // a re-extraction cache: when the file's ParsedFile is here, + // scope-resolution skips its own `extractParsedFile` call. + parsedFiles: allParsedFiles, }; } diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index a20d1e4b0..a3fa81be7 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -20,6 +20,7 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import type { StructureOutput } from './structure.js'; import type { BindingAccumulator } from '../binding-accumulator.js'; +import type { ParsedFile } from 'gitnexus-shared'; import type { ExtractedFetchCall, ExtractedRoute, @@ -81,6 +82,19 @@ export interface ParseOutput { * `scopeTreeCache.clear()` after its extract loop finishes. */ readonly scopeTreeCache: ASTCache; + /** + * Per-file `ParsedFile` artifacts produced by workers' calls to + * `extractParsedFile`. Threaded through to `scopeResolutionPhase` + * as a re-extraction cache: when a file's ParsedFile is present here, + * scope-resolution can skip its own `extractParsedFile` (which would + * otherwise re-parse the file with tree-sitter on the main thread, + * costing ~58s on a 1000-file repo). + * + * Empty for files that went through the sequential parse fallback — + * sequential doesn't emit ParsedFile artifacts; scope-resolution + * falls back to a fresh extract for those. + */ + readonly parsedFiles: readonly ParsedFile[]; } export const parsePhase: PipelinePhase = { diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index c2fda9777..98a9f8994 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -93,13 +93,25 @@ export const scopeResolutionPhase: PipelinePhase = { // Worker-mode parses leave the cache empty for those files; they // also fall back to a fresh parse — no correctness impact. const parseOutput = getPhaseOutput(deps, 'parse'); - const { scopeTreeCache, resolutionContext } = parseOutput; + const { scopeTreeCache, resolutionContext, parsedFiles: workerParsedFiles } = parseOutput; // SemanticModel populated during `parse`: scope-resolution consumes // TypeRegistry / MethodRegistry / SymbolTable lookups instead of // rebuilding parallel indexes. See ARCHITECTURE.md § "Semantic-model // source of truth". const model = resolutionContext.model; + // Build a per-file lookup of ParsedFile artifacts the workers (or + // sequential extracts) already produced. Threading this into + // `runScopeResolution` lets the per-language extract loop short- + // circuit `extractParsedFile` — the dominant cost on the warm-cache + // path, since workers can't return tree-sitter Trees across the + // MessageChannel and scope-resolution would otherwise re-parse + // every file from scratch on the main thread. + const preExtractedByPath = new Map(); + for (const pf of workerParsedFiles) { + preExtractedByPath.set(pf.filePath, pf); + } + let totalFiles = 0; let totalImports = 0; let totalRefs = 0; @@ -143,6 +155,7 @@ export const scopeResolutionPhase: PipelinePhase = { files, treeCache: scopeTreeCache, resolutionConfig, + preExtractedParsedFiles: preExtractedByPath, onWarn: (msg) => { if (isSemanticModelValidatorEnabled()) { logger.warn(`[scope-resolution:${lang}] ${msg}`); diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index 558c9ef30..73d585aae 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -72,6 +72,22 @@ interface RunScopeResolutionInput { * provider doesn't supply a config loader. */ readonly resolutionConfig?: unknown; + /** + * Pre-extracted ParsedFile artifacts keyed by file path. When a + * file is present here, the extract loop reuses it directly and + * skips `extractParsedFile` (which would re-parse the file with + * tree-sitter on the main thread). Only files matching the + * provider's language are honored — the loop verifies this + * implicitly by language filter at the call-site (scopeResolution + * phase). + * + * Worker-mode parses produce these ParsedFile artifacts as a side + * effect of `extractParsedFile` running inside the worker; threading + * them here is what lets the warm-cache analyze run skip the ~58s + * scope-resolution re-parse loop on a multi-thousand-file repo. + * Cache miss is safe — falls back to fresh extract. + */ + readonly preExtractedParsedFiles?: ReadonlyMap; } interface RunScopeResolutionStats { @@ -104,22 +120,39 @@ export function runScopeResolution( const parsedFiles: ParsedFile[] = []; let filesSkipped = 0; const treeCache = input.treeCache; + const preExtracted = input.preExtractedParsedFiles; + let preExtractedHits = 0; for (const file of files) { - const cachedTree = treeCache?.get(file.path); - const parsed = extractParsedFile( - provider.languageProvider, - file.content, - file.path, - onWarn, - cachedTree, - ); + let parsed: ParsedFile | undefined; + // Fast path: a worker (during the parse phase) already produced a + // ParsedFile for this file via `extractParsedFile`. Reuse it + // directly — skips a tree-sitter re-parse on the main thread. + if (preExtracted !== undefined) { + parsed = preExtracted.get(file.path); + if (parsed !== undefined) preExtractedHits++; + } if (parsed === undefined) { - filesSkipped++; - continue; + const cachedTree = treeCache?.get(file.path); + parsed = extractParsedFile( + provider.languageProvider, + file.content, + file.path, + onWarn, + cachedTree, + ); + if (parsed === undefined) { + filesSkipped++; + continue; + } } provider.populateOwners(parsed); parsedFiles.push(parsed); } + if (PROF && preExtracted !== undefined) { + logger.warn( + `[scope-resolution prof] pre-extracted hits: ${preExtractedHits}/${files.length}`, + ); + } provider.populateWorkspaceOwners?.(parsedFiles, { fileContents: getFileContents() }); // Reconcile scope-resolution's ownership view into the SemanticModel. diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 11a999c4a..9a7dda1c7 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -67,6 +67,33 @@ export const computeChunkHash = (entries: Array<{ filePath: string; contentHash: return sha256Hex(joined); }; +/** + * JSON replacer that round-trips Map/Set instances through plain JSON. + * + * `ParseWorkerResult.parsedFiles[*].scopes[*].typeBindings` is a + * `ReadonlyMap`; without this transform it serializes + * to `{}` and downstream code that iterates / `.get()`s on it crashes + * with "is not iterable". Applied symmetrically by `mapReviver` on + * load so the in-memory shape stays Map-typed. + */ +const MAP_TAG = '__$mapEntries$__'; +const SET_TAG = '__$setValues$__'; + +const mapReplacer = (_key: string, value: unknown): unknown => { + if (value instanceof Map) return { [MAP_TAG]: Array.from(value.entries()) }; + if (value instanceof Set) return { [SET_TAG]: Array.from(value.values()) }; + return value; +}; + +const mapReviver = (_key: string, value: unknown): unknown => { + if (value && typeof value === 'object') { + const v = value as Record; + if (Array.isArray(v[MAP_TAG])) return new Map(v[MAP_TAG] as [unknown, unknown][]); + if (Array.isArray(v[SET_TAG])) return new Set(v[SET_TAG] as unknown[]); + } + return value; +}; + /** * Load the parse cache. Returns an empty cache on any failure (missing * file, corrupt JSON, version mismatch). Never throws on a normal load. @@ -75,7 +102,7 @@ export const loadParseCache = async (storagePath: string): Promise = const cachePath = path.join(storagePath, CACHE_FILENAME); try { const raw = await fs.readFile(cachePath, 'utf-8'); - const data = JSON.parse(raw) as ParseCacheFile; + const data = JSON.parse(raw, mapReviver) as ParseCacheFile; if ( typeof data !== 'object' || data === null || @@ -112,7 +139,7 @@ export const saveParseCache = async ( }; // Compact JSON; this file can be tens of MB on a large repo and pretty- // printing roughly doubles size for no value. - await fs.writeFile(tmpPath, JSON.stringify(out), 'utf-8'); + await fs.writeFile(tmpPath, JSON.stringify(out, mapReplacer), 'utf-8'); await fs.rename(tmpPath, cachePath); };