diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 35a59dab4..e9770fe9f 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -766,6 +766,15 @@ export const processCalls = async ( importedRawReturnTypesMap?: ReadonlyMap>, heritageMap?: HeritageMap, bindingAccumulator?: BindingAccumulator, + /** + * Optional cache for compiled `Parser.Query` objects keyed by language name. + * When provided, compiled queries are reused across calls instead of being + * re-compiled from the query string for every file. Callers that invoke + * `processCalls` many times with single-file batches (e.g. the cross-file + * propagation phase) should pass a long-lived map here to avoid O(N) + * query recompilation overhead. + */ + compiledQueryCache?: Map, ): Promise => { const parser = await loadParser(); const collectedHeritage: ExtractedHeritage[] = []; @@ -843,7 +852,11 @@ export const processCalls = async ( let matches; try { const lang = parser.getLanguage(); - const query = new Parser.Query(lang, queryStr); + let query = compiledQueryCache?.get(language); + if (!query) { + query = new Parser.Query(lang, queryStr); + compiledQueryCache?.set(language, query); + } matches = query.matches(tree.rootNode); } catch (queryError) { logger.warn({ queryError }, `Query error for ${file.path}:`); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts index 1d8d000fc..53913b954 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts @@ -16,12 +16,13 @@ import { } from '../call-processor.js'; import type { createResolutionContext } from '../model/resolution-context.js'; import { createASTCache } from '../ast-cache.js'; -import { type PipelineProgress, getLanguageFromFilename } from 'gitnexus-shared'; +import { type PipelineProgress, getLanguageFromFilename, type SupportedLanguages } from 'gitnexus-shared'; import { readFileContents } from '../filesystem-walker.js'; import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js'; import { topologicalLevelSort } from '../utils/graph-sort.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import { isDev } from '../utils/env.js'; +import type Parser from 'tree-sitter'; import { logger } from '../../logger.js'; /** Max AST trees to keep in LRU cache for cross-file binding propagation. */ @@ -128,8 +129,33 @@ export async function runCrossFileBindingPropagation( const crossFileStart = Date.now(); const crossFileDeadlineMs = getCrossFileTimeoutMs(); const astCache = createASTCache(AST_CACHE_CAP); + // Compiled query objects keyed by language name. Shared across all processCalls + // invocations in this phase so the same tree-sitter query string is only + // compiled once per language instead of once per file (O(1) vs O(N)). + const queryCache = new Map(); let timedOut = false; + // Snapshot total topological candidates for progress math. We walk the + // levels once more here (fast — no I/O) so we can report meaningful + // percentages rather than a frozen display. + let totalCandidates = 0; + for (const level of levels) { + for (const filePath of level) { + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + const imports = ctx.namedImportMap.get(filePath); + if (!imports) continue; + if (!allPathSet.has(filePath)) continue; + const lang = getLanguageFromFilename(filePath); + if (!lang || !isLanguageAvailable(lang)) continue; + totalCandidates++; + } + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + } + const cappedTotal = Math.min(totalCandidates, MAX_CROSS_FILE_REPROCESS); + + /** Emit a progress event every PROGRESS_INTERVAL files so the UI stays alive. */ + const PROGRESS_INTERVAL = 25; + for (const level of levels) { if (timedOut) break; @@ -214,8 +240,24 @@ export async function runCrossFileBindingPropagation( bindings.size > 0 ? bindings : undefined, importedReturnTypesMap.size > 0 ? importedReturnTypesMap : undefined, importedRawReturnTypesMap.size > 0 ? importedRawReturnTypesMap : undefined, + undefined, + undefined, + queryCache, ); crossFileResolved++; + + // Emit progress every PROGRESS_INTERVAL files so the UI shows real + // movement instead of a frozen display (cross-file can take minutes + // on large repos with many cross-file imports). + if (crossFileResolved % PROGRESS_INTERVAL === 0 || crossFileResolved === cappedTotal) { + const pct = cappedTotal > 0 ? Math.round((crossFileResolved / cappedTotal) * 8) : 0; + onProgress({ + phase: 'parsing', + percent: 82 + pct, + message: `Cross-file type propagation (${crossFileResolved}/${cappedTotal} files)...`, + stats: { filesProcessed: crossFileResolved, totalFiles, nodesCreated: graph.nodeCount }, + }); + } } if (crossFileResolved >= MAX_CROSS_FILE_REPROCESS) {