diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index e5d4c4054d..722f73c8b2 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -286,7 +286,11 @@ const processParsingSequential = async ( // Skip very large files — they can crash tree-sitter or cause OOM if (file.content.length > 512 * 1024) continue; - await loadLanguage(language, file.path); + try { + await loadLanguage(language, file.path); + } catch { + continue; // parser unavailable — already warned in pipeline + } let tree; try { diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index bccfa79ee3..8a7d9d0ee8 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -11,6 +11,7 @@ import { createASTCache } from './ast-cache.js'; import { PipelineProgress, PipelineResult } from '../../types/pipeline.js'; import { walkRepositoryPaths, readFileContents } from './filesystem-walker.js'; import { getLanguageFromFilename } from './utils.js'; +import { isLanguageAvailable } from '../tree-sitter/parser-loader.js'; import { createWorkerPool, WorkerPool } from './workers/worker-pool.js'; const isDev = process.env.NODE_ENV === 'development'; @@ -88,7 +89,23 @@ export const runPipelineFromRepo = async ( // Group parseable files into byte-budget chunks so only ~20MB of source // is in memory at a time. Each chunk is: read → parse → extract → free. - const parseableScanned = scannedFiles.filter(f => getLanguageFromFilename(f.path)); + const parseableScanned = scannedFiles.filter(f => { + const lang = getLanguageFromFilename(f.path); + return lang && isLanguageAvailable(lang); + }); + + // Warn about files skipped due to unavailable parsers + const skippedByLang = new Map(); + for (const f of scannedFiles) { + const lang = getLanguageFromFilename(f.path); + if (lang && !isLanguageAvailable(lang)) { + skippedByLang.set(lang, (skippedByLang.get(lang) || 0) + 1); + } + } + for (const [lang, count] of skippedByLang) { + console.warn(`Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild tree-sitter-${lang}`); + } + const totalParseable = parseableScanned.length; // Build byte-budget chunks diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 25836e7e96..036693d76c 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -576,14 +576,22 @@ const processBatch = (files: ParseWorkerInput[], onProgress?: (filesProcessed: n // Process regular files for this language if (regularFiles.length > 0) { - setLanguage(language, regularFiles[0].path); - processFileGroup(regularFiles, language, queryString, result, onFileProcessed); + try { + setLanguage(language, regularFiles[0].path); + processFileGroup(regularFiles, language, queryString, result, onFileProcessed); + } catch { + // parser unavailable — skip this language group + } } // Process tsx files separately (different grammar) if (tsxFiles.length > 0) { - setLanguage(language, tsxFiles[0].path); - processFileGroup(tsxFiles, language, queryString, result, onFileProcessed); + try { + setLanguage(language, tsxFiles[0].path); + processFileGroup(tsxFiles, language, queryString, result, onFileProcessed); + } catch { + // parser unavailable — skip this language group + } } } diff --git a/gitnexus/src/core/tree-sitter/parser-loader.ts b/gitnexus/src/core/tree-sitter/parser-loader.ts index 3cc9fc0259..f7eadc1ab1 100644 --- a/gitnexus/src/core/tree-sitter/parser-loader.ts +++ b/gitnexus/src/core/tree-sitter/parser-loader.ts @@ -36,6 +36,9 @@ const languageMap: Record = { ...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}), }; +export const isLanguageAvailable = (language: SupportedLanguages): boolean => + language in languageMap; + export const loadParser = async (): Promise => { if (parser) return parser; parser = new Parser();