From 33aa774f036637f80d0195e03e0bca9640e06a77 Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Fri, 20 Feb 2026 20:24:48 +0530 Subject: [PATCH] =?UTF-8?q?fix:=20worker=20pool=20robustness=20=E2=80=94?= =?UTF-8?q?=20timeout,=20exit=20handler,=20512KB=20guards,=20error=20repor?= =?UTF-8?q?ting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- .claude/settings.local.json | 8 ++- AGENTS.md | 2 +- CLAUDE.md | 2 +- gitnexus/src/core/embeddings/embedder.ts | 11 +++- .../src/core/ingestion/filesystem-walker.ts | 24 ++++++-- .../src/core/ingestion/parsing-processor.ts | 5 +- .../core/ingestion/workers/parse-worker.ts | 16 ++++-- .../src/core/ingestion/workers/worker-pool.ts | 55 ++++++++++++++++--- 8 files changed, 99 insertions(+), 24 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index a2d91f924..380bbf354 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -65,7 +65,13 @@ "WebFetch(domain:smartlogic.io)", "Bash(ls:*)", "Bash(wc:*)", - "Bash(grep:*)" + "Bash(grep:*)", + "Bash(powershell -Command:*)", + "Bash(cmd /c \"dir /s C:\\\\Users\\\\ADMIN\\\\.cache\\\\huggingface 2>nul | findstr /i \"\"File\\(s\\)\"\"\")", + "Bash(du:*)", + "mcp__desktop-commander__list_directory", + "Bash(python3 -c \":*)", + "mcp__gitnexus__detect_changes" ] }, "enableAllProjectMcpServers": true, diff --git a/AGENTS.md b/AGENTS.md index 290e07e9b..da4101fe0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ # GitNexus MCP -This project is indexed by GitNexus as **GitnexusV2** (1295 symbols, 3262 relationships, 99 execution flows). +This project is indexed by GitNexus as **GitnexusV2** (1312 symbols, 3315 relationships, 101 execution flows). GitNexus provides a knowledge graph over this codebase — call chains, blast radius, execution flows, and semantic search. diff --git a/CLAUDE.md b/CLAUDE.md index cf96c279c..54f787339 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,7 +1,7 @@ # GitNexus MCP -This project is indexed by GitNexus as **GitnexusV2** (1295 symbols, 3262 relationships, 99 execution flows). +This project is indexed by GitNexus as **GitnexusV2** (1312 symbols, 3315 relationships, 101 execution flows). GitNexus provides a knowledge graph over this codebase — call chains, blast radius, execution flows, and semantic search. diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index aeb084d1a..eb1139202 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -1,12 +1,19 @@ /** * Embedder Module - * + * * Singleton factory for transformers.js embedding pipeline. * Handles model loading, caching, and both single and batch embedding operations. - * + * * Uses snowflake-arctic-embed-xs by default (22M params, 384 dims, ~90MB) */ +// Suppress ONNX Runtime native warnings (e.g. VerifyEachNodeIsAssignedToAnEp) +// Must be set BEFORE onnxruntime-node is imported by transformers.js +// Level 3 = Error only (skips Warning/Info) +if (!process.env.ORT_LOG_LEVEL) { + process.env.ORT_LOG_LEVEL = '3'; +} + import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers'; import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js'; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index bfad1574c..c7a2e5d47 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -10,6 +10,9 @@ export interface FileEntry { const READ_CONCURRENCY = 32; +/** Skip files larger than 512KB — they're usually generated/vendored and crash tree-sitter */ +const MAX_FILE_SIZE = 512 * 1024; + export const walkRepository = async ( repoPath: string, onProgress?: (current: number, total: number, filePath: string) => void @@ -23,19 +26,26 @@ export const walkRepository = async ( const filtered = files.filter(file => !shouldIgnorePath(file)); const entries: FileEntry[] = []; let processed = 0; + let skippedLarge = 0; for (let start = 0; start < filtered.length; start += READ_CONCURRENCY) { const batch = filtered.slice(start, start + READ_CONCURRENCY); const results = await Promise.allSettled( - batch.map(relativePath => - fs.readFile(path.join(repoPath, relativePath), 'utf-8') - .then(content => ({ path: relativePath.replace(/\\/g, '/'), content })) - ) + batch.map(async relativePath => { + const fullPath = path.join(repoPath, relativePath); + const stat = await fs.stat(fullPath); + if (stat.size > MAX_FILE_SIZE) { + skippedLarge++; + return null; + } + const content = await fs.readFile(fullPath, 'utf-8'); + return { path: relativePath.replace(/\\/g, '/'), content }; + }) ); for (const result of results) { processed++; - if (result.status === 'fulfilled') { + if (result.status === 'fulfilled' && result.value !== null) { entries.push(result.value); onProgress?.(processed, filtered.length, result.value.path); } else { @@ -44,5 +54,9 @@ export const walkRepository = async ( } } + if (skippedLarge > 0) { + console.warn(` Skipped ${skippedLarge} files larger than ${MAX_FILE_SIZE / 1024}KB`); + } + return entries; }; diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 4d02e3f89..e7bb344a4 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -209,6 +209,9 @@ const processParsingSequential = async ( if (!language) continue; + // Skip very large files — they can crash tree-sitter or cause OOM + if (file.content.length > 512 * 1024) continue; + await loadLanguage(language, file.path); let tree; @@ -335,7 +338,7 @@ export const processParsing = async ( try { return await processParsingWithWorkers(graph, files, symbolTable, astCache, workerPool, onFileProgress); } catch (err) { - console.warn('Worker pool parsing failed, falling back to sequential:', err); + console.warn('Worker pool parsing failed, falling back to sequential:', err instanceof Error ? err.message : err); } } diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 573884f36..31e4d02e3 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -402,6 +402,9 @@ const processFileGroup = ( } for (const file of files) { + // Skip very large files — they can crash tree-sitter or cause OOM + if (file.content.length > 512 * 1024) continue; + let tree; try { tree = parser.parse(file.content, undefined, { bufferSize: 1024 * 256 }); @@ -528,8 +531,13 @@ const processFileGroup = ( // ============================================================================ parentPort!.on('message', (files: ParseWorkerInput[]) => { - const result = processBatch(files, (filesProcessed) => { - parentPort!.postMessage({ type: 'progress', filesProcessed }); - }); - parentPort!.postMessage({ type: 'result', data: result }); + try { + const result = processBatch(files, (filesProcessed) => { + parentPort!.postMessage({ type: 'progress', filesProcessed }); + }); + parentPort!.postMessage({ type: 'result', data: result }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + parentPort!.postMessage({ type: 'error', error: message }); + } }); diff --git a/gitnexus/src/core/ingestion/workers/worker-pool.ts b/gitnexus/src/core/ingestion/workers/worker-pool.ts index aad23b178..5c548dd75 100644 --- a/gitnexus/src/core/ingestion/workers/worker-pool.ts +++ b/gitnexus/src/core/ingestion/workers/worker-pool.ts @@ -50,29 +50,66 @@ export const createWorkerPool = (workerUrl: URL, poolSize?: number): WorkerPool const promises = chunks.map((chunk, i) => { const worker = workers[i]; return new Promise((resolve, reject) => { + let settled = false; + const cleanup = () => { + clearTimeout(timer); + worker.removeListener('message', handler); + worker.removeListener('error', errorHandler); + worker.removeListener('exit', exitHandler); + }; + + const timer = setTimeout(() => { + if (!settled) { + settled = true; + cleanup(); + reject(new Error(`Worker ${i} timed out after 5 minutes (chunk: ${chunk.length} items). Worker may have crashed or is processing too much data.`)); + } + }, 5 * 60 * 1000); + const handler = (msg: any) => { + if (settled) return; if (msg && msg.type === 'progress') { - // Intermediate progress from worker workerProgress[i] = msg.filesProcessed; if (onProgress) { const total = workerProgress.reduce((a, b) => a + b, 0); onProgress(total); } + } else if (msg && msg.type === 'error') { + // Error reported by worker via postMessage + settled = true; + cleanup(); + reject(new Error(`Worker ${i} error: ${msg.error}`)); } else if (msg && msg.type === 'result') { - // Final result - worker.removeListener('message', handler); + settled = true; + cleanup(); resolve(msg.data); } else { - // Legacy: treat any non-typed message as result (backward compat) - worker.removeListener('message', handler); + // Legacy: treat any non-typed message as result + settled = true; + cleanup(); resolve(msg); } }; + + const errorHandler = (err: any) => { + if (!settled) { + settled = true; + cleanup(); + reject(err); + } + }; + + const exitHandler = (code: number) => { + if (!settled) { + settled = true; + cleanup(); + reject(new Error(`Worker ${i} exited unexpectedly with code ${code}. This usually indicates an out-of-memory crash or native addon failure.`)); + } + }; + worker.on('message', handler); - worker.once('error', (err) => { - worker.removeListener('message', handler); - reject(err); - }); + worker.once('error', errorHandler); + worker.once('exit', exitHandler); worker.postMessage(chunk); }); });