From 493827222df050e9c51b16fb9722aada680853e5 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 17 May 2026 16:28:07 +0100 Subject: [PATCH] fix(ingestion): Raise `analyze` auto-heap to 16GB and tighten cross-platform OOM guidance for UE5-scale repositories (#1652) --- gitnexus/src/cli/analyze.ts | 73 ++++++- .../integration/analyze-heap-oom-e2e.test.ts | 74 +++++++ .../test/unit/analyze-heap-respawn.test.ts | 200 ++++++++++++++++++ 3 files changed, 344 insertions(+), 3 deletions(-) create mode 100644 gitnexus/test/integration/analyze-heap-oom-e2e.test.ts create mode 100644 gitnexus/test/unit/analyze-heap-respawn.test.ts diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index ec3636afc..e24b8c894 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -68,13 +68,69 @@ const installFatalHandlers = (): void => { }); }; -const HEAP_MB = 8192; -const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`; +const HEAP_MB = 16384; +const TEST_RESPAWN_HEAP_MB = Number(process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB); +const RESPAWN_HEAP_MB = + Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0 + ? Math.floor(TEST_RESPAWN_HEAP_MB) + : HEAP_MB; +const HEAP_FLAG = `--max-old-space-size=${RESPAWN_HEAP_MB}`; /** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */ const STACK_KB = 4096; const STACK_FLAG = `--stack-size=${STACK_KB}`; -/** Re-exec the process with an 8GB heap and larger stack if we're currently below that. */ +/** + * Heuristic for "child re-exec likely died from V8 OOM". + * + * Platform-independent detection is best-effort: V8/Node usually emit + * stable heap-exhaustion phrases in stderr/message across Linux/macOS/Windows + * (for example "JavaScript heap out of memory" or "Reached heap limit"), + * while some environments only expose status/signal (e.g. 134/SIGABRT). + * We combine both text signatures and process-exit signatures. + */ +const childProcessLikelyOom = (err: unknown): boolean => { + if (!err || typeof err !== 'object') return false; + const e = err as { + status?: unknown; + signal?: unknown; + stderr?: unknown; + stdout?: unknown; + message?: unknown; + }; + + const hasHeapOomSignature = (v: unknown): boolean => { + const text = ( + Buffer.isBuffer(v) ? v.toString('utf8') : typeof v === 'string' ? v : '' + ).toLowerCase(); + if (!text) return false; + return ( + text.includes('javascript heap out of memory') || + text.includes('reached heap limit') || + text.includes('allocation failed - javascript heap out of memory') || + text.includes('fatalprocessoutofmemory') + ); + }; + + const fields = [e.message, e.stderr, e.stdout]; + if (fields.some((v) => hasHeapOomSignature(v))) return true; + + const hasAnyChildOutput = [e.stderr, e.stdout].some( + (v) => (Buffer.isBuffer(v) && v.length > 0) || (typeof v === 'string' && v.length > 0), + ); + if (hasAnyChildOutput) return false; + + return e.status === 134 || e.signal === 'SIGABRT'; +}; + +const forceHeapOOMForTestIfEnabled = (): void => { + if (process.env.GITNEXUS_TEST_FORCE_HEAP_OOM !== '1') return; + // Allocate JS strings (not Buffers) so pressure lands on V8 heap itself. + // Buffers can allocate off-heap, which makes OOM triggering less reliable. + const chunks: string[] = []; + for (;;) chunks.push('x'.repeat(1024 * 1024)); +}; + +/** Re-exec the process with a 16GB heap and larger stack if we're currently below that. */ function ensureHeap(): boolean { const nodeOpts = process.env.NODE_OPTIONS || ''; if (nodeOpts.includes('--max-old-space-size')) return false; @@ -93,6 +149,16 @@ function ensureHeap(): boolean { env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() }, }); } catch (e: any) { + if (childProcessLikelyOom(e)) { + cliError( + ` Analysis likely ran out of memory.\n` + + ` Retry with a larger heap if your machine allows it:\n` + + ` NODE_OPTIONS="--max-old-space-size=24576" gitnexus analyze [your-args]\n` + + ` (Windows: set NODE_OPTIONS=--max-old-space-size=24576 && gitnexus analyze [your-args])\n` + + ` If this persists, it may be a native crash unrelated to heap size.\n`, + { recoveryHint: 'heap-oom-respawn' }, + ); + } process.exitCode = e.status ?? 1; } return true; @@ -185,6 +251,7 @@ export const shouldGenerateCommunitySkillFiles = ( export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOptions) => { if (ensureHeap()) return; + forceHeapOOMForTestIfEnabled(); // Install fatal handlers immediately after re-exec resolution so any // async error that escapes the try/catch below (#1169) surfaces with diff --git a/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts b/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts new file mode 100644 index 000000000..e576baa7a --- /dev/null +++ b/gitnexus/test/integration/analyze-heap-oom-e2e.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(testDir, '../..'); +const distCli = path.join(repoRoot, 'dist', 'cli', 'index.js'); +const fixtureSource = path.resolve(testDir, '..', 'fixtures', 'mini-repo'); + +const runAnalyzeWithForcedOom = (cwd: string, gitnexusHome: string) => + spawnSync(process.execPath, [distCli, 'analyze'], { + cwd, + encoding: 'utf8', + timeout: process.env.CI ? 40_000 : 20_000, + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + GITNEXUS_HOME: gitnexusHome, + NODE_OPTIONS: '', + GITNEXUS_TEST_RESPAWN_HEAP_MB: '32', + GITNEXUS_TEST_FORCE_HEAP_OOM: '1', + CI: '1', + }, + }); + +describe('analyze OOM guidance (real child-process OOM)', () => { + it('prints OOM guidance with Unix and Windows commands when respawned child truly OOMs', () => { + if (!fs.existsSync(distCli)) { + throw new Error( + 'dist/cli/index.js missing — run `npm run build` first (or use `npm run test:integration`, which builds via pretest:integration).', + ); + } + + const oomTestRepoParent = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-oom-e2e-repo-')); + const oomTestGitnexusHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-oom-e2e-home-')); + const repoPath = path.join(oomTestRepoParent, 'mini-repo'); + + fs.cpSync(fixtureSource, repoPath, { recursive: true }); + spawnSync('git', ['init'], { cwd: repoPath, stdio: 'pipe' }); + spawnSync('git', ['add', '-A'], { cwd: repoPath, stdio: 'pipe' }); + spawnSync('git', ['commit', '-m', 'initial commit'], { + cwd: repoPath, + stdio: 'pipe', + env: { + ...process.env, + GIT_AUTHOR_NAME: 'test', + GIT_AUTHOR_EMAIL: 'test@test', + GIT_COMMITTER_NAME: 'test', + GIT_COMMITTER_EMAIL: 'test@test', + }, + }); + + try { + const result = runAnalyzeWithForcedOom(repoPath, oomTestGitnexusHome); + const combinedOutput = `${result.stderr}\n${result.stdout}`; + + expect(result.status).not.toBeNull(); + expect(result.status).not.toBe(0); + expect(combinedOutput).toContain('Analysis likely ran out of memory.'); + expect(combinedOutput).toContain( + 'NODE_OPTIONS="--max-old-space-size=24576" gitnexus analyze [your-args]', + ); + expect(combinedOutput).toContain( + '(Windows: set NODE_OPTIONS=--max-old-space-size=24576 && gitnexus analyze [your-args])', + ); + } finally { + fs.rmSync(oomTestRepoParent, { recursive: true, force: true }); + fs.rmSync(oomTestGitnexusHome, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/gitnexus/test/unit/analyze-heap-respawn.test.ts b/gitnexus/test/unit/analyze-heap-respawn.test.ts new file mode 100644 index 000000000..2f094ddbf --- /dev/null +++ b/gitnexus/test/unit/analyze-heap-respawn.test.ts @@ -0,0 +1,200 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const execFileSyncMock = vi.fn(); +const getHeapStatisticsMock = vi.fn(); + +vi.mock('child_process', async () => { + const actual = await vi.importActual('child_process'); + return { ...actual, execFileSync: execFileSyncMock }; +}); + +vi.mock('v8', () => ({ + default: { + getHeapStatistics: getHeapStatisticsMock, + }, +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + closeLbug: vi.fn(async () => undefined), +})); + +describe('analyzeCommand heap respawn', () => { + let initialNodeOptions: string | undefined; + + beforeEach(() => { + initialNodeOptions = process.env.NODE_OPTIONS; + vi.resetModules(); + execFileSyncMock.mockReset(); + getHeapStatisticsMock.mockReset(); + process.exitCode = undefined; + }); + + afterEach(() => { + if (initialNodeOptions === undefined) delete process.env.NODE_OPTIONS; + else process.env.NODE_OPTIONS = initialNodeOptions; + }); + + it('re-execs analyze with 16GB heap when no max-old-space-size is present', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(execFileSyncMock).toHaveBeenCalledTimes(1); + const [, args, opts] = execFileSyncMock.mock.calls[0]; + expect(args).toContain('--max-old-space-size=16384'); + expect(opts.env.NODE_OPTIONS).toContain('--max-old-space-size=16384'); + }); + + it('does not re-exec when NODE_OPTIONS already defines max-old-space-size', async () => { + process.env.NODE_OPTIONS = '--max-old-space-size=32768'; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand('/__gitnexus_nonexistent__', {}); + + expect(execFileSyncMock).not.toHaveBeenCalled(); + }); + + it('prints heap guidance when respawned analyze exits with likely OOM', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + execFileSyncMock.mockImplementationOnce(() => { + const err = new Error('child failed') as Error & { status?: number; signal?: string }; + err.status = undefined; + err.signal = 'SIGABRT'; + throw err; + }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + // Signal-only child failures do not carry a numeric status, so the CLI + // falls back to exit code 1. + expect(process.exitCode).toBe(1); + const oomGuidance = cap + .records() + .find((r) => r.msg.includes('Analysis likely ran out of memory.')); + expect(oomGuidance).toBeDefined(); + const msg = oomGuidance?.msg ?? ''; + expect(msg).toContain('NODE_OPTIONS="--max-old-space-size=24576"'); + expect(msg).toContain('[your-args]'); + expect(msg).toContain('native crash unrelated to heap size'); + cap.restore(); + }); + + it('prints heap guidance when child stderr contains heap OOM signature', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + execFileSyncMock.mockImplementationOnce(() => { + const err = new Error('Command failed') as Error & { + status?: number; + signal?: string; + stderr?: Buffer; + }; + err.status = 1; + err.signal = undefined; + err.stderr = Buffer.from( + 'FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory', + ); + throw err; + }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(1); + expect(cap.records().some((r) => r.msg.includes('Analysis likely ran out of memory.'))).toBe( + true, + ); + cap.restore(); + }); + + it('prints heap guidance when child stdout contains heap OOM signature', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + execFileSyncMock.mockImplementationOnce(() => { + const err = new Error('Command failed') as Error & { + status?: number; + signal?: string; + stdout?: string; + }; + err.status = 1; + err.signal = undefined; + err.stdout = 'FATAL ERROR: JavaScript heap out of memory'; + throw err; + }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(1); + expect(cap.records().some((r) => r.msg.includes('Analysis likely ran out of memory.'))).toBe( + true, + ); + cap.restore(); + }); + + it('prints heap guidance when child exits 134 without output', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + execFileSyncMock.mockImplementationOnce(() => { + const err = new Error('Command failed') as Error & { + status?: number; + signal?: string; + stderr?: string; + stdout?: string; + }; + err.status = 134; + err.signal = undefined; + err.stderr = ''; + err.stdout = ''; + throw err; + }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(134); + expect(cap.records().some((r) => r.msg.includes('Analysis likely ran out of memory.'))).toBe( + true, + ); + cap.restore(); + }); + + it('does not print heap guidance for non-OOM child failures with output', async () => { + delete process.env.NODE_OPTIONS; + getHeapStatisticsMock.mockReturnValue({ heap_size_limit: 512 * 1024 * 1024 }); + execFileSyncMock.mockImplementationOnce(() => { + const err = new Error('Command failed') as Error & { + status?: number; + signal?: string; + stderr?: Buffer; + }; + err.status = 2; + err.signal = undefined; + err.stderr = Buffer.from('parser failed: invalid token'); + throw err; + }); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(2); + expect(cap.records().some((r) => r.msg.includes('Analysis likely ran out of memory.'))).toBe( + false, + ); + cap.restore(); + }); +});