diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 46cedc434..7d76c8814 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -20,6 +20,7 @@ import { } from '../storage/repo-manager.js'; import { getGitRoot, hasGitDir } from '../storage/git.js'; import { runFullAnalysis } from '../core/run-analyze.js'; +import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; import fs from 'fs/promises'; const HEAP_MB = 8192; @@ -78,6 +79,12 @@ export interface AnalyzeOptions { * `allowDuplicateName` option end-to-end. */ allowDuplicateName?: boolean; + /** + * Override the walker's large-file skip threshold (#991). Value in KB; + * clamped downstream to the tree-sitter 32 MB ceiling. Sets + * `GITNEXUS_MAX_FILE_SIZE` for the rest of the pipeline. + */ + maxFileSize?: string; } export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOptions) => { @@ -87,6 +94,10 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption process.env.GITNEXUS_VERBOSE = '1'; } + if (options?.maxFileSize) { + process.env.GITNEXUS_MAX_FILE_SIZE = options.maxFileSize; + } + console.log('\n GitNexus Analyzer\n'); let repoPath: string; @@ -132,6 +143,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption ); } + const maxFileSizeBanner = getMaxFileSizeBannerMessage(); + if (maxFileSizeBanner) { + console.log(`${maxFileSizeBanner}\n`); + } + // ── CLI progress bar setup ───────────────────────────────────────── const bar = new cliProgress.SingleBar( { diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index fe19ba753..ad03f05c8 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -39,9 +39,15 @@ program 'Leaves `-r ` ambiguous for the two paths; use -r to disambiguate.', ) .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)') + .option( + '--max-file-size ', + 'Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).', + ) .addHelpText( 'after', - '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)', + '\nEnvironment variables:\n' + + ' GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n' + + ' GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.', ) .action(createLazyAction(() => import('./analyze.js'), 'analyzeCommand')); diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 575efc0f3..71a4046f2 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -1,4 +1,5 @@ import { isVerboseIngestionEnabled } from './utils/verbose.js'; +import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js'; import fs from 'fs/promises'; import path from 'path'; import { glob } from 'glob'; @@ -22,9 +23,6 @@ export interface FilePath { const READ_CONCURRENCY = 32; -/** Skip files larger than 512KB — they're usually generated/vendored and crash tree-sitter */ -const MAX_FILE_SIZE = 512 * 1024; - /** * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded. * Memory: ~10MB for 100K files vs ~1GB+ with content. @@ -34,6 +32,7 @@ export const walkRepositoryPaths = async ( onProgress?: (current: number, total: number, filePath: string) => void, ): Promise => { const ignoreFilter = await createIgnoreFilter(repoPath); + const maxFileSizeBytes = getMaxFileSizeBytes(); const filtered = await glob('**/*', { cwd: repoPath, @@ -52,7 +51,7 @@ export const walkRepositoryPaths = async ( batch.map(async (relativePath) => { const fullPath = path.join(repoPath, relativePath); const stat = await fs.stat(fullPath); - if (stat.size > MAX_FILE_SIZE) { + if (stat.size > maxFileSizeBytes) { skippedLarge++; skippedLargePaths.push(relativePath.replace(/\\/g, '/')); return null; @@ -73,9 +72,9 @@ export const walkRepositoryPaths = async ( } if (skippedLarge > 0) { - console.warn( - ` Skipped ${skippedLarge} large files (>${MAX_FILE_SIZE / 1024}KB, likely generated/vendored)`, - ); + const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES; + const suffix = isDefault ? ', likely generated/vendored' : ''; + console.warn(` Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`); if (isVerboseIngestionEnabled()) { for (const p of skippedLargePaths) { console.warn(` - ${p}`); diff --git a/gitnexus/src/core/ingestion/utils/max-file-size.ts b/gitnexus/src/core/ingestion/utils/max-file-size.ts new file mode 100644 index 000000000..0c418bfd4 --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/max-file-size.ts @@ -0,0 +1,64 @@ +import { TREE_SITTER_MAX_BUFFER } from '../constants.js'; + +/** Default threshold (512 KB). Files larger than this are skipped by the walker. */ +export const DEFAULT_MAX_FILE_SIZE_BYTES = 512 * 1024; + +/** Hard upper bound — tree-sitter refuses buffers above this regardless. */ +export const MAX_FILE_SIZE_UPPER_BOUND_BYTES = TREE_SITTER_MAX_BUFFER; + +const warned = new Set(); + +const warnOnce = (key: string, message: string): void => { + if (warned.has(key)) return; + warned.add(key); + console.warn(message); +}; + +/** + * Resolve the effective file-size skip threshold (bytes) for the walker. + * Reads `GITNEXUS_MAX_FILE_SIZE` (KB). Invalid values fall back to the default + * and emit a one-time warning. Values above the tree-sitter ceiling are clamped. + */ +export const getMaxFileSizeBytes = (): number => { + const raw = process.env.GITNEXUS_MAX_FILE_SIZE; + if (!raw) return DEFAULT_MAX_FILE_SIZE_BYTES; + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { + warnOnce( + `invalid:${raw}`, + ` GITNEXUS_MAX_FILE_SIZE must be a positive integer (KB), got "${raw}" — using default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`, + ); + return DEFAULT_MAX_FILE_SIZE_BYTES; + } + + const bytes = parsed * 1024; + if (bytes > MAX_FILE_SIZE_UPPER_BOUND_BYTES) { + warnOnce( + `clamp:${raw}`, + ` GITNEXUS_MAX_FILE_SIZE=${parsed}KB exceeds tree-sitter ceiling (${MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024}KB) — clamping`, + ); + return MAX_FILE_SIZE_UPPER_BOUND_BYTES; + } + return bytes; +}; + +/** + * Build the CLI banner message announcing an active file-size override. + * Returns `null` when the effective threshold equals the default — the caller + * should print nothing in that case. The returned message reflects the + * *effective* post-clamp threshold, not the raw env value, so operators reading + * startup output see the actual configuration the walker will use. + */ +export const getMaxFileSizeBannerMessage = (): string | null => { + const effectiveBytes = getMaxFileSizeBytes(); + if (effectiveBytes === DEFAULT_MAX_FILE_SIZE_BYTES) return null; + const effectiveKb = effectiveBytes / 1024; + const defaultKb = DEFAULT_MAX_FILE_SIZE_BYTES / 1024; + return ` GITNEXUS_MAX_FILE_SIZE: effective threshold ${effectiveKb}KB (default ${defaultKb}KB)`; +}; + +/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */ +export const _resetMaxFileSizeWarnings = (): void => { + warned.clear(); +}; diff --git a/gitnexus/test/integration/filesystem-walker.test.ts b/gitnexus/test/integration/filesystem-walker.test.ts index 15090b688..accb24e37 100644 --- a/gitnexus/test/integration/filesystem-walker.test.ts +++ b/gitnexus/test/integration/filesystem-walker.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; import fs from 'fs/promises'; import path from 'path'; import os from 'os'; @@ -6,6 +6,7 @@ import { walkRepositoryPaths, readFileContents, } from '../../src/core/ingestion/filesystem-walker.js'; +import { _resetMaxFileSizeWarnings } from '../../src/core/ingestion/utils/max-file-size.js'; describe('filesystem-walker', () => { let tmpDir: string; @@ -321,4 +322,80 @@ describe('filesystem-walker', () => { expect(contents.size).toBeLessThanOrEqual(1); }); }); + + describe('large file skip threshold (#991)', () => { + let sizeDir: string; + const BIG_FILE = 'src/big.ts'; + const BIG_FILE_BYTES = 600 * 1024; + const ORIGINAL_ENV = process.env.GITNEXUS_MAX_FILE_SIZE; + let warnSpy: ReturnType; + + beforeAll(async () => { + sizeDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-size-test-')); + await fs.mkdir(path.join(sizeDir, 'src'), { recursive: true }); + await fs.writeFile(path.join(sizeDir, 'src', 'small.ts'), 'export const x = 1;'); + await fs.writeFile(path.join(sizeDir, BIG_FILE), 'x'.repeat(BIG_FILE_BYTES)); + }); + + afterAll(async () => { + await fs.rm(sizeDir, { recursive: true, force: true }); + }); + + beforeEach(() => { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + _resetMaxFileSizeWarnings(); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + afterEach(() => { + if (ORIGINAL_ENV === undefined) { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + } else { + process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL_ENV; + } + warnSpy.mockRestore(); + }); + + it('skips a 600KB file by default', async () => { + const files = await walkRepositoryPaths(sizeDir); + const paths = files.map((f) => f.path.replace(/\\/g, '/')); + expect(paths).toContain('src/small.ts'); + expect(paths).not.toContain(BIG_FILE); + }); + + it('includes the 600KB file when GITNEXUS_MAX_FILE_SIZE=1024', async () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '1024'; + const files = await walkRepositoryPaths(sizeDir); + const paths = files.map((f) => f.path.replace(/\\/g, '/')); + expect(paths).toContain(BIG_FILE); + }); + + it('falls back to default and warns once on invalid GITNEXUS_MAX_FILE_SIZE', async () => { + process.env.GITNEXUS_MAX_FILE_SIZE = 'abc'; + const files = await walkRepositoryPaths(sizeDir); + const paths = files.map((f) => f.path.replace(/\\/g, '/')); + expect(paths).not.toContain(BIG_FILE); + const invalidWarnings = warnSpy.mock.calls.filter((c) => + String(c[0]).includes('must be a positive integer'), + ); + expect(invalidWarnings).toHaveLength(1); + }); + + it('omits the "generated/vendored" suffix when threshold is overridden', async () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; + await walkRepositoryPaths(sizeDir); + const skipWarnings = warnSpy.mock.calls.filter((c) => String(c[0]).includes('Skipped ')); + expect(skipWarnings.length).toBeGreaterThan(0); + for (const call of skipWarnings) { + expect(String(call[0])).not.toContain('generated/vendored'); + } + }); + + it('keeps the "generated/vendored" suffix under the default threshold', async () => { + await walkRepositoryPaths(sizeDir); + const skipWarnings = warnSpy.mock.calls.filter((c) => String(c[0]).includes('Skipped ')); + expect(skipWarnings.length).toBeGreaterThan(0); + expect(String(skipWarnings[0][0])).toContain('generated/vendored'); + }); + }); }); diff --git a/gitnexus/test/unit/max-file-size.test.ts b/gitnexus/test/unit/max-file-size.test.ts new file mode 100644 index 000000000..074e148e7 --- /dev/null +++ b/gitnexus/test/unit/max-file-size.test.ts @@ -0,0 +1,150 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { + DEFAULT_MAX_FILE_SIZE_BYTES, + MAX_FILE_SIZE_UPPER_BOUND_BYTES, + getMaxFileSizeBytes, + getMaxFileSizeBannerMessage, + _resetMaxFileSizeWarnings, +} from '../../src/core/ingestion/utils/max-file-size.js'; + +describe('getMaxFileSizeBytes', () => { + const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE; + let warnSpy: ReturnType; + + beforeEach(() => { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + _resetMaxFileSizeWarnings(); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + } else { + process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL; + } + warnSpy.mockRestore(); + }); + + it('returns the default when the env var is unset', () => { + expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('parses a positive integer value as KB', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '1024'; + expect(getMaxFileSizeBytes()).toBe(1024 * 1024); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it('clamps values above the tree-sitter ceiling', () => { + // One KB above the 32 MB ceiling. + const aboveCeilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024 + 1; + process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb); + expect(getMaxFileSizeBytes()).toBe(MAX_FILE_SIZE_UPPER_BOUND_BYTES); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(warnSpy.mock.calls[0][0]).toContain('clamping'); + }); + + it.each(['abc', '0', '-512', '1.5', 'NaN', ''])( + 'falls back to the default and warns on invalid value %s', + (raw) => { + if (raw === '') { + // Empty string is treated as unset by the util (raw falsy check). + process.env.GITNEXUS_MAX_FILE_SIZE = raw; + expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES); + expect(warnSpy).not.toHaveBeenCalled(); + return; + } + process.env.GITNEXUS_MAX_FILE_SIZE = raw; + expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(warnSpy.mock.calls[0][0]).toContain('must be a positive integer'); + }, + ); + + it('deduplicates warnings for the same invalid value', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = 'abc'; + getMaxFileSizeBytes(); + getMaxFileSizeBytes(); + getMaxFileSizeBytes(); + expect(warnSpy).toHaveBeenCalledTimes(1); + }); + + it('warns separately for distinct invalid values', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = 'abc'; + getMaxFileSizeBytes(); + process.env.GITNEXUS_MAX_FILE_SIZE = 'xyz'; + getMaxFileSizeBytes(); + expect(warnSpy).toHaveBeenCalledTimes(2); + }); + + it('_resetMaxFileSizeWarnings re-enables warnings after reset', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = 'abc'; + getMaxFileSizeBytes(); + expect(warnSpy).toHaveBeenCalledTimes(1); + + getMaxFileSizeBytes(); + expect(warnSpy).toHaveBeenCalledTimes(1); + + _resetMaxFileSizeWarnings(); + getMaxFileSizeBytes(); + expect(warnSpy).toHaveBeenCalledTimes(2); + }); + + it('DEFAULT_MAX_FILE_SIZE_BYTES is 512 KB', () => { + expect(DEFAULT_MAX_FILE_SIZE_BYTES).toBe(512 * 1024); + }); +}); + +describe('getMaxFileSizeBannerMessage', () => { + const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE; + let warnSpy: ReturnType; + + beforeEach(() => { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + _resetMaxFileSizeWarnings(); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_MAX_FILE_SIZE; + } else { + process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL; + } + warnSpy.mockRestore(); + }); + + it('returns null when the env var is unset (default threshold)', () => { + expect(getMaxFileSizeBannerMessage()).toBeNull(); + }); + + it('returns null when the env var equals the default (in KB)', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = String(DEFAULT_MAX_FILE_SIZE_BYTES / 1024); + expect(getMaxFileSizeBannerMessage()).toBeNull(); + }); + + it('returns null when an invalid value falls back to the default', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = 'abc'; + expect(getMaxFileSizeBannerMessage()).toBeNull(); + }); + + it('reports the raised effective threshold in KB', () => { + process.env.GITNEXUS_MAX_FILE_SIZE = '1024'; + const banner = getMaxFileSizeBannerMessage(); + expect(banner).not.toBeNull(); + expect(banner).toContain('effective threshold 1024KB'); + expect(banner).toContain(`default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`); + }); + + it('reports the clamped (post-ceiling) threshold, not the raw input', () => { + const ceilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024; + const aboveCeilingKb = ceilingKb + 1024; + process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb); + const banner = getMaxFileSizeBannerMessage(); + expect(banner).not.toBeNull(); + expect(banner).toContain(`effective threshold ${ceilingKb}KB`); + expect(banner).not.toContain(`${aboveCeilingKb}KB`); + }); +});