feat(ingestion): make large-file skip threshold configurable (#1044)

* feat(ingestion): make large-file skip threshold configurable

The walker previously hardcoded a 512KB skip threshold, which silently dropped legitimate large source files (e.g. ~900KB hand-written Java service classes) during analysis with no way to override short of editing source.

Allow overrides via the GITNEXUS_MAX_FILE_SIZE env var (KB) — consistent with the existing GITNEXUS_NO_GITIGNORE / GITNEXUS_VERBOSE patterns — and a matching --max-file-size <kb> flag on gitnexus analyze.

- New utility getMaxFileSizeBytes() in core/ingestion/utils/max-file-size.ts parses the env var, falls back to the 512KB default for missing/invalid values, and clamps against TREE_SITTER_MAX_BUFFER (32MB) to keep the downstream parser safe.
- filesystem-walker.ts now resolves the threshold per call and drops the 'likely generated/vendored' editorial when the user has explicitly raised the limit.
- analyze CLI wires --max-file-size to the env var and echoes a one-line notice when the threshold is overridden, mirroring how --no-gitignore is handled.
- index.ts documents the new flag and env var under the analyze help text.
- Warnings for invalid or out-of-range values are emitted exactly once per distinct value to avoid log spam.

Tests:
- New test/unit/max-file-size.test.ts covers defaults, KB parsing, clamp-at-ceiling, invalid-input fallback + warn-once, and distinct-value warnings.
- test/integration/filesystem-walker.test.ts gains a 'large file skip threshold (#991)' block: 600KB fixture skipped by default, included under GITNEXUS_MAX_FILE_SIZE=1024, invalid values fall back and warn once, and the 'generated/vendored' suffix is only emitted under the default threshold.

Closes #991

* fix(cli): show effective clamped max-file-size in banner

Addresses the PR #1044 review finding: the startup banner printed the raw GITNEXUS_MAX_FILE_SIZE value rather than the clamped effective threshold, producing misleading telemetry when the value exceeded the 32 MB tree-sitter ceiling.

The banner is also suppressed when the effective threshold equals the default, removing log noise when operators explicitly set the value to the current default.

Extracted the logic into a new getMaxFileSizeBannerMessage() helper and pinned the behavior with unit tests covering default, raised override, invalid fallback, and above-ceiling clamp cases.

gitnexus/src/cli/analyze.ts

@@ -20,6 +20,7 @@ import {
 } from '../storage/repo-manager.js';
 import { getGitRoot, hasGitDir } from '../storage/git.js';
 import { runFullAnalysis } from '../core/run-analyze.js';
+import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
 import fs from 'fs/promises';
 
 const HEAP_MB = 8192;
@@ -78,6 +79,12 @@ export interface AnalyzeOptions {
    * `allowDuplicateName` option end-to-end.
    */
   allowDuplicateName?: boolean;
+  /**
+   * Override the walker's large-file skip threshold (#991). Value in KB;
+   * clamped downstream to the tree-sitter 32 MB ceiling. Sets
+   * `GITNEXUS_MAX_FILE_SIZE` for the rest of the pipeline.
+   */
+  maxFileSize?: string;
 }
 
 export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOptions) => {
@@ -87,6 +94,10 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
     process.env.GITNEXUS_VERBOSE = '1';
   }
 
+  if (options?.maxFileSize) {
+    process.env.GITNEXUS_MAX_FILE_SIZE = options.maxFileSize;
+  }
+
   console.log('\n  GitNexus Analyzer\n');
 
   let repoPath: string;
@@ -132,6 +143,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
     );
   }
 
+  const maxFileSizeBanner = getMaxFileSizeBannerMessage();
+  if (maxFileSizeBanner) {
+    console.log(`${maxFileSizeBanner}\n`);
+  }
+
   // ── CLI progress bar setup ─────────────────────────────────────────
   const bar = new cliProgress.SingleBar(
     {

gitnexus/src/cli/index.ts

@@ -39,9 +39,15 @@ program
       'Leaves `-r <name>` ambiguous for the two paths; use -r <path> to disambiguate.',
   )
   .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)')
+  .option(
+    '--max-file-size <kb>',
+    'Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).',
+  )
   .addHelpText(
     'after',
-    '\nEnvironment variables:\n  GITNEXUS_NO_GITIGNORE=1  Skip .gitignore parsing (still reads .gitnexusignore)',
+    '\nEnvironment variables:\n' +
+      '  GITNEXUS_NO_GITIGNORE=1   Skip .gitignore parsing (still reads .gitnexusignore)\n' +
+      '  GITNEXUS_MAX_FILE_SIZE=N  Override large-file skip threshold (KB). Default 512, max 32768.',
   )
   .action(createLazyAction(() => import('./analyze.js'), 'analyzeCommand'));
 

gitnexus/src/core/ingestion/filesystem-walker.ts

@@ -1,4 +1,5 @@
 import { isVerboseIngestionEnabled } from './utils/verbose.js';
+import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js';
 import fs from 'fs/promises';
 import path from 'path';
 import { glob } from 'glob';
@@ -22,9 +23,6 @@ export interface FilePath {
 
 const READ_CONCURRENCY = 32;
 
-/** Skip files larger than 512KB — they're usually generated/vendored and crash tree-sitter */
-const MAX_FILE_SIZE = 512 * 1024;
-
 /**
  * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded.
  * Memory: ~10MB for 100K files vs ~1GB+ with content.
@@ -34,6 +32,7 @@ export const walkRepositoryPaths = async (
   onProgress?: (current: number, total: number, filePath: string) => void,
 ): Promise<ScannedFile[]> => {
   const ignoreFilter = await createIgnoreFilter(repoPath);
+  const maxFileSizeBytes = getMaxFileSizeBytes();
 
   const filtered = await glob('**/*', {
     cwd: repoPath,
@@ -52,7 +51,7 @@ export const walkRepositoryPaths = async (
       batch.map(async (relativePath) => {
         const fullPath = path.join(repoPath, relativePath);
         const stat = await fs.stat(fullPath);
-        if (stat.size > MAX_FILE_SIZE) {
+        if (stat.size > maxFileSizeBytes) {
           skippedLarge++;
           skippedLargePaths.push(relativePath.replace(/\\/g, '/'));
           return null;
@@ -73,9 +72,9 @@ export const walkRepositoryPaths = async (
   }
 
   if (skippedLarge > 0) {
-    console.warn(
-      `  Skipped ${skippedLarge} large files (>${MAX_FILE_SIZE / 1024}KB, likely generated/vendored)`,
-    );
+    const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES;
+    const suffix = isDefault ? ', likely generated/vendored' : '';
+    console.warn(`  Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
     if (isVerboseIngestionEnabled()) {
       for (const p of skippedLargePaths) {
         console.warn(`  - ${p}`);

gitnexus/src/core/ingestion/utils/max-file-size.ts

@@ -0,0 +1,64 @@
+import { TREE_SITTER_MAX_BUFFER } from '../constants.js';
+
+/** Default threshold (512 KB). Files larger than this are skipped by the walker. */
+export const DEFAULT_MAX_FILE_SIZE_BYTES = 512 * 1024;
+
+/** Hard upper bound — tree-sitter refuses buffers above this regardless. */
+export const MAX_FILE_SIZE_UPPER_BOUND_BYTES = TREE_SITTER_MAX_BUFFER;
+
+const warned = new Set<string>();
+
+const warnOnce = (key: string, message: string): void => {
+  if (warned.has(key)) return;
+  warned.add(key);
+  console.warn(message);
+};
+
+/**
+ * Resolve the effective file-size skip threshold (bytes) for the walker.
+ * Reads `GITNEXUS_MAX_FILE_SIZE` (KB). Invalid values fall back to the default
+ * and emit a one-time warning. Values above the tree-sitter ceiling are clamped.
+ */
+export const getMaxFileSizeBytes = (): number => {
+  const raw = process.env.GITNEXUS_MAX_FILE_SIZE;
+  if (!raw) return DEFAULT_MAX_FILE_SIZE_BYTES;
+
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
+    warnOnce(
+      `invalid:${raw}`,
+      `  GITNEXUS_MAX_FILE_SIZE must be a positive integer (KB), got "${raw}" — using default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`,
+    );
+    return DEFAULT_MAX_FILE_SIZE_BYTES;
+  }
+
+  const bytes = parsed * 1024;
+  if (bytes > MAX_FILE_SIZE_UPPER_BOUND_BYTES) {
+    warnOnce(
+      `clamp:${raw}`,
+      `  GITNEXUS_MAX_FILE_SIZE=${parsed}KB exceeds tree-sitter ceiling (${MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024}KB) — clamping`,
+    );
+    return MAX_FILE_SIZE_UPPER_BOUND_BYTES;
+  }
+  return bytes;
+};
+
+/**
+ * Build the CLI banner message announcing an active file-size override.
+ * Returns `null` when the effective threshold equals the default — the caller
+ * should print nothing in that case. The returned message reflects the
+ * *effective* post-clamp threshold, not the raw env value, so operators reading
+ * startup output see the actual configuration the walker will use.
+ */
+export const getMaxFileSizeBannerMessage = (): string | null => {
+  const effectiveBytes = getMaxFileSizeBytes();
+  if (effectiveBytes === DEFAULT_MAX_FILE_SIZE_BYTES) return null;
+  const effectiveKb = effectiveBytes / 1024;
+  const defaultKb = DEFAULT_MAX_FILE_SIZE_BYTES / 1024;
+  return `  GITNEXUS_MAX_FILE_SIZE: effective threshold ${effectiveKb}KB (default ${defaultKb}KB)`;
+};
+
+/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */
+export const _resetMaxFileSizeWarnings = (): void => {
+  warned.clear();
+};

gitnexus/test/integration/filesystem-walker.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
 import fs from 'fs/promises';
 import path from 'path';
 import os from 'os';
@@ -6,6 +6,7 @@ import {
   walkRepositoryPaths,
   readFileContents,
 } from '../../src/core/ingestion/filesystem-walker.js';
+import { _resetMaxFileSizeWarnings } from '../../src/core/ingestion/utils/max-file-size.js';
 
 describe('filesystem-walker', () => {
   let tmpDir: string;
@@ -321,4 +322,80 @@ describe('filesystem-walker', () => {
       expect(contents.size).toBeLessThanOrEqual(1);
     });
   });
+
+  describe('large file skip threshold (#991)', () => {
+    let sizeDir: string;
+    const BIG_FILE = 'src/big.ts';
+    const BIG_FILE_BYTES = 600 * 1024;
+    const ORIGINAL_ENV = process.env.GITNEXUS_MAX_FILE_SIZE;
+    let warnSpy: ReturnType<typeof vi.spyOn>;
+
+    beforeAll(async () => {
+      sizeDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-size-test-'));
+      await fs.mkdir(path.join(sizeDir, 'src'), { recursive: true });
+      await fs.writeFile(path.join(sizeDir, 'src', 'small.ts'), 'export const x = 1;');
+      await fs.writeFile(path.join(sizeDir, BIG_FILE), 'x'.repeat(BIG_FILE_BYTES));
+    });
+
+    afterAll(async () => {
+      await fs.rm(sizeDir, { recursive: true, force: true });
+    });
+
+    beforeEach(() => {
+      delete process.env.GITNEXUS_MAX_FILE_SIZE;
+      _resetMaxFileSizeWarnings();
+      warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+    });
+
+    afterEach(() => {
+      if (ORIGINAL_ENV === undefined) {
+        delete process.env.GITNEXUS_MAX_FILE_SIZE;
+      } else {
+        process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL_ENV;
+      }
+      warnSpy.mockRestore();
+    });
+
+    it('skips a 600KB file by default', async () => {
+      const files = await walkRepositoryPaths(sizeDir);
+      const paths = files.map((f) => f.path.replace(/\\/g, '/'));
+      expect(paths).toContain('src/small.ts');
+      expect(paths).not.toContain(BIG_FILE);
+    });
+
+    it('includes the 600KB file when GITNEXUS_MAX_FILE_SIZE=1024', async () => {
+      process.env.GITNEXUS_MAX_FILE_SIZE = '1024';
+      const files = await walkRepositoryPaths(sizeDir);
+      const paths = files.map((f) => f.path.replace(/\\/g, '/'));
+      expect(paths).toContain(BIG_FILE);
+    });
+
+    it('falls back to default and warns once on invalid GITNEXUS_MAX_FILE_SIZE', async () => {
+      process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
+      const files = await walkRepositoryPaths(sizeDir);
+      const paths = files.map((f) => f.path.replace(/\\/g, '/'));
+      expect(paths).not.toContain(BIG_FILE);
+      const invalidWarnings = warnSpy.mock.calls.filter((c) =>
+        String(c[0]).includes('must be a positive integer'),
+      );
+      expect(invalidWarnings).toHaveLength(1);
+    });
+
+    it('omits the "generated/vendored" suffix when threshold is overridden', async () => {
+      process.env.GITNEXUS_MAX_FILE_SIZE = '1';
+      await walkRepositoryPaths(sizeDir);
+      const skipWarnings = warnSpy.mock.calls.filter((c) => String(c[0]).includes('Skipped '));
+      expect(skipWarnings.length).toBeGreaterThan(0);
+      for (const call of skipWarnings) {
+        expect(String(call[0])).not.toContain('generated/vendored');
+      }
+    });
+
+    it('keeps the "generated/vendored" suffix under the default threshold', async () => {
+      await walkRepositoryPaths(sizeDir);
+      const skipWarnings = warnSpy.mock.calls.filter((c) => String(c[0]).includes('Skipped '));
+      expect(skipWarnings.length).toBeGreaterThan(0);
+      expect(String(skipWarnings[0][0])).toContain('generated/vendored');
+    });
+  });
 });

gitnexus/test/unit/max-file-size.test.ts

@@ -0,0 +1,150 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import {
+  DEFAULT_MAX_FILE_SIZE_BYTES,
+  MAX_FILE_SIZE_UPPER_BOUND_BYTES,
+  getMaxFileSizeBytes,
+  getMaxFileSizeBannerMessage,
+  _resetMaxFileSizeWarnings,
+} from '../../src/core/ingestion/utils/max-file-size.js';
+
+describe('getMaxFileSizeBytes', () => {
+  const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE;
+  let warnSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    delete process.env.GITNEXUS_MAX_FILE_SIZE;
+    _resetMaxFileSizeWarnings();
+    warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+  });
+
+  afterEach(() => {
+    if (ORIGINAL === undefined) {
+      delete process.env.GITNEXUS_MAX_FILE_SIZE;
+    } else {
+      process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL;
+    }
+    warnSpy.mockRestore();
+  });
+
+  it('returns the default when the env var is unset', () => {
+    expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('parses a positive integer value as KB', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = '1024';
+    expect(getMaxFileSizeBytes()).toBe(1024 * 1024);
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('clamps values above the tree-sitter ceiling', () => {
+    // One KB above the 32 MB ceiling.
+    const aboveCeilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024 + 1;
+    process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb);
+    expect(getMaxFileSizeBytes()).toBe(MAX_FILE_SIZE_UPPER_BOUND_BYTES);
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+    expect(warnSpy.mock.calls[0][0]).toContain('clamping');
+  });
+
+  it.each(['abc', '0', '-512', '1.5', 'NaN', ''])(
+    'falls back to the default and warns on invalid value %s',
+    (raw) => {
+      if (raw === '') {
+        // Empty string is treated as unset by the util (raw falsy check).
+        process.env.GITNEXUS_MAX_FILE_SIZE = raw;
+        expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
+        expect(warnSpy).not.toHaveBeenCalled();
+        return;
+      }
+      process.env.GITNEXUS_MAX_FILE_SIZE = raw;
+      expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
+      expect(warnSpy).toHaveBeenCalledTimes(1);
+      expect(warnSpy.mock.calls[0][0]).toContain('must be a positive integer');
+    },
+  );
+
+  it('deduplicates warnings for the same invalid value', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
+    getMaxFileSizeBytes();
+    getMaxFileSizeBytes();
+    getMaxFileSizeBytes();
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('warns separately for distinct invalid values', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
+    getMaxFileSizeBytes();
+    process.env.GITNEXUS_MAX_FILE_SIZE = 'xyz';
+    getMaxFileSizeBytes();
+    expect(warnSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it('_resetMaxFileSizeWarnings re-enables warnings after reset', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
+    getMaxFileSizeBytes();
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+
+    getMaxFileSizeBytes();
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+
+    _resetMaxFileSizeWarnings();
+    getMaxFileSizeBytes();
+    expect(warnSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it('DEFAULT_MAX_FILE_SIZE_BYTES is 512 KB', () => {
+    expect(DEFAULT_MAX_FILE_SIZE_BYTES).toBe(512 * 1024);
+  });
+});
+
+describe('getMaxFileSizeBannerMessage', () => {
+  const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE;
+  let warnSpy: ReturnType<typeof vi.spyOn>;
+
+  beforeEach(() => {
+    delete process.env.GITNEXUS_MAX_FILE_SIZE;
+    _resetMaxFileSizeWarnings();
+    warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+  });
+
+  afterEach(() => {
+    if (ORIGINAL === undefined) {
+      delete process.env.GITNEXUS_MAX_FILE_SIZE;
+    } else {
+      process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL;
+    }
+    warnSpy.mockRestore();
+  });
+
+  it('returns null when the env var is unset (default threshold)', () => {
+    expect(getMaxFileSizeBannerMessage()).toBeNull();
+  });
+
+  it('returns null when the env var equals the default (in KB)', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = String(DEFAULT_MAX_FILE_SIZE_BYTES / 1024);
+    expect(getMaxFileSizeBannerMessage()).toBeNull();
+  });
+
+  it('returns null when an invalid value falls back to the default', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
+    expect(getMaxFileSizeBannerMessage()).toBeNull();
+  });
+
+  it('reports the raised effective threshold in KB', () => {
+    process.env.GITNEXUS_MAX_FILE_SIZE = '1024';
+    const banner = getMaxFileSizeBannerMessage();
+    expect(banner).not.toBeNull();
+    expect(banner).toContain('effective threshold 1024KB');
+    expect(banner).toContain(`default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`);
+  });
+
+  it('reports the clamped (post-ceiling) threshold, not the raw input', () => {
+    const ceilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024;
+    const aboveCeilingKb = ceilingKb + 1024;
+    process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb);
+    const banner = getMaxFileSizeBannerMessage();
+    expect(banner).not.toBeNull();
+    expect(banner).toContain(`effective threshold ${ceilingKb}KB`);
+    expect(banner).not.toContain(`${aboveCeilingKb}KB`);
+  });
+});