mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-23 00:41:36 +00:00
* feat(ingestion): make large-file skip threshold configurable The walker previously hardcoded a 512KB skip threshold, which silently dropped legitimate large source files (e.g. ~900KB hand-written Java service classes) during analysis with no way to override short of editing source. Allow overrides via the GITNEXUS_MAX_FILE_SIZE env var (KB) — consistent with the existing GITNEXUS_NO_GITIGNORE / GITNEXUS_VERBOSE patterns — and a matching --max-file-size <kb> flag on gitnexus analyze. - New utility getMaxFileSizeBytes() in core/ingestion/utils/max-file-size.ts parses the env var, falls back to the 512KB default for missing/invalid values, and clamps against TREE_SITTER_MAX_BUFFER (32MB) to keep the downstream parser safe. - filesystem-walker.ts now resolves the threshold per call and drops the 'likely generated/vendored' editorial when the user has explicitly raised the limit. - analyze CLI wires --max-file-size to the env var and echoes a one-line notice when the threshold is overridden, mirroring how --no-gitignore is handled. - index.ts documents the new flag and env var under the analyze help text. - Warnings for invalid or out-of-range values are emitted exactly once per distinct value to avoid log spam. Tests: - New test/unit/max-file-size.test.ts covers defaults, KB parsing, clamp-at-ceiling, invalid-input fallback + warn-once, and distinct-value warnings. - test/integration/filesystem-walker.test.ts gains a 'large file skip threshold (#991)' block: 600KB fixture skipped by default, included under GITNEXUS_MAX_FILE_SIZE=1024, invalid values fall back and warn once, and the 'generated/vendored' suffix is only emitted under the default threshold. Closes #991 * fix(cli): show effective clamped max-file-size in banner Addresses the PR #1044 review finding: the startup banner printed the raw GITNEXUS_MAX_FILE_SIZE value rather than the clamped effective threshold, producing misleading telemetry when the value exceeded the 32 MB tree-sitter ceiling. The banner is also suppressed when the effective threshold equals the default, removing log noise when operators explicitly set the value to the current default. Extracted the logic into a new getMaxFileSizeBannerMessage() helper and pinned the behavior with unit tests covering default, raised override, invalid fallback, and above-ceiling clamp cases.
150 lines
5.1 KiB
TypeScript
150 lines
5.1 KiB
TypeScript
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
import {
|
|
DEFAULT_MAX_FILE_SIZE_BYTES,
|
|
MAX_FILE_SIZE_UPPER_BOUND_BYTES,
|
|
getMaxFileSizeBytes,
|
|
getMaxFileSizeBannerMessage,
|
|
_resetMaxFileSizeWarnings,
|
|
} from '../../src/core/ingestion/utils/max-file-size.js';
|
|
|
|
describe('getMaxFileSizeBytes', () => {
|
|
const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
let warnSpy: ReturnType<typeof vi.spyOn>;
|
|
|
|
beforeEach(() => {
|
|
delete process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
_resetMaxFileSizeWarnings();
|
|
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (ORIGINAL === undefined) {
|
|
delete process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
} else {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL;
|
|
}
|
|
warnSpy.mockRestore();
|
|
});
|
|
|
|
it('returns the default when the env var is unset', () => {
|
|
expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
|
|
expect(warnSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('parses a positive integer value as KB', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = '1024';
|
|
expect(getMaxFileSizeBytes()).toBe(1024 * 1024);
|
|
expect(warnSpy).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('clamps values above the tree-sitter ceiling', () => {
|
|
// One KB above the 32 MB ceiling.
|
|
const aboveCeilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024 + 1;
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb);
|
|
expect(getMaxFileSizeBytes()).toBe(MAX_FILE_SIZE_UPPER_BOUND_BYTES);
|
|
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
expect(warnSpy.mock.calls[0][0]).toContain('clamping');
|
|
});
|
|
|
|
it.each(['abc', '0', '-512', '1.5', 'NaN', ''])(
|
|
'falls back to the default and warns on invalid value %s',
|
|
(raw) => {
|
|
if (raw === '') {
|
|
// Empty string is treated as unset by the util (raw falsy check).
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = raw;
|
|
expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
|
|
expect(warnSpy).not.toHaveBeenCalled();
|
|
return;
|
|
}
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = raw;
|
|
expect(getMaxFileSizeBytes()).toBe(DEFAULT_MAX_FILE_SIZE_BYTES);
|
|
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
expect(warnSpy.mock.calls[0][0]).toContain('must be a positive integer');
|
|
},
|
|
);
|
|
|
|
it('deduplicates warnings for the same invalid value', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
|
|
getMaxFileSizeBytes();
|
|
getMaxFileSizeBytes();
|
|
getMaxFileSizeBytes();
|
|
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('warns separately for distinct invalid values', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
|
|
getMaxFileSizeBytes();
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = 'xyz';
|
|
getMaxFileSizeBytes();
|
|
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('_resetMaxFileSizeWarnings re-enables warnings after reset', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
|
|
getMaxFileSizeBytes();
|
|
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
|
|
getMaxFileSizeBytes();
|
|
expect(warnSpy).toHaveBeenCalledTimes(1);
|
|
|
|
_resetMaxFileSizeWarnings();
|
|
getMaxFileSizeBytes();
|
|
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('DEFAULT_MAX_FILE_SIZE_BYTES is 512 KB', () => {
|
|
expect(DEFAULT_MAX_FILE_SIZE_BYTES).toBe(512 * 1024);
|
|
});
|
|
});
|
|
|
|
describe('getMaxFileSizeBannerMessage', () => {
|
|
const ORIGINAL = process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
let warnSpy: ReturnType<typeof vi.spyOn>;
|
|
|
|
beforeEach(() => {
|
|
delete process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
_resetMaxFileSizeWarnings();
|
|
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
});
|
|
|
|
afterEach(() => {
|
|
if (ORIGINAL === undefined) {
|
|
delete process.env.GITNEXUS_MAX_FILE_SIZE;
|
|
} else {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = ORIGINAL;
|
|
}
|
|
warnSpy.mockRestore();
|
|
});
|
|
|
|
it('returns null when the env var is unset (default threshold)', () => {
|
|
expect(getMaxFileSizeBannerMessage()).toBeNull();
|
|
});
|
|
|
|
it('returns null when the env var equals the default (in KB)', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = String(DEFAULT_MAX_FILE_SIZE_BYTES / 1024);
|
|
expect(getMaxFileSizeBannerMessage()).toBeNull();
|
|
});
|
|
|
|
it('returns null when an invalid value falls back to the default', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = 'abc';
|
|
expect(getMaxFileSizeBannerMessage()).toBeNull();
|
|
});
|
|
|
|
it('reports the raised effective threshold in KB', () => {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = '1024';
|
|
const banner = getMaxFileSizeBannerMessage();
|
|
expect(banner).not.toBeNull();
|
|
expect(banner).toContain('effective threshold 1024KB');
|
|
expect(banner).toContain(`default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`);
|
|
});
|
|
|
|
it('reports the clamped (post-ceiling) threshold, not the raw input', () => {
|
|
const ceilingKb = MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024;
|
|
const aboveCeilingKb = ceilingKb + 1024;
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = String(aboveCeilingKb);
|
|
const banner = getMaxFileSizeBannerMessage();
|
|
expect(banner).not.toBeNull();
|
|
expect(banner).toContain(`effective threshold ${ceilingKb}KB`);
|
|
expect(banner).not.toContain(`${aboveCeilingKb}KB`);
|
|
});
|
|
});
|