import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; import fs from 'fs/promises'; import path from 'path'; import os from 'os'; import { walkRepositoryPaths, readFileContents, } from '../../src/core/ingestion/filesystem-walker.js'; import { _resetMaxFileSizeWarnings } from '../../src/core/ingestion/utils/max-file-size.js'; import { _captureLogger } from '../../src/core/logger.js'; describe('filesystem-walker', () => { let tmpDir: string; beforeAll(async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-test-')); // Create test directory structure await fs.mkdir(path.join(tmpDir, 'src'), { recursive: true }); await fs.mkdir(path.join(tmpDir, 'src', 'components'), { recursive: true }); await fs.mkdir(path.join(tmpDir, 'node_modules', 'lodash'), { recursive: true }); await fs.mkdir(path.join(tmpDir, '.git'), { recursive: true }); await fs.writeFile(path.join(tmpDir, 'src', 'index.ts'), 'export const main = () => {}'); await fs.writeFile(path.join(tmpDir, 'src', 'utils.ts'), 'export const helper = () => {}'); await fs.writeFile( path.join(tmpDir, 'src', 'components', 'Button.tsx'), 'export const Button = () =>
', ); await fs.writeFile( path.join(tmpDir, 'node_modules', 'lodash', 'index.js'), 'module.exports = {}', ); await fs.writeFile(path.join(tmpDir, '.git', 'HEAD'), 'ref: refs/heads/main'); await fs.writeFile(path.join(tmpDir, 'package.json'), '{}'); await fs.writeFile( path.join(tmpDir, 'src', 'image.png'), Buffer.from([0x89, 0x50, 0x4e, 0x47]), ); }); afterAll(async () => { try { await fs.rm(tmpDir, { recursive: true, force: true }); } catch { /* best-effort */ } }); describe('walkRepositoryPaths', () => { it('discovers source files', async () => { const files = await walkRepositoryPaths(tmpDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.some((p) => p.includes('src/index.ts'))).toBe(true); expect(paths.some((p) => p.includes('src/utils.ts'))).toBe(true); }); it('discovers nested files', async () => { const files = await walkRepositoryPaths(tmpDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.some((p) => p.includes('components/Button.tsx'))).toBe(true); }); it('skips node_modules', async () => { const files = await walkRepositoryPaths(tmpDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.every((p) => !p.includes('node_modules'))).toBe(true); }); it('skips .git directory', async () => { const files = await walkRepositoryPaths(tmpDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.every((p) => !p.includes('.git/'))).toBe(true); }); it('returns file sizes', async () => { const files = await walkRepositoryPaths(tmpDir); for (const file of files) { expect(typeof file.size).toBe('number'); expect(file.size).toBeGreaterThan(0); } }); it('calls progress callback', async () => { const onProgress = vi.fn(); await walkRepositoryPaths(tmpDir, onProgress); expect(onProgress).toHaveBeenCalled(); }); // ─── Unhappy paths ──────────────────────────────────────────────── it('throws or returns empty for non-existent directory', async () => { try { const files = await walkRepositoryPaths('/nonexistent/path/xyz123'); // If it doesn't throw, it should return empty expect(files).toEqual([]); } catch (err: any) { expect(err).toBeDefined(); } }); it('returns empty for directory with only ignored files', async () => { const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-empty-')); await fs.mkdir(path.join(emptyDir, '.git'), { recursive: true }); await fs.writeFile(path.join(emptyDir, '.git', 'HEAD'), 'ref: refs/heads/main'); try { const files = await walkRepositoryPaths(emptyDir); expect(files).toEqual([]); } finally { await fs.rm(emptyDir, { recursive: true, force: true }); } }); it('returns empty for truly empty directory', async () => { const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-truly-empty-')); try { const files = await walkRepositoryPaths(emptyDir); expect(files).toEqual([]); } finally { await fs.rm(emptyDir, { recursive: true, force: true }); } }); }); describe('.gitignore support', () => { let gitignoreDir: string; beforeAll(async () => { gitignoreDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-gitignore-')); // Create directory structure await fs.mkdir(path.join(gitignoreDir, 'src'), { recursive: true }); await fs.mkdir(path.join(gitignoreDir, 'data', 'cache'), { recursive: true }); await fs.mkdir(path.join(gitignoreDir, 'logs'), { recursive: true }); // Source files (should be indexed) await fs.writeFile( path.join(gitignoreDir, 'src', 'index.ts'), 'export const main = () => {}', ); await fs.writeFile( path.join(gitignoreDir, 'src', 'utils.ts'), 'export const helper = () => {}', ); // Data files (should be ignored via .gitignore) await fs.writeFile(path.join(gitignoreDir, 'data', 'cache', 'file.json'), '{}'); await fs.writeFile(path.join(gitignoreDir, 'logs', 'app.log'), 'log entry'); // .gitignore await fs.writeFile(path.join(gitignoreDir, '.gitignore'), 'data/\nlogs/\n'); }); afterAll(async () => { await fs.rm(gitignoreDir, { recursive: true, force: true }); }); it('excludes directories listed in .gitignore', async () => { const files = await walkRepositoryPaths(gitignoreDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); // Source files should be present expect(paths.some((p) => p.includes('src/index.ts'))).toBe(true); expect(paths.some((p) => p.includes('src/utils.ts'))).toBe(true); // Ignored directories should not be present expect(paths.every((p) => !p.includes('data/'))).toBe(true); expect(paths.every((p) => !p.includes('logs/'))).toBe(true); }); it('still applies hardcoded ignore list alongside .gitignore', async () => { // Add node_modules (hardcoded ignore) to verify both work await fs.mkdir(path.join(gitignoreDir, 'node_modules', 'pkg'), { recursive: true }); await fs.writeFile( path.join(gitignoreDir, 'node_modules', 'pkg', 'index.js'), 'module.exports = {}', ); const files = await walkRepositoryPaths(gitignoreDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.every((p) => !p.includes('node_modules'))).toBe(true); expect(paths.every((p) => !p.includes('data/'))).toBe(true); await fs.rm(path.join(gitignoreDir, 'node_modules'), { recursive: true, force: true }); }); }); describe('ambiguous source-directory names (#3039)', () => { let sourceDir: string; beforeAll(async () => { sourceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-source-names-')); await fs.mkdir(path.join(sourceDir, 'apps', 'client', 'src', 'shared', 'env'), { recursive: true, }); await fs.mkdir(path.join(sourceDir, 'packages', 'ai', 'src', 'generated'), { recursive: true, }); await fs.mkdir(path.join(sourceDir, 'build-cache', 'generated'), { recursive: true }); await fs.mkdir(path.join(sourceDir, 'env'), { recursive: true }); await fs.mkdir(path.join(sourceDir, 'generated'), { recursive: true }); await fs.mkdir(path.join(sourceDir, 'backend', 'env', 'Scripts'), { recursive: true }); await fs.mkdir(path.join(sourceDir, 'backend', 'env', 'include'), { recursive: true }); await fs.mkdir(path.join(sourceDir, 'backend', 'env', 'share'), { recursive: true }); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'shared', 'env', 'getAppEnv.ts'), 'export const getAppEnv = () => "test";\n', ); await fs.writeFile( path.join(sourceDir, 'packages', 'ai', 'src', 'generated', 'bundle.ts'), 'export const bundled = true;\n', ); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'vite-env.d.ts'), 'declare const APP_ENV: string;\n', ); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'service.ts'), 'export class UserService {}\n', ); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'service.d.ts'), 'export declare class UserService {}\n', ); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'legacy.js'), 'export class LegacyService {}\n', ); await fs.writeFile( path.join(sourceDir, 'apps', 'client', 'src', 'legacy.d.ts'), 'export declare class LegacyService {}\n', ); await fs.writeFile( path.join(sourceDir, 'build-cache', 'generated', 'ignored.ts'), 'export const ignored = true;\n', ); await fs.writeFile(path.join(sourceDir, '.gitignore'), 'build-cache/generated/\n'); await fs.writeFile(path.join(sourceDir, 'env', 'pyvenv.cfg'), 'home = python\n'); await fs.writeFile(path.join(sourceDir, 'env', 'settings.py'), 'VALUE = 1\n'); await fs.writeFile(path.join(sourceDir, 'backend', 'env', 'pyvenv.cfg'), 'home = python\n'); await fs.writeFile( path.join(sourceDir, 'backend', 'env', 'Scripts', 'activate_this.py'), 'VALUE = 1\n', ); await fs.writeFile( path.join(sourceDir, 'backend', 'env', 'include', 'header.py'), 'VALUE = 1\n', ); await fs.writeFile( path.join(sourceDir, 'backend', 'env', 'share', 'manual.py'), 'VALUE = 1\n', ); await fs.writeFile( path.join(sourceDir, 'generated', 'client.ts'), 'export const generatedClient = true;\n', ); }); afterAll(async () => { await fs.rm(sourceDir, { recursive: true, force: true }); }); it('discovers nested env/generated and .d.ts source while pruning root artifacts', async () => { const files = await walkRepositoryPaths(sourceDir); const paths = files.map((file) => file.path); expect(paths).toContain('apps/client/src/shared/env/getAppEnv.ts'); expect(paths).toContain('packages/ai/src/generated/bundle.ts'); expect(paths).toContain('apps/client/src/vite-env.d.ts'); expect(paths).toContain('apps/client/src/service.ts'); expect(paths).not.toContain('apps/client/src/service.d.ts'); expect(paths).toContain('apps/client/src/legacy.js'); expect(paths).toContain('apps/client/src/legacy.d.ts'); expect(paths).not.toContain('build-cache/generated/ignored.ts'); expect(paths).not.toContain('env/settings.py'); expect(paths).not.toContain('backend/env/Scripts/activate_this.py'); expect(paths).not.toContain('backend/env/include/header.py'); expect(paths).not.toContain('backend/env/share/manual.py'); expect(paths).not.toContain('generated/client.ts'); }); it('preserves case variants that were not hardcoded ignore names', async () => { const caseDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-source-case-')); try { await fs.mkdir(path.join(caseDir, 'Generated'), { recursive: true }); await fs.mkdir(path.join(caseDir, 'Env'), { recursive: true }); await fs.writeFile( path.join(caseDir, 'Generated', 'client.cs'), 'public class GeneratedClient {}\n', ); await fs.writeFile( path.join(caseDir, 'Env', 'settings.ts'), 'export const environment = "test";\n', ); const paths = (await walkRepositoryPaths(caseDir)).map((file) => file.path); expect(paths).toContain('Generated/client.cs'); expect(paths).toContain('Env/settings.ts'); } finally { await fs.rm(caseDir, { recursive: true, force: true }); } }); }); describe('.gitnexusignore support', () => { let nexusignoreDir: string; beforeAll(async () => { nexusignoreDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-nexusignore-')); await fs.mkdir(path.join(nexusignoreDir, 'src'), { recursive: true }); await fs.mkdir(path.join(nexusignoreDir, 'local', 'grafana'), { recursive: true }); await fs.writeFile( path.join(nexusignoreDir, 'src', 'index.ts'), 'export const main = () => {}', ); await fs.writeFile(path.join(nexusignoreDir, 'local', 'grafana', 'module.js'), 'var x = 1;'); // Only .gitnexusignore, no .gitignore await fs.writeFile(path.join(nexusignoreDir, '.gitnexusignore'), 'local/\n'); }); afterAll(async () => { await fs.rm(nexusignoreDir, { recursive: true, force: true }); }); it('excludes directories listed in .gitnexusignore', async () => { const files = await walkRepositoryPaths(nexusignoreDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.some((p) => p.includes('src/index.ts'))).toBe(true); expect(paths.every((p) => !p.includes('local/'))).toBe(true); }); }); describe('.gitnexusignore negation of hardcoded directories', () => { let partsDir: string; beforeAll(async () => { partsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-parts-')); await fs.mkdir(path.join(partsDir, 'parts', 'src', 'main', 'java', 'com', 'example'), { recursive: true, }); await fs.mkdir( path.join( partsDir, 'admin', 'src', 'main', 'java', 'com', 'example', 'controller', 'parts', ), { recursive: true }, ); await fs.mkdir(path.join(partsDir, 'node_modules', 'pkg'), { recursive: true }); await fs.writeFile(path.join(partsDir, '.gitnexusignore'), '!parts/\n'); await fs.writeFile( path.join(partsDir, 'parts', 'src', 'main', 'java', 'com', 'example', 'Part.java'), 'package com.example; class Part {}', ); await fs.writeFile( path.join( partsDir, 'admin', 'src', 'main', 'java', 'com', 'example', 'controller', 'parts', 'PartsController.java', ), 'package com.example.controller.parts; class PartsController {}', ); await fs.writeFile( path.join(partsDir, 'node_modules', 'pkg', 'index.js'), 'module.exports = {}', ); }); afterAll(async () => { await fs.rm(partsDir, { recursive: true, force: true }); }); it('traverses top-level and nested parts directories when explicitly unignored (#2673)', async () => { const files = await walkRepositoryPaths(partsDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths).toContain('parts/src/main/java/com/example/Part.java'); expect(paths).toContain( 'admin/src/main/java/com/example/controller/parts/PartsController.java', ); expect(paths.every((p) => !p.includes('node_modules/'))).toBe(true); }); }); describe('combined .gitignore + .gitnexusignore', () => { let combinedDir: string; beforeAll(async () => { combinedDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-combined-')); await fs.mkdir(path.join(combinedDir, 'src'), { recursive: true }); await fs.mkdir(path.join(combinedDir, 'data'), { recursive: true }); await fs.mkdir(path.join(combinedDir, 'local', 'plugins'), { recursive: true }); await fs.writeFile(path.join(combinedDir, 'src', 'index.ts'), 'export const main = () => {}'); await fs.writeFile(path.join(combinedDir, 'data', 'dump.json'), '{}'); await fs.writeFile(path.join(combinedDir, 'local', 'plugins', 'module.js'), 'var x = 1;'); await fs.writeFile(path.join(combinedDir, '.gitignore'), 'data/\n'); await fs.writeFile(path.join(combinedDir, '.gitnexusignore'), 'local/\n'); }); afterAll(async () => { await fs.rm(combinedDir, { recursive: true, force: true }); }); it('excludes directories from both files', async () => { const files = await walkRepositoryPaths(combinedDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.some((p) => p.includes('src/index.ts'))).toBe(true); expect(paths.every((p) => !p.includes('data/'))).toBe(true); expect(paths.every((p) => !p.includes('local/'))).toBe(true); }); }); describe('GITNEXUS_NO_GITIGNORE env var', () => { let envDir: string; beforeAll(async () => { envDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-noignore-')); await fs.mkdir(path.join(envDir, 'src'), { recursive: true }); await fs.mkdir(path.join(envDir, 'data'), { recursive: true }); await fs.writeFile(path.join(envDir, 'src', 'index.ts'), 'export const main = () => {}'); await fs.writeFile(path.join(envDir, 'data', 'dump.json'), '{}'); await fs.writeFile(path.join(envDir, '.gitignore'), 'data/\n'); }); afterAll(async () => { await fs.rm(envDir, { recursive: true, force: true }); }); it('excludes gitignored directory by default', async () => { const files = await walkRepositoryPaths(envDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.every((p) => !p.includes('data/'))).toBe(true); }); it('includes gitignored directory when GITNEXUS_NO_GITIGNORE is set', async () => { const original = process.env.GITNEXUS_NO_GITIGNORE; process.env.GITNEXUS_NO_GITIGNORE = '1'; try { const files = await walkRepositoryPaths(envDir); const paths = files.map((f) => f.path.replace(/\\/g, '/')); expect(paths.some((p) => p.includes('data/dump.json'))).toBe(true); } finally { if (original === undefined) { delete process.env.GITNEXUS_NO_GITIGNORE; } else { process.env.GITNEXUS_NO_GITIGNORE = original; } } }); }); describe('readFileContents', () => { it('reads file contents by relative paths', async () => { const contents = await readFileContents(tmpDir, ['src/index.ts', 'src/utils.ts']); expect(contents.get('src/index.ts')).toContain('main'); expect(contents.get('src/utils.ts')).toContain('helper'); }); it('handles empty path list', async () => { const contents = await readFileContents(tmpDir, []); expect(contents.size).toBe(0); }); it('skips non-existent files gracefully', async () => { const contents = await readFileContents(tmpDir, ['nonexistent.ts']); expect(contents.size).toBe(0); }); // ─── Unhappy paths ──────────────────────────────────────────────── it('skips multiple non-existent files gracefully', async () => { const contents = await readFileContents(tmpDir, ['a.ts', 'b.ts', 'c.ts']); expect(contents.size).toBe(0); }); it('handles binary file content without crashing', async () => { const contents = await readFileContents(tmpDir, ['src/image.png']); // May return content or skip — should not throw expect(contents.size).toBeLessThanOrEqual(1); }); }); describe('large file skip threshold (#991)', () => { let sizeDir: string; const BIG_FILE = 'src/big.ts'; const BIG_DECLARATION = 'src/big.d.ts'; const BIG_FILE_BYTES = 600 * 1024; const ORIGINAL_ENV = process.env.GITNEXUS_MAX_FILE_SIZE; let cap: ReturnType