From dbdac52976469012395765ec7a61d1f155605ffd Mon Sep 17 00:00:00 2001 From: brianburkett <1503764+brianburkett@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:45:29 -0700 Subject: [PATCH 1/5] feat(ingestion): opt-in indexing of named dot directories (GITNEXUS_INCLUDE_DOT_DIRS) --- gitnexus/README.md | 11 +++ .../group/extractors/include-extractor.ts | 9 +- gitnexus/src/core/index-content-drift.ts | 3 + .../src/core/ingestion/filesystem-walker.ts | 18 +++- .../core/ingestion/utils/include-dot-dirs.ts | 89 ++++++++++++++++++ gitnexus/src/core/run-analyze.ts | 4 + gitnexus/src/storage/repo-meta.ts | 5 ++ .../integration/filesystem-walker.test.ts | 90 +++++++++++++++++++ .../test/unit/group/include-extractor.test.ts | 21 +++++ gitnexus/test/unit/include-dot-dirs.test.ts | 77 ++++++++++++++++ .../test/unit/index-content-drift.test.ts | 46 ++++++++++ 11 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 gitnexus/src/core/ingestion/utils/include-dot-dirs.ts create mode 100644 gitnexus/test/unit/include-dot-dirs.test.ts diff --git a/gitnexus/README.md b/gitnexus/README.md index 373b64e35..5231c5fb3 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -856,6 +856,17 @@ npx gitnexus analyze Values above **32768 KB (32 MB)** are clamped to the tree-sitter parser ceiling; invalid values fall back to the 512 KB default with a one-time warning. When an override is active, `analyze` prints the effective threshold in its startup banner (e.g. `GITNEXUS_MAX_FILE_SIZE: effective threshold 2048KB (default 512KB)`). +### Code under a dot directory is missing + +The walker skips hidden paths, so symbols defined under a directory such as `.scallywag/` or `.config/` never reach the graph: `impact` and `context` answer `Target not found`, and `detect_changes` reports no changed symbols for a diff that only touches them. To index specific dot directories, list their names (comma-separated) and re-run `analyze`: + +```bash +export GITNEXUS_INCLUDE_DOT_DIRS=.scallywag,.config +npx gitnexus analyze +``` + +Each name matches that directory at any depth (`pkg/.scallywag/` too). Hidden files and other dot directories inside it stay excluded, ignore rules still apply (the built-in list covers `.github/`, so also add `!.github/` to `.gitnexusignore` to index it), and version-control directories such as `.git` are refused. The list is recorded with the index, so `status` compares against the same file set even if the variable changes later. + ### Process detection reports missing flows On a large repository, `analyze` may warn that `[processes] … whole flows are MISSING`. That means ranked entry points or completed flows were sampled away by the analyze-time detection budget — not that the code path is absent, and not the query-time `IMPACT_MAX_CHUNKS` cap. diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts index bd11a388d..f3f251136 100644 --- a/gitnexus/src/core/group/extractors/include-extractor.ts +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -24,6 +24,10 @@ import { readSafe } from './fs-utils.js'; import { buildSuffixIndex, type SuffixIndex } from '../../ingestion/utils/suffix-index.js'; import { createIgnoreFilter } from '../../../config/ignore-service.js'; import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { + getIncludeDotDirs, + indexableGlobPatterns, +} from '../../ingestion/utils/include-dot-dirs.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; import { logger } from '../../logger.js'; @@ -364,6 +368,8 @@ export class IncludeExtractor implements ContractExtractor { * - `createIgnoreFilter` honors `.gitignore`, `.gitnexusignore`, the * hardcoded ignore list, and `.gitnexusignore` last-match-wins * negation. + * - `indexableGlobPatterns` adds the `GITNEXUS_INCLUDE_DOT_DIRS` + * directories to the non-hidden file set. * - Only regular file targets are indexable (file symlinks are retained). * - `getMaxFileSizeBytes()` drops files larger than the cap so we * never emit `File:` UIDs for files ingestion would skip. @@ -382,8 +388,9 @@ export class IncludeExtractor implements ContractExtractor { private async discoverIndexableFiles(repoPath: string): Promise { const ignoreFilter = await createIgnoreFilter(repoPath); const maxFileSizeBytes = getMaxFileSizeBytes(); + const patterns = indexableGlobPatterns(getIncludeDotDirs()); - const candidates = await glob('**/*', { + const candidates = await glob(patterns, { cwd: repoPath, nodir: true, dot: false, diff --git a/gitnexus/src/core/index-content-drift.ts b/gitnexus/src/core/index-content-drift.ts index 3599db7b1..20a91f74e 100644 --- a/gitnexus/src/core/index-content-drift.ts +++ b/gitnexus/src/core/index-content-drift.ts @@ -103,6 +103,9 @@ export const detectIndexContentDrift = async ( const scanned = await walkRepositoryPaths(repoPath, undefined, { quiet: true, maxFileSizeBytes: coverage?.maxFileSizeBytes, + // Replay the analyze-time list; an index without one covered no dot + // directories, whatever the current environment says. + includeDotDirs: coverage?.includeDotDirs ?? [], }); const scannedPaths = scanned.map((file) => file.path).filter((p) => !isGitNexusManagedPath(p)); const scannedSet = new Set(scannedPaths); diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 9dc47e7b6..63f790b3f 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -1,5 +1,10 @@ import { isVerboseIngestionEnabled } from './utils/verbose.js'; import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js'; +import { + getIncludeDotDirs, + indexableGlobPatterns, + normalizeIncludeDotDirs, +} from './utils/include-dot-dirs.js'; import fs from 'fs/promises'; import { readdir } from 'node:fs'; import path from 'path'; @@ -73,6 +78,14 @@ export interface WalkRepositoryOptions { * silently drop a file that the index actually covers. */ maxFileSizeBytes?: number; + /** + * Dot directories to index besides the default non-hidden file set, e.g. + * `['.scallywag']`. Defaults to `GITNEXUS_INCLUDE_DOT_DIRS`. `status` replays + * the list recorded at analyze time, for the same reason as + * `maxFileSizeBytes`. Hidden files and other dot directories inside a listed + * directory stay excluded, and ignore rules still apply. + */ + includeDotDirs?: readonly string[]; } /** @@ -103,10 +116,13 @@ export const walkRepositoryPaths = async ( await assertWalkRootIsDirectory(repoPath); const ignoreFilter = await createIgnoreFilter(repoPath); const maxFileSizeBytes = options.maxFileSizeBytes ?? getMaxFileSizeBytes(); + const includeDotDirs = normalizeIncludeDotDirs(options.includeDotDirs ?? getIncludeDotDirs()); let enumerationError: NodeJS.ErrnoException | undefined; + const patterns = indexableGlobPatterns(includeDotDirs); + const filtered = ( - await glob('**/*', { + await glob(patterns, { cwd: repoPath, nodir: true, dot: false, diff --git a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts new file mode 100644 index 000000000..d86bee032 --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts @@ -0,0 +1,89 @@ +import { logger } from '../../logger.js'; + +/** + * Directory names that must never be indexed, even when listed. These hold + * version-control internals, not source. + */ +const REFUSED_DOT_DIRS = new Set(['.git', '.svn', '.hg', '.bzr']); + +/** A single path segment: a leading "." followed by letters, digits, ".", "_" or "-". */ +const DOT_DIR_NAME = /^\.[A-Za-z0-9._-]+$/; + +const warned = new Set(); + +const warnOnce = (key: string, message: string): void => { + if (warned.has(key)) return; + warned.add(key); + logger.warn(message); +}; + +/** + * Validate a list of dot-directory names to index. Each entry must be a single + * path segment that starts with "." (e.g. ".scallywag", ".github"). Version + * control directories are refused. Invalid entries are dropped with a one-time + * warning. The result is deduplicated and sorted so it can be recorded in the + * index coverage policy and compared later. + */ +export const normalizeIncludeDotDirs = (entries: readonly string[]): string[] => { + const accepted = new Set(); + for (const raw of entries) { + const entry = raw.trim().replace(/\/+$/, ''); + if (entry === '') continue; + // Plain characters only: the name is spliced into a glob pattern, so path + // separators and glob syntax (* ? [ { !) must never reach it. + if (!DOT_DIR_NAME.test(entry) || entry === '..') { + warnOnce( + `invalid:${raw}`, + ` GITNEXUS_INCLUDE_DOT_DIRS entries must be single dot-directory names like ".github", got "${raw}" — ignoring it`, + ); + continue; + } + if (REFUSED_DOT_DIRS.has(entry)) { + warnOnce( + `refused:${entry}`, + ` GITNEXUS_INCLUDE_DOT_DIRS cannot include version-control directory "${entry}" — ignoring it`, + ); + continue; + } + accepted.add(entry); + } + return [...accepted].sort(); +}; + +/** + * Resolve the dot directories the walker should index in addition to the + * default (non-hidden) file set. Reads `GITNEXUS_INCLUDE_DOT_DIRS`, a + * comma-separated list of directory names. Unset or empty means none, which + * keeps the default behaviour unchanged. + */ +export const getIncludeDotDirs = (): string[] => { + const raw = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + if (!raw) return []; + return normalizeIncludeDotDirs(raw.split(',')); +}; + +/** + * The glob patterns for the indexable file set: every non-hidden file, plus + * each listed dot directory at any depth. `dot: false` keeps hidden paths + * out, but glob still matches a dot segment that a pattern names literally; + * glob deduplicates matches across patterns. Every file discovery that must + * agree with ingestion builds its patterns here. + */ +export const indexableGlobPatterns = (includeDotDirs: readonly string[]): string[] => [ + '**/*', + ...includeDotDirs.flatMap((dir) => [`${dir}/**/*`, `**/${dir}/**/*`]), +]; + +/** + * The `indexCoverage` fragment recording the active list: `{ includeDotDirs }` + * when any directory is listed, otherwise `{}`. + */ +export const includeDotDirsCoverage = (): { includeDotDirs?: string[] } => { + const includeDotDirs = getIncludeDotDirs(); + return includeDotDirs.length > 0 ? { includeDotDirs } : {}; +}; + +/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */ +export const _resetIncludeDotDirsWarnings = (): void => { + warned.clear(); +}; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 7cec865c2..30d578e94 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -278,6 +278,7 @@ import { } from '../storage/git.js'; import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; import { getMaxFileSizeBytes } from './ingestion/utils/max-file-size.js'; +import { includeDotDirsCoverage } from './ingestion/utils/include-dot-dirs.js'; import { cacheRowCount, discardScopedEmbeddingSpills, @@ -5388,6 +5389,9 @@ async function runFullAnalysisInner( indexCoverage: hasGitDir(repoPath) ? { maxFileSizeBytes: getMaxFileSizeBytes(), + // Stamped only when set, so indexes built without the option keep + // their existing metadata shape. + ...includeDotDirsCoverage(), dirtyPaths: ( listWorkingTreeDirtyPaths(repoPath) ?? Object.keys(newFileHashesRecord) ).filter( diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index f62684714..e97c80e36 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -398,6 +398,11 @@ export interface RepoMeta { */ indexCoverage?: { maxFileSizeBytes: number; + /** + * Dot directories indexed via `GITNEXUS_INCLUDE_DOT_DIRS`. Absent means + * none: the index predates the option or was built without it. + */ + includeDotDirs?: string[]; dirtyPaths?: string[]; }; /** diff --git a/gitnexus/test/integration/filesystem-walker.test.ts b/gitnexus/test/integration/filesystem-walker.test.ts index 4cf6ca5e4..d81e6a8f5 100644 --- a/gitnexus/test/integration/filesystem-walker.test.ts +++ b/gitnexus/test/integration/filesystem-walker.test.ts @@ -479,6 +479,96 @@ describe('filesystem-walker', () => { }); }); + describe('opt-in dot directories (GITNEXUS_INCLUDE_DOT_DIRS)', () => { + let dotDir: string; + + beforeAll(async () => { + dotDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-dotdirs-')); + const write = async (rel: string, content: string) => { + await fs.mkdir(path.dirname(path.join(dotDir, rel)), { recursive: true }); + await fs.writeFile(path.join(dotDir, rel), content); + }; + await write('src/index.ts', 'export const main = () => {}'); + await write('.scallywag/root.ts', 'export const root = 1;'); + await write('pkg/.scallywag/unit.ts', 'export const unit = 1;'); + await write('pkg/.scallywag/nested/testing.ts', 'export const testing = 1;'); + await write('pkg/.scallywag/.env.local', 'SECRET=1'); + await write('pkg/.scallywag/.cache/blob.ts', 'export const cached = 1;'); + await write('pkg/.scallywag/generated/out.ts', 'export const generated = 1;'); + await write('pkg/.other/hidden.ts', 'export const other = 1;'); + await write('.git/HEAD', 'ref: refs/heads/main'); + await write('.gitnexusignore', 'pkg/.scallywag/generated/\n'); + }); + + afterAll(async () => { + await fs.rm(dotDir, { recursive: true, force: true }); + }); + + const walkPaths = async (includeDotDirs?: readonly string[]) => + (await walkRepositoryPaths(dotDir, undefined, { includeDotDirs })).map((f) => f.path); + + it('skips dot directories by default', async () => { + const original = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + try { + expect(await walkPaths()).toEqual(['src/index.ts']); + } finally { + if (original !== undefined) process.env.GITNEXUS_INCLUDE_DOT_DIRS = original; + } + }); + + it('indexes a listed dot directory at any depth, keeping hidden files and other dot dirs out', async () => { + expect(await walkPaths(['.scallywag'])).toEqual([ + '.scallywag/root.ts', + 'pkg/.scallywag/nested/testing.ts', + 'pkg/.scallywag/unit.ts', + 'src/index.ts', + ]); + }); + + it('reads the list from GITNEXUS_INCLUDE_DOT_DIRS when no option is passed', async () => { + const original = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.other'; + try { + expect(await walkPaths()).toEqual(['pkg/.other/hidden.ts', 'src/index.ts']); + } finally { + if (original === undefined) { + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + } else { + process.env.GITNEXUS_INCLUDE_DOT_DIRS = original; + } + } + }); + + it('keeps a listed directory on the built-in ignore list out until .gitnexusignore negates it', async () => { + const ghDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-walker-dotdirs-github-')); + try { + await fs.mkdir(path.join(ghDir, '.github', 'scripts'), { recursive: true }); + await fs.writeFile(path.join(ghDir, '.github', 'scripts', 'queue.mjs'), 'export {};'); + await fs.writeFile(path.join(ghDir, 'index.ts'), 'export {};'); + const walk = async () => + (await walkRepositoryPaths(ghDir, undefined, { includeDotDirs: ['.github'] })).map( + (f) => f.path, + ); + + expect(await walk()).toEqual(['index.ts']); + await fs.writeFile(path.join(ghDir, '.gitnexusignore'), '!.github/\n'); + expect(await walk()).toEqual(['.github/scripts/queue.mjs', 'index.ts']); + } finally { + await fs.rm(ghDir, { recursive: true, force: true }); + } + }); + + it('never indexes .git even when listed', async () => { + const cap = _captureLogger(); + try { + expect(await walkPaths(['.git'])).toEqual(['src/index.ts']); + } finally { + cap.restore(); + } + }); + }); + describe('readFileContents', () => { it('reads file contents by relative paths', async () => { const contents = await readFileContents(tmpDir, ['src/index.ts', 'src/utils.ts']); diff --git a/gitnexus/test/unit/group/include-extractor.test.ts b/gitnexus/test/unit/group/include-extractor.test.ts index 8bc0bfc06..614229fb6 100644 --- a/gitnexus/test/unit/group/include-extractor.test.ts +++ b/gitnexus/test/unit/group/include-extractor.test.ts @@ -623,6 +623,27 @@ int auto_main() { return 0; }`, else process.env.GITNEXUS_MAX_FILE_SIZE = previous; } }); + + it('registers headers in a dot directory only when GITNEXUS_INCLUDE_DOT_DIRS lists it', async () => { + const previous = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + writeFile('.config/listed.h', '#pragma once'); + writeFile('.cache/unlisted.h', '#pragma once'); + try { + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + const before = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + expect(before.map((c) => c.contractId)).not.toContain('include::.config/listed.h'); + + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.config'; + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::.config/listed.h'); + expect(providerIds).not.toContain('include::.cache/unlisted.h'); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + else process.env.GITNEXUS_INCLUDE_DOT_DIRS = previous; + } + }); }); describe('Windows SIGSEGV regression — large input must route through parseSourceSafe', () => { diff --git a/gitnexus/test/unit/include-dot-dirs.test.ts b/gitnexus/test/unit/include-dot-dirs.test.ts new file mode 100644 index 000000000..ab9a3ed3f --- /dev/null +++ b/gitnexus/test/unit/include-dot-dirs.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + getIncludeDotDirs, + includeDotDirsCoverage, + indexableGlobPatterns, + normalizeIncludeDotDirs, + _resetIncludeDotDirsWarnings, +} from '../../src/core/ingestion/utils/include-dot-dirs.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +describe('include-dot-dirs', () => { + const ORIGINAL = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + let cap: ReturnType; + + beforeEach(() => { + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + _resetIncludeDotDirsWarnings(); + cap = _captureLogger(); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + } else { + process.env.GITNEXUS_INCLUDE_DOT_DIRS = ORIGINAL; + } + cap.restore(); + }); + + it('returns no directories when the env var is unset', () => { + expect(getIncludeDotDirs()).toEqual([]); + expect(includeDotDirsCoverage()).toEqual({}); + expect(cap.records().length).toBe(0); + }); + + it('parses a comma-separated list, trimmed, deduplicated and sorted', () => { + process.env.GITNEXUS_INCLUDE_DOT_DIRS = ' .scallywag, .github/,.scallywag '; + expect(getIncludeDotDirs()).toEqual(['.github', '.scallywag']); + expect(includeDotDirsCoverage()).toEqual({ includeDotDirs: ['.github', '.scallywag'] }); + expect(cap.records().length).toBe(0); + }); + + it('refuses version-control directories with a warning', () => { + expect(normalizeIncludeDotDirs(['.git', '.hg', '.scallywag'])).toEqual(['.scallywag']); + const messages = cap.records().map((r) => String(r.msg)); + expect(messages.some((m) => m.includes('".git"'))).toBe(true); + expect(messages.some((m) => m.includes('".hg"'))).toBe(true); + }); + + it.each(['scallywag', '.', '..', 'a/.b', '.a/b', '.a\\b', '.{a,b}', '.a*', '.[ab]', '!.a'])( + 'rejects %j as not a plain dot-directory name', + (entry) => { + expect(normalizeIncludeDotDirs([entry])).toEqual([]); + expect(cap.records().length).toBe(1); + }, + ); + + it('builds the default pattern alone when no directory is listed', () => { + expect(indexableGlobPatterns([])).toEqual(['**/*']); + }); + + it('adds each listed directory at the root and at any depth', () => { + expect(indexableGlobPatterns(['.github', '.scallywag'])).toEqual([ + '**/*', + '.github/**/*', + '**/.github/**/*', + '.scallywag/**/*', + '**/.scallywag/**/*', + ]); + }); + + it('warns once per invalid entry', () => { + normalizeIncludeDotDirs(['bad', 'bad']); + normalizeIncludeDotDirs(['bad']); + expect(cap.records().length).toBe(1); + }); +}); diff --git a/gitnexus/test/unit/index-content-drift.test.ts b/gitnexus/test/unit/index-content-drift.test.ts index 04b26785b..dd0e78b65 100644 --- a/gitnexus/test/unit/index-content-drift.test.ts +++ b/gitnexus/test/unit/index-content-drift.test.ts @@ -224,6 +224,52 @@ describe('detectIndexContentDrift', () => { expect(await detectIndexContentDrift(repo, recorded)).toMatchObject({ kind: 'current' }); }); + it('replays the analyze-time dot-directory list, whatever the environment says now', async () => { + const repo = makeRepo({ + 'a.js': 'export const a = 1;\n', + 'pkg/.scallywag/unit.js': 'export const unit = 1;\n', + }); + const policy = { maxFileSizeBytes: 512 * 1024, includeDotDirs: ['.scallywag'] }; + const recorded = await recordCoverage(repo, { includeDotDirs: policy.includeDotDirs }); + expect(Object.keys(recorded)).toContain('pkg/.scallywag/unit.js'); + + const original = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + try { + expect(await detectIndexContentDrift(repo, recorded, policy)).toMatchObject({ + kind: 'current', + }); + fs.writeFileSync(path.join(repo, 'pkg/.scallywag/unit.js'), 'export const unit = 2;\n'); + expect(await detectIndexContentDrift(repo, recorded, policy)).toMatchObject({ + kind: 'drifted', + changed: ['pkg/.scallywag/unit.js'], + }); + } finally { + if (original !== undefined) process.env.GITNEXUS_INCLUDE_DOT_DIRS = original; + } + }); + + it('does not report dot-directory files as added for an index built without the option', async () => { + const repo = makeRepo({ 'a.js': 'export const a = 1;\n' }); + const recorded = await recordCoverage(repo, { includeDotDirs: [] }); + fs.mkdirSync(path.join(repo, '.scallywag')); + fs.writeFileSync(path.join(repo, '.scallywag/unit.js'), 'export const unit = 1;\n'); + + const original = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.scallywag'; + try { + expect( + await detectIndexContentDrift(repo, recorded, { maxFileSizeBytes: 512 * 1024 }), + ).toMatchObject({ kind: 'current' }); + } finally { + if (original === undefined) { + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + } else { + process.env.GITNEXUS_INCLUDE_DOT_DIRS = original; + } + } + }); + it('treats a covered file that can no longer be read as changed, not current', async () => { if (typeof process.getuid === 'function' && process.getuid() === 0) { return; From b90581df6779b3e68bcd0bde76d6e87e444c1ef9 Mon Sep 17 00:00:00 2001 From: brianburkett <1503764+brianburkett@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:20:18 -0700 Subject: [PATCH 2/5] fix(walker): a listed dot directory absent from the repo is empty, not a scan failure The root-level pattern makes glob read the listed directory itself; in strict enumeration (onPathsDiscovered) that ENOENT was recorded as fatal. Skip ENOENT only for the listed roots; every other read error stays fatal. --- .../src/core/ingestion/filesystem-walker.ts | 7 ++++++- .../integration/filesystem-walker.test.ts | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 63f790b3f..09323c4d5 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -120,6 +120,11 @@ export const walkRepositoryPaths = async ( let enumerationError: NodeJS.ErrnoException | undefined; const patterns = indexableGlobPatterns(includeDotDirs); + // A listed dot directory may be absent from this repo; glob still reads its + // root-level pattern, and that ENOENT means "no files", not lost metadata. + const optionalRoots = new Set(includeDotDirs.map((dir) => path.join(repoPath, dir))); + const isAbsentListedRoot = (directory: unknown, error: NodeJS.ErrnoException): boolean => + error.code === 'ENOENT' && optionalRoots.has(path.resolve(repoPath, String(directory))); const filtered = ( await glob(patterns, { @@ -135,7 +140,7 @@ export const walkRepositoryPaths = async ( : { readdir(directory, readOptions, callback) { readdir(directory, readOptions, (error, entries) => { - if (error) enumerationError ??= error; + if (error && !isAbsentListedRoot(directory, error)) enumerationError ??= error; callback(error, entries); }); }, diff --git a/gitnexus/test/integration/filesystem-walker.test.ts b/gitnexus/test/integration/filesystem-walker.test.ts index d81e6a8f5..728a3e461 100644 --- a/gitnexus/test/integration/filesystem-walker.test.ts +++ b/gitnexus/test/integration/filesystem-walker.test.ts @@ -559,6 +559,26 @@ describe('filesystem-walker', () => { } }); + it('treats a listed dot directory absent from the repo as empty, even in strict enumeration', async () => { + const discovered: string[] = []; + const paths = ( + await walkRepositoryPaths(dotDir, undefined, { + includeDotDirs: ['.absent', '.scallywag'], + onPathsDiscovered: async (found) => { + discovered.push(...found); + }, + }) + ).map((f) => f.path); + + expect(paths).toEqual([ + '.scallywag/root.ts', + 'pkg/.scallywag/nested/testing.ts', + 'pkg/.scallywag/unit.ts', + 'src/index.ts', + ]); + expect(discovered.length).toBeGreaterThan(0); + }); + it('never indexes .git even when listed', async () => { const cap = _captureLogger(); try { From bfef50ac31b5d18d120bbfeeddc80baf8c6b91e5 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 28 Oct 2026 19:59:58 +0000 Subject: [PATCH 3/5] fix(ingestion): protect dot-directory indexing --- docs/cli-reference.md | 1 + gitnexus/README.md | 6 +- .../group/extractors/include-extractor.ts | 6 +- .../src/core/ingestion/filesystem-walker.ts | 10 +- .../core/ingestion/utils/include-dot-dirs.ts | 78 +++++++++++++-- gitnexus/src/core/run-analyze.ts | 8 +- .../integration/filesystem-walker.test.ts | 96 +++++++++++++++++++ .../test/unit/group/include-extractor.test.ts | 36 +++++++ gitnexus/test/unit/include-dot-dirs.test.ts | 18 ++++ gitnexus/test/unit/run-analyze.test.ts | 58 +++++++++++ 10 files changed, 303 insertions(+), 14 deletions(-) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index dd4885cd5..b0b26ad9f 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -237,6 +237,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | | `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | | `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_INCLUDE_DOT_DIRS` | unset | Comma-separated dot-directory names to index, e.g. `.github,.scallywag`. Changes take effect on the next analyze; status replays the recorded list. Hidden files, credential paths, and symlinks under opted-in paths remain excluded. Ignore rules still apply. | Indexing trusted source under hidden directories. For `.github`, also add `!.github/` to `.gitnexusignore`. Path protection does not scan source text for secrets. | | `GITNEXUS_MAX_PROCESSES` | dynamic (`max(20, round(symbols/10))`) | Analyze-time process-detection process cap. Equivalent to `--max-processes ` / `.gitnexusrc` `maxProcesses`. Explicit values replace the dynamic formula (not a multiplier). `0` is invalid, not unlimited. Changing this re-detects flows on the next analyze without `--force`. Distinct from query-time `IMPACT_MAX_CHUNKS`. | `[processes] … whole flows are MISSING` names `--max-processes` after entry points were never traced or flows were dropped. Tracing does not start the next entry once collected traces already reach `maxProcesses * 2`; a started entry can still emit every trace that entry produces. | | `GITNEXUS_MAX_PROCESS_BRANCHING` | `4` | Analyze-time per-node branching cap during flow tracing. Equivalent to `--max-process-branching `. Shape-only: raising it shortens fewer traces; it does not restore whole missing flows. | A flow is present but `calleesDropped` is high at debug. | | `GITNEXUS_MAX_PROCESS_TRACE_DEPTH` | `10` | Analyze-time DFS depth cap during flow tracing. Equivalent to `--max-process-trace-depth `. Shape-only. | A reported flow is shorter than the code path (`tracesDepthCapped` at debug). | diff --git a/gitnexus/README.md b/gitnexus/README.md index 5231c5fb3..4d6acc374 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -865,7 +865,11 @@ export GITNEXUS_INCLUDE_DOT_DIRS=.scallywag,.config npx gitnexus analyze ``` -Each name matches that directory at any depth (`pkg/.scallywag/` too). Hidden files and other dot directories inside it stay excluded, ignore rules still apply (the built-in list covers `.github/`, so also add `!.github/` to `.gitnexusignore` to index it), and version-control directories such as `.git` are refused. The list is recorded with the index, so `status` compares against the same file set even if the variable changes later. +Each name matches that directory at any depth (`pkg/.scallywag/` too). Hidden files and other dot directories inside it stay excluded. Ignore rules still apply (the built-in list covers `.github/`, so also add `!.github/` to `.gitnexusignore` to index it). Changing or removing the list takes effect on the next `analyze`, even on a clean checkout. The list is recorded with the index, so `status` compares against the same file set even if the variable changes later. + +Version-control and credential directories are refused case-insensitively, including `.git`, `.ssh`, `.aws`, `.azure`, `.kube`, `.gnupg`, and `.docker`. Inside opted-in directories, known credential files, key/certificate files, and credential stores such as `.config/gcloud/` and `.config/gh/` stay excluded even when ignore rules negate their exclusions. Symlink files and directory links under these paths are excluded, including a listed directory that is itself a symlink. These checks run before content or resolution metadata is read. + +This is path-based protection, not secret-content scanning. List only directories whose contents you trust to expose through GitNexus's code tools; a hardcoded secret in an otherwise allowed source file can still be indexed. Use `.gitnexusignore` for project-specific secret paths. ### Process detection reports missing flows diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts index f3f251136..d669346f6 100644 --- a/gitnexus/src/core/group/extractors/include-extractor.ts +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -27,6 +27,7 @@ import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; import { getIncludeDotDirs, indexableGlobPatterns, + dotDirectoryIgnoreFilter, } from '../../ingestion/utils/include-dot-dirs.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; import { logger } from '../../logger.js'; @@ -388,13 +389,14 @@ export class IncludeExtractor implements ContractExtractor { private async discoverIndexableFiles(repoPath: string): Promise { const ignoreFilter = await createIgnoreFilter(repoPath); const maxFileSizeBytes = getMaxFileSizeBytes(); - const patterns = indexableGlobPatterns(getIncludeDotDirs()); + const includeDotDirs = getIncludeDotDirs(); + const patterns = indexableGlobPatterns(includeDotDirs); const candidates = await glob(patterns, { cwd: repoPath, nodir: true, dot: false, - ignore: ignoreFilter, + ignore: includeDotDirs.length > 0 ? dotDirectoryIgnoreFilter(ignoreFilter) : ignoreFilter, }); const survivors: string[] = []; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 09323c4d5..c02d9de67 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -4,6 +4,7 @@ import { getIncludeDotDirs, indexableGlobPatterns, normalizeIncludeDotDirs, + dotDirectoryIgnoreFilter, } from './utils/include-dot-dirs.js'; import fs from 'fs/promises'; import { readdir } from 'node:fs'; @@ -122,16 +123,19 @@ export const walkRepositoryPaths = async ( const patterns = indexableGlobPatterns(includeDotDirs); // A listed dot directory may be absent from this repo; glob still reads its // root-level pattern, and that ENOENT means "no files", not lost metadata. - const optionalRoots = new Set(includeDotDirs.map((dir) => path.join(repoPath, dir))); + const optionalRoots = new Set(includeDotDirs.map((dir) => path.resolve(repoPath, dir))); const isAbsentListedRoot = (directory: unknown, error: NodeJS.ErrnoException): boolean => - error.code === 'ENOENT' && optionalRoots.has(path.resolve(repoPath, String(directory))); + (error.code === 'ENOENT' && optionalRoots.has(path.resolve(repoPath, String(directory)))) || + // A dotfile can share a listed directory's name. Glob probes it as a + // directory; ENOTDIR means no matching subtree, not an unreadable one. + (error.code === 'ENOTDIR' && includeDotDirs.includes(path.basename(String(directory)))); const filtered = ( await glob(patterns, { cwd: repoPath, nodir: true, dot: false, - ignore: ignoreFilter, + ignore: includeDotDirs.length > 0 ? dotDirectoryIgnoreFilter(ignoreFilter) : ignoreFilter, // glob treats unreadable directories as empty. Required metadata must // not silently lose declarations, so retain the first enumeration error. fs: diff --git a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts index d86bee032..0db3a38cf 100644 --- a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts +++ b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts @@ -1,10 +1,48 @@ import { logger } from '../../logger.js'; +import type { IgnoreLike } from 'glob'; +import type { Path } from 'path-scurry'; /** * Directory names that must never be indexed, even when listed. These hold - * version-control internals, not source. + * version-control internals or credential stores, not source. */ -const REFUSED_DOT_DIRS = new Set(['.git', '.svn', '.hg', '.bzr']); +const REFUSED_DOT_DIRS = new Set([ + '.git', + '.svn', + '.hg', + '.bzr', + '.ssh', + '.aws', + '.azure', + '.kube', + '.gnupg', + '.docker', + '.npm', + '.pki', + '.password-store', + '.vault', +]); + +const CREDENTIAL_CONFIG_DIRS = new Set(['gcloud', 'gh', 'hub', 'aws', 'azure', 'sops', 'op']); +const CREDENTIAL_FILES = new Set([ + 'credentials', + 'credentials.json', + 'credentials.db', + 'application_default_credentials.json', + 'access_tokens.db', + 'accesstokens.json', + 'auth.json', + 'tokens.json', + 'secrets.json', + 'secrets.yaml', + 'secrets.yml', + 'secrets.toml', + 'id_rsa', + 'id_dsa', + 'id_ecdsa', + 'id_ed25519', +]); +const KEY_FILE = /\.(?:pem|key|crt|cer|p12|pfx)$/i; /** A single path segment: a leading "." followed by letters, digits, ".", "_" or "-". */ const DOT_DIR_NAME = /^\.[A-Za-z0-9._-]+$/; @@ -31,17 +69,18 @@ export const normalizeIncludeDotDirs = (entries: readonly string[]): string[] => if (entry === '') continue; // Plain characters only: the name is spliced into a glob pattern, so path // separators and glob syntax (* ? [ { !) must never reach it. - if (!DOT_DIR_NAME.test(entry) || entry === '..') { + // Windows strips trailing dots, which would turn ".git." into ".git". + if (!DOT_DIR_NAME.test(entry) || entry.endsWith('.')) { warnOnce( `invalid:${raw}`, - ` GITNEXUS_INCLUDE_DOT_DIRS entries must be single dot-directory names like ".github", got "${raw}" — ignoring it`, + ' Ignoring invalid GITNEXUS_INCLUDE_DOT_DIRS entry; expected a single dot-directory name like ".github"', ); continue; } - if (REFUSED_DOT_DIRS.has(entry)) { + if (REFUSED_DOT_DIRS.has(entry.toLowerCase())) { warnOnce( `refused:${entry}`, - ` GITNEXUS_INCLUDE_DOT_DIRS cannot include version-control directory "${entry}" — ignoring it`, + ` GITNEXUS_INCLUDE_DOT_DIRS cannot include version-control or credential directory "${entry}" — ignoring it`, ); continue; } @@ -74,6 +113,33 @@ export const indexableGlobPatterns = (includeDotDirs: readonly string[]): string ...includeDotDirs.flatMap((dir) => [`${dir}/**/*`, `**/${dir}/**/*`]), ]; +/** + * Opting into source directories must not opt into credentials or symlink + * targets. Apply this before user ignore negations and metadata discovery. + * Ordinary non-hidden paths retain the existing ignore/symlink policy. + */ +export const dotDirectoryIgnoreFilter = (base: IgnoreLike): IgnoreLike => { + const protectedPath = (entry: Path): boolean => { + const parts = entry.relative().replace(/\\/g, '/').toLowerCase().split('/'); + if (!parts.some((part) => part.startsWith('.'))) return false; + if (parts.some((part) => REFUSED_DOT_DIRS.has(part))) return true; + if (parts.some((part, i) => part === '.config' && CREDENTIAL_CONFIG_DIRS.has(parts[i + 1]))) + return true; + const name = parts[parts.length - 1]; + if (CREDENTIAL_FILES.has(name) || KEY_FILE.test(name)) return true; + // A literal root in a glob can traverse a symlink even with follow:false. + // Check ancestors too, so such a match cannot reach the content readers. + for (let current: Path | undefined = entry; current?.relative(); current = current.parent) { + if (current.isSymbolicLink()) return true; + } + return false; + }; + return { + ignored: (entry) => protectedPath(entry) || (base.ignored?.(entry) ?? false), + childrenIgnored: (entry) => protectedPath(entry) || (base.childrenIgnored?.(entry) ?? false), + }; +}; + /** * The `indexCoverage` fragment recording the active list: `{ includeDotDirs }` * when any directory is listed, otherwise `{}`. diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 30d578e94..4e6dd278f 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -278,7 +278,7 @@ import { } from '../storage/git.js'; import { isGitNexusManagedPath } from '../storage/gitnexus-managed-paths.js'; import { getMaxFileSizeBytes } from './ingestion/utils/max-file-size.js'; -import { includeDotDirsCoverage } from './ingestion/utils/include-dot-dirs.js'; +import { getIncludeDotDirs, includeDotDirsCoverage } from './ingestion/utils/include-dot-dirs.js'; import { cacheRowCount, discardScopedEmbeddingSpills, @@ -2573,13 +2573,17 @@ async function runFullAnalysisInner( applyCollectedForce(); // ── Early-return: already up to date ────────────────────────────── + const includeDotDirsChanged = + JSON.stringify(existingMeta?.indexCoverage?.includeDotDirs ?? []) !== + JSON.stringify(getIncludeDotDirs()); if ( existingMeta && !existingMeta.embeddingCheckpoint && !options.force && existingMeta.lastCommit === currentCommit && !ftsModeChanged && - !processDetectionMismatch + !processDetectionMismatch && + !includeDotDirsChanged ) { // Non-git folders have currentCommit = '' — always rebuild since we can't detect changes if (currentCommit !== '') { diff --git a/gitnexus/test/integration/filesystem-walker.test.ts b/gitnexus/test/integration/filesystem-walker.test.ts index 728a3e461..a05223c3c 100644 --- a/gitnexus/test/integration/filesystem-walker.test.ts +++ b/gitnexus/test/integration/filesystem-walker.test.ts @@ -587,6 +587,102 @@ describe('filesystem-walker', () => { cap.restore(); } }); + + it('keeps credentials out even when ignore rules explicitly unignore them', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-dotdirs-credentials-')); + const blocked = [ + '.GIT/private.ts', + '.aws/credentials', + '.ssh/id_rsa', + '.docker/config.json', + '.config/gcloud/application_default_credentials.json', + '.config/gh/hosts.yml', + '.source/credentials.json', + '.source/private.KEY', + '.source/secrets.yaml', + '.source/.env', + '.source/.npmrc', + '.source/.other/hidden.ts', + ]; + const cap = _captureLogger(); + try { + for (const rel of [...blocked, '.source/code.ts', '.config/tool/source.ts']) { + await fs.mkdir(path.dirname(path.join(root, rel)), { recursive: true }); + await fs.writeFile(path.join(root, rel), 'SYNTHETIC_TEST_ONLY'); + } + await fs.writeFile(path.join(root, '.gitnexusignore'), '!**\n'); + const discovered: string[] = []; + const files = await walkRepositoryPaths(root, undefined, { + includeDotDirs: ['.GIT', '.aws', '.ssh', '.docker', '.config', '.source'], + onPathsDiscovered: async (paths) => { + discovered.push(...paths); + }, + }); + const expected = ['.config/tool/source.ts', '.source/code.ts']; + expect(files.map((file) => file.path)).toEqual(expected); + expect([...discovered].sort()).toEqual(expected); + } finally { + cap.restore(); + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('does not follow opted-in directory links or file aliases to hidden secrets', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-dotdirs-links-')); + const external = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-dotdirs-external-')); + try { + await fs.mkdir(path.join(root, '.source')); + await fs.writeFile(path.join(root, '.source', 'code.ts'), 'export const safe = 1;'); + await fs.writeFile(path.join(root, '.source', '.env'), 'SYNTHETIC_TEST_ONLY'); + await fs.writeFile(path.join(external, 'private.ts'), 'SYNTHETIC_TEST_ONLY'); + await fs.symlink( + external, + path.join(root, '.linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + await fs.symlink( + external, + path.join(root, '.source', 'nested'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + // File symlinks require privileges on Windows; directory junctions above do not. + if (process.platform !== 'win32') { + await fs.symlink( + path.join(root, '.source', '.env'), + path.join(root, '.source', 'alias.ts'), + ); + } + const discovered: string[] = []; + const files = await walkRepositoryPaths(root, undefined, { + includeDotDirs: ['.linked', '.source'], + onPathsDiscovered: async (paths) => { + discovered.push(...paths); + }, + }); + expect(files.map((file) => file.path)).toEqual(['.source/code.ts']); + expect(discovered).toEqual(['.source/code.ts']); + } finally { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(external, { recursive: true, force: true }); + } + }); + + it('treats a listed name that is a file as no directory, including nested names', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-dotdirs-file-')); + try { + await fs.mkdir(path.join(root, 'pkg')); + await fs.writeFile(path.join(root, '.source'), 'SYNTHETIC_TEST_ONLY'); + await fs.writeFile(path.join(root, 'pkg', '.source'), 'SYNTHETIC_TEST_ONLY'); + await fs.writeFile(path.join(root, 'index.ts'), 'export {};'); + const files = await walkRepositoryPaths(root, undefined, { + includeDotDirs: ['.source'], + onPathsDiscovered: async () => {}, + }); + expect(files.map((file) => file.path)).toEqual(['index.ts']); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); }); describe('readFileContents', () => { diff --git a/gitnexus/test/unit/group/include-extractor.test.ts b/gitnexus/test/unit/group/include-extractor.test.ts index 614229fb6..528aaedfc 100644 --- a/gitnexus/test/unit/group/include-extractor.test.ts +++ b/gitnexus/test/unit/group/include-extractor.test.ts @@ -543,6 +543,42 @@ int main(){return 0;}`, // ---- PR #1156 Codex follow-up: discovery aligned with ingestion ---- describe('follow-up: file discovery honors createIgnoreFilter and getMaxFileSizeBytes', () => { + it('uses the same credential and symlink exclusions as ingestion for dot directories', async () => { + const previous = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + const external = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-external-')); + try { + writeFile('.gitnexusignore', '!**\n'); + writeFile('.config/tool/allowed.h', '#pragma once'); + writeFile('.config/gcloud/private.h', '#pragma once'); + writeFile('.source/local.h', '#pragma once'); + writeFile('.source/.env', 'SYNTHETIC_TEST_ONLY'); + fs.writeFileSync(path.join(external, 'private.h'), '#pragma once'); + fs.symlinkSync( + external, + path.join(tmpDir, '.linked'), + process.platform === 'win32' ? 'junction' : 'dir', + ); + if (process.platform !== 'win32') { + fs.symlinkSync( + path.join(tmpDir, '.source', '.env'), + path.join(tmpDir, '.source', 'alias.h'), + ); + } + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.config,.source,.linked'; + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + expect( + contracts + .filter((c) => c.role === 'provider') + .map((c) => c.contractId) + .sort(), + ).toEqual(['include::.config/tool/allowed.h', 'include::.source/local.h']); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + else process.env.GITNEXUS_INCLUDE_DOT_DIRS = previous; + fs.rmSync(external, { recursive: true, force: true }); + } + }); + it('does not emit a provider contract for a header excluded by .gitignore', async () => { writeFile('.gitignore', 'vendor-headers/\n'); writeFile('vendor-headers/blocked.h', '#pragma once'); diff --git a/gitnexus/test/unit/include-dot-dirs.test.ts b/gitnexus/test/unit/include-dot-dirs.test.ts index ab9a3ed3f..3b12aa60d 100644 --- a/gitnexus/test/unit/include-dot-dirs.test.ts +++ b/gitnexus/test/unit/include-dot-dirs.test.ts @@ -47,6 +47,24 @@ describe('include-dot-dirs', () => { expect(messages.some((m) => m.includes('".hg"'))).toBe(true); }); + it.each(['.GIT', '.SvN', '.HG', '.Bzr', '.ssh', '.AWS', '.azure', '.kube', '.gnupg', '.docker'])( + 'refuses the protected directory %s regardless of case', + (entry) => { + expect(normalizeIncludeDotDirs([entry])).toEqual([]); + }, + ); + + it.each(['...', '.git.', '.ssh.'])('rejects Windows path aliases such as %s', (entry) => { + expect(normalizeIncludeDotDirs([entry])).toEqual([]); + }); + + it('does not echo invalid environment values into logs', () => { + const invalid = 'SYNTHETIC_PRIVATE_VALUE\nforged log entry'; + normalizeIncludeDotDirs([invalid]); + expect(cap.records()).toHaveLength(1); + expect(JSON.stringify(cap.records())).not.toContain(invalid.split('\n')[0]); + }); + it.each(['scallywag', '.', '..', 'a/.b', '.a/b', '.a\\b', '.{a,b}', '.a*', '.[ab]', '!.a'])( 'rejects %j as not a plain dot-directory name', (entry) => { diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index 9086c880c..c6de59d07 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -46,6 +46,64 @@ const currentParserProvenance = () => { }; describe('run-analyze module', () => { + it('adds and removes dot-directory coverage on a clean same-commit analyze', async () => { + const tmpRepo = await createTempDir('gitnexus-dotdirs-policy-'); + const tmpHome = await createTempDir('gitnexus-dotdirs-home-'); + const savedHome = process.env.GITNEXUS_HOME; + const savedDirs = process.env.GITNEXUS_INCLUDE_DOT_DIRS; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + try { + const git = (...args: string[]) => + execFileSync('git', args, { cwd: tmpRepo.dbPath, stdio: 'pipe' }); + git('init'); + await fs.mkdir(path.join(tmpRepo.dbPath, '.source')); + await fs.writeFile( + path.join(tmpRepo.dbPath, 'main.ts'), + 'export function main() { return 1; }', + ); + await fs.writeFile( + path.join(tmpRepo.dbPath, '.source', 'hidden.ts'), + 'export function hidden() { return 2; }', + ); + git('add', '.'); + git('-c', 'user.name=test', '-c', 'user.email=test@test', 'commit', '-m', 'fixture'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const analyze = () => + runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true, skipFts: true }, + { onProgress: () => {} }, + ); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await analyze(); + expect((await loadMeta(storagePath))?.fileHashes).not.toHaveProperty('.source/hidden.ts'); + expect((await analyze()).alreadyUpToDate).toBe(true); + + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.source'; + expect((await analyze()).alreadyUpToDate).not.toBe(true); + const included = await loadMeta(storagePath); + expect(Object.keys(included?.fileHashes ?? {})).toContain('.source/hidden.ts'); + expect(included?.indexCoverage?.includeDotDirs).toEqual(['.source']); + + process.env.GITNEXUS_INCLUDE_DOT_DIRS = ' .source/, .source '; + expect((await analyze()).alreadyUpToDate).toBe(true); + delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + expect((await analyze()).alreadyUpToDate).not.toBe(true); + expect(Object.keys((await loadMeta(storagePath))?.fileHashes ?? {})).not.toContain( + '.source/hidden.ts', + ); + expect((await loadMeta(storagePath))?.indexCoverage?.includeDotDirs).toBeUndefined(); + } finally { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + if (savedDirs === undefined) delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; + else process.env.GITNEXUS_INCLUDE_DOT_DIRS = savedDirs; + await tmpRepo.cleanup(); + await tmpHome.cleanup(); + } + }, 120_000); + it('exports runFullAnalysis as a function', async () => { const mod = await import('../../src/core/run-analyze.js'); expect(typeof mod.runFullAnalysis).toBe('function'); From 38d18906cb45d30d6cd3a5018afa00a0cbafbb11 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 28 Oct 2026 20:25:07 +0000 Subject: [PATCH 4/5] fix(ingestion): use glob public path type Keep the parse-cache dependency ownership audit intact and strengthen the unlisted-directory control with a non-ignored path. --- gitnexus/src/core/ingestion/utils/include-dot-dirs.ts | 3 +-- gitnexus/test/unit/group/include-extractor.test.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts index 0db3a38cf..21ea63361 100644 --- a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts +++ b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts @@ -1,6 +1,5 @@ import { logger } from '../../logger.js'; -import type { IgnoreLike } from 'glob'; -import type { Path } from 'path-scurry'; +import type { IgnoreLike, Path } from 'glob'; /** * Directory names that must never be indexed, even when listed. These hold diff --git a/gitnexus/test/unit/group/include-extractor.test.ts b/gitnexus/test/unit/group/include-extractor.test.ts index 528aaedfc..fdda09ae4 100644 --- a/gitnexus/test/unit/group/include-extractor.test.ts +++ b/gitnexus/test/unit/group/include-extractor.test.ts @@ -663,7 +663,7 @@ int auto_main() { return 0; }`, it('registers headers in a dot directory only when GITNEXUS_INCLUDE_DOT_DIRS lists it', async () => { const previous = process.env.GITNEXUS_INCLUDE_DOT_DIRS; writeFile('.config/listed.h', '#pragma once'); - writeFile('.cache/unlisted.h', '#pragma once'); + writeFile('.unlisted-source/unlisted.h', '#pragma once'); try { delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; const before = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); @@ -674,7 +674,13 @@ int auto_main() { return 0; }`, const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); expect(providerIds).toContain('include::.config/listed.h'); - expect(providerIds).not.toContain('include::.cache/unlisted.h'); + expect(providerIds).not.toContain('include::.unlisted-source/unlisted.h'); + + process.env.GITNEXUS_INCLUDE_DOT_DIRS = '.config,.unlisted-source'; + const both = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + expect(both.filter((c) => c.role === 'provider').map((p) => p.contractId)).toContain( + 'include::.unlisted-source/unlisted.h', + ); } finally { if (previous === undefined) delete process.env.GITNEXUS_INCLUDE_DOT_DIRS; else process.env.GITNEXUS_INCLUDE_DOT_DIRS = previous; From 2fa8694342c9a1e653e593bc433099997e7b48a4 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Wed, 28 Oct 2026 21:19:17 +0000 Subject: [PATCH 5/5] fix(ingestion): verify symlinks after globbing Glob can emit descendants before its cached parent type identifies a symlink. Verify filesystem ancestry before metadata callbacks and content readers, sharing parent checks across candidates. Regression coverage includes already-emitted linked paths and fail-closed inspection errors. A 1000-walk stress probe changed from 36 leaks to zero. --- .../group/extractors/include-extractor.ts | 7 ++- .../src/core/ingestion/filesystem-walker.ts | 5 +- .../core/ingestion/utils/include-dot-dirs.ts | 39 ++++++++++++- gitnexus/test/unit/include-dot-dirs.test.ts | 56 ++++++++++++++++++- 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts index d669346f6..4fe811151 100644 --- a/gitnexus/src/core/group/extractors/include-extractor.ts +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -28,6 +28,7 @@ import { getIncludeDotDirs, indexableGlobPatterns, dotDirectoryIgnoreFilter, + filterDotDirectorySymlinks, } from '../../ingestion/utils/include-dot-dirs.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; import { logger } from '../../logger.js'; @@ -400,7 +401,11 @@ export class IncludeExtractor implements ContractExtractor { }); const survivors: string[] = []; - for (const rel of candidates) { + const filtered = + includeDotDirs.length > 0 + ? await filterDotDirectorySymlinks(repoPath, candidates) + : candidates; + for (const rel of filtered) { try { const stat = await fs.stat(path.join(repoPath, rel)); if (!stat.isFile()) continue; diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index c02d9de67..baf2faf3a 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -5,6 +5,7 @@ import { indexableGlobPatterns, normalizeIncludeDotDirs, dotDirectoryIgnoreFilter, + filterDotDirectorySymlinks, } from './utils/include-dot-dirs.js'; import fs from 'fs/promises'; import { readdir } from 'node:fs'; @@ -130,7 +131,7 @@ export const walkRepositoryPaths = async ( // directory; ENOTDIR means no matching subtree, not an unreadable one. (error.code === 'ENOTDIR' && includeDotDirs.includes(path.basename(String(directory)))); - const filtered = ( + const candidates = ( await glob(patterns, { cwd: repoPath, nodir: true, @@ -152,6 +153,8 @@ export const walkRepositoryPaths = async ( }) ).map((filePath) => filePath.replace(/\\/g, '/')); if (enumerationError !== undefined) throw enumerationError; + const filtered = + includeDotDirs.length > 0 ? await filterDotDirectorySymlinks(repoPath, candidates) : candidates; await options.onPathsDiscovered?.(filtered); const entries: ScannedFile[] = []; let processed = 0; diff --git a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts index 21ea63361..1934bb475 100644 --- a/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts +++ b/gitnexus/src/core/ingestion/utils/include-dot-dirs.ts @@ -1,5 +1,7 @@ import { logger } from '../../logger.js'; import type { IgnoreLike, Path } from 'glob'; +import fs from 'node:fs/promises'; +import path from 'node:path'; /** * Directory names that must never be indexed, even when listed. These hold @@ -126,8 +128,9 @@ export const dotDirectoryIgnoreFilter = (base: IgnoreLike): IgnoreLike => { return true; const name = parts[parts.length - 1]; if (CREDENTIAL_FILES.has(name) || KEY_FILE.test(name)) return true; - // A literal root in a glob can traverse a symlink even with follow:false. - // Check ancestors too, so such a match cannot reach the content readers. + // Prune known links here. Glob can emit descendants before its cached + // parent type is known; filterDotDirectorySymlinks checks the filesystem + // after enumeration, before any metadata callback or content reader. for (let current: Path | undefined = entry; current?.relative(); current = current.parent) { if (current.isSymbolicLink()) return true; } @@ -139,6 +142,38 @@ export const dotDirectoryIgnoreFilter = (base: IgnoreLike): IgnoreLike => { }; }; +/** Validate actual ancestry rather than relying on glob's asynchronously populated cache. */ +export const filterDotDirectorySymlinks = async ( + repoPath: string, + candidates: readonly string[], +): Promise => { + const checked = new Map>(); + const isSafe = (relativePath: string): Promise => { + if (relativePath === '.') return Promise.resolve(true); + const cached = checked.get(relativePath); + if (cached) return cached; + const result = (async () => { + // Check parents first so a linked directory's descendants are never inspected. + if (!(await isSafe(path.dirname(relativePath)))) return false; + try { + return !(await fs.lstat(path.join(repoPath, relativePath))).isSymbolicLink(); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT' || code === 'ENOTDIR') return false; + throw error; + } + })(); + checked.set(relativePath, result); + return result; + }; + const safe = await Promise.all( + candidates.map((candidate) => + candidate.split(/[\\/]/).some((part) => part.startsWith('.')) ? isSafe(candidate) : true, + ), + ); + return candidates.filter((_, index) => safe[index]); +}; + /** * The `indexCoverage` fragment recording the active list: `{ includeDotDirs }` * when any directory is listed, otherwise `{}`. diff --git a/gitnexus/test/unit/include-dot-dirs.test.ts b/gitnexus/test/unit/include-dot-dirs.test.ts index 3b12aa60d..7ebaffa29 100644 --- a/gitnexus/test/unit/include-dot-dirs.test.ts +++ b/gitnexus/test/unit/include-dot-dirs.test.ts @@ -1,9 +1,13 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; import { getIncludeDotDirs, includeDotDirsCoverage, indexableGlobPatterns, normalizeIncludeDotDirs, + filterDotDirectorySymlinks, _resetIncludeDotDirsWarnings, } from '../../src/core/ingestion/utils/include-dot-dirs.js'; import { _captureLogger } from '../../src/core/logger.js'; @@ -92,4 +96,54 @@ describe('include-dot-dirs', () => { normalizeIncludeDotDirs(['bad']); expect(cap.records().length).toBe(1); }); + + it('rejects already-enumerated paths beneath directory links before content discovery', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-dot-links-')); + const repo = path.join(root, 'repo'); + const external = path.join(root, 'external'); + try { + await fs.mkdir(path.join(repo, '.source'), { recursive: true }); + await fs.mkdir(path.join(external, '.source'), { recursive: true }); + await fs.writeFile(path.join(repo, '.source', 'allowed.h'), '#pragma once'); + await fs.writeFile(path.join(repo, 'visible.h'), '#pragma once'); + await fs.writeFile(path.join(external, 'private.h'), 'SYNTHETIC_TEST_ONLY'); + await fs.writeFile(path.join(external, '.source', 'private.h'), 'SYNTHETIC_TEST_ONLY'); + const linkType = process.platform === 'win32' ? 'junction' : 'dir'; + await fs.symlink(external, path.join(repo, '.linked'), linkType); + await fs.symlink(external, path.join(repo, 'alias'), linkType); + // Glob can emit these descendants while its cached parent type is unknown. + const candidates = [ + '.linked/private.h', + 'alias/.source/private.h', + '.source/missing.h', + '.source/allowed.h', + 'visible.h', + ]; + if (process.platform !== 'win32') { + await fs.symlink( + path.join(external, 'private.h'), + path.join(repo, '.source', 'file-link.h'), + ); + candidates.push('.source/file-link.h'); + } + expect(await filterDotDirectorySymlinks(repo, candidates)).toEqual([ + '.source/allowed.h', + 'visible.h', + ]); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); + + it('fails closed when a dot-directory ancestor cannot be inspected', async () => { + const error = Object.assign(new Error('unreadable directory'), { code: 'EACCES' }); + const lstat = vi.spyOn(fs, 'lstat').mockRejectedValue(error); + try { + await expect(filterDotDirectorySymlinks('/synthetic-repo', ['.source/file.h'])).rejects.toBe( + error, + ); + } finally { + lstat.mockRestore(); + } + }); });