From bd8d8ed081d53e5d267916e0d79b477ec2addc12 Mon Sep 17 00:00:00 2001 From: Carter LaSalle Date: Thu, 6 Aug 2026 16:12:45 -0700 Subject: [PATCH] feat(walker): opt into indexing hidden paths via GITNEXUS_INDEX_HIDDEN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `walkRepositoryPaths` passes `dot: false` to glob, so hidden (dot-prefixed) paths are never enumerated. A repository that keeps first-party source under a dot-directory is therefore almost entirely unindexed, and the failure is silent: analyze reports success, and `context()` returns "not found" for symbols whose file has no node at all. `.gitnexusignore` negation cannot recover those paths. Negation only un-filters `DEFAULT_IGNORE_LIST` entries, and that check runs *after* enumeration — a path glob never yielded cannot be re-included. Add `GITNEXUS_INDEX_HIDDEN` as an opt-in. The default stays `false`: `dot: false` is the primary mechanism keeping `.git/`, `.vscode/` and `.venv/` out of the index, with `DEFAULT_IGNORE_LIST` as defense-in-depth behind it, so flipping the default would change the indexed file set for every existing repository. `DEFAULT_IGNORE_LIST` still applies when enabled, so `.git/` and `node_modules/` stay excluded; callers narrow the rest with `.gitnexusignore`. The setting changes which files the walker enumerates, so an index built with it on is not interchangeable with one built with it off. It is registered as an analysis feature (`walker.index-hidden`) so the toggle is part of the index identity and either direction forces the rebuild the new scope requires — without that, enabling it silently no-ops on an existing index because the walker never re-runs. Verified on a two-file fixture (`.hidden_src/buried.py`, `visible/plain.py`): default -> 3 nodes, only visible/plain.py =1 (no clean) -> "missing:walker.index-hidden" -> rebuild -> 6 nodes, both back to default -> "unexpected:walker.index-hidden" -> rebuild -> 3 nodes `.git/` stayed excluded throughout. 14 new unit tests; `npx tsc --noEmit` clean. --- README.md | 1 + gitnexus/src/cli/analyze.ts | 6 + .../src/core/ingestion/filesystem-walker.ts | 9 +- .../src/core/ingestion/utils/index-hidden.ts | 89 +++++++++++++++ gitnexus/src/core/run-analyze.ts | 2 + gitnexus/test/unit/index-hidden.test.ts | 103 ++++++++++++++++++ 6 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 gitnexus/src/core/ingestion/utils/index-hidden.ts create mode 100644 gitnexus/test/unit/index-hidden.test.ts diff --git a/README.md b/README.md index d6534f658..8897fae7e 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | | `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | | `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_INDEX_HIDDEN` | `0` (excluded) | When `1`/`true`, the walker enumerates hidden (dot-prefixed) paths. `DEFAULT_IGNORE_LIST` still applies, so `.git/` and `node_modules/` stay excluded; narrow the rest with `.gitnexusignore`. | Repos that keep first-party source under a dot-directory. Without it those paths are never enumerated, so `.gitnexusignore` negation cannot re-include them and analyze reports success while skipping them. | | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | | `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 6851c2478..33f138959 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -53,6 +53,7 @@ import { import { runFullAnalysis } from '../core/run-analyze.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; +import { getIndexHiddenBannerMessage } from '../core/ingestion/utils/index-hidden.js'; import { warnMissingOptionalGrammars, getOptionalGrammarExtensions } from './optional-grammars.js'; import { glob } from 'glob'; import fs from 'fs/promises'; @@ -1297,6 +1298,11 @@ const analyzeCommandImpl = async ( console.log(`${maxFileSizeBanner}\n`); } + const indexHiddenBanner = getIndexHiddenBannerMessage(); + if (indexHiddenBanner) { + console.log(`${indexHiddenBanner}\n`); + } + // ── CLI progress bar setup ───────────────────────────────────────── const barOptions: cliProgress.Options & { terminal?: CliProgressTerminal } = { format: ' {bar} {percentage}% | {phase}', diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 823b3670f..3d02cc268 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -1,5 +1,6 @@ import { isVerboseIngestionEnabled } from './utils/verbose.js'; import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js'; +import { isIndexHiddenEnabled } from './utils/index-hidden.js'; import fs from 'fs/promises'; import path from 'path'; import { glob } from 'glob'; @@ -49,7 +50,13 @@ export const walkRepositoryPaths = async ( const filtered = await glob('**/*', { cwd: repoPath, nodir: true, - dot: false, + // Hidden paths are excluded by default — this is the primary mechanism that + // keeps `.git/`, `.vscode/` and `.venv/` out of the index, with the + // `DEFAULT_IGNORE_LIST` check in `ignore-service.ts` as defense-in-depth + // behind it. Opt in with `GITNEXUS_INDEX_HIDDEN=1` for repositories that + // keep first-party source under a dot-directory; `DEFAULT_IGNORE_LIST` + // still applies, so `.git/` and `node_modules/` remain excluded. + dot: isIndexHiddenEnabled(), ignore: ignoreFilter, }); const entries: ScannedFile[] = []; diff --git a/gitnexus/src/core/ingestion/utils/index-hidden.ts b/gitnexus/src/core/ingestion/utils/index-hidden.ts new file mode 100644 index 000000000..52ff0454f --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/index-hidden.ts @@ -0,0 +1,89 @@ +import { logger } from '../../logger.js'; +import type { AnalysisFeatureDescriptor } from '../../analysis-features.js'; + +/** + * Default: hidden (dot-prefixed) paths are NOT indexed. + * + * Kept `false` deliberately. `glob`'s `dot` option is the primary mechanism that + * keeps `.git/`, `.vscode/`, `.venv/` and friends out of the index; the + * `DEFAULT_IGNORE_LIST` check in `config/ignore-service.ts` is defense-in-depth + * behind it, not a replacement. Flipping this default would change the indexed + * file set for every existing repository. + */ +export const DEFAULT_INDEX_HIDDEN = false; + +const warned = new Set(); + +const warnOnce = (key: string, message: string): void => { + if (warned.has(key)) return; + warned.add(key); + logger.warn(message); +}; + +/** + * Resolve whether the walker should enumerate hidden (dot-prefixed) paths. + * Reads `GITNEXUS_INDEX_HIDDEN`. Invalid values fall back to the default and + * emit a one-time warning. + * + * Why this exists: a repository that keeps first-party source under a + * dot-directory (for example `.codeops/src/`, or tooling that namespaces its + * package under a leading dot) is silently ~unindexed — the walker never + * enumerates those paths, so `.gitnexusignore` negation cannot re-include them + * either (negation only un-filters `DEFAULT_IGNORE_LIST` entries, which are + * checked *after* enumeration). Analysis still reports success, so the gap is + * invisible. This opt-in makes such repositories indexable without changing + * behavior for anyone else. + * + * `DEFAULT_IGNORE_LIST` still applies when enabled, so `.git/` and + * `node_modules/` stay excluded; narrow the rest with `.gitnexusignore`. + */ +export const isIndexHiddenEnabled = (): boolean => { + const raw = process.env.GITNEXUS_INDEX_HIDDEN; + if (raw === undefined || raw === '') return DEFAULT_INDEX_HIDDEN; + + const normalized = raw.trim().toLowerCase(); + if (normalized === '1' || normalized === 'true') return true; + if (normalized === '0' || normalized === 'false') return false; + + warnOnce( + `invalid:${raw}`, + ` GITNEXUS_INDEX_HIDDEN must be one of 1/0/true/false, got "${raw}" — using default ${DEFAULT_INDEX_HIDDEN}`, + ); + return DEFAULT_INDEX_HIDDEN; +}; + +/** + * Build the CLI banner message announcing that hidden-path indexing is active. + * Returns `null` when the effective setting equals the default — the caller + * should print nothing in that case. + */ +export const getIndexHiddenBannerMessage = (): string | null => { + if (isIndexHiddenEnabled() === DEFAULT_INDEX_HIDDEN) return null; + return ' GITNEXUS_INDEX_HIDDEN: indexing hidden (dot-prefixed) paths — DEFAULT_IGNORE_LIST still applies'; +}; + +/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */ +export const _resetIndexHiddenWarnings = (): void => { + warned.clear(); +}; + +/** + * Hidden-path indexing changes *which files the walker enumerates*, so an index + * built with it enabled is not interchangeable with one built without it. + * + * Declaring it as an analysis feature makes the toggle part of the index + * identity, so flipping it forces the full rebuild the new scope requires. + * Both directions are covered because `findAnalysisFeatureMismatches` treats a + * missing key and an extra key as mismatches: enabling it reports + * `missing:walker.index-hidden` against an index built without it, and + * disabling it leaves a recorded key this build no longer promises. + * + * Without this, `GITNEXUS_INDEX_HIDDEN=1` silently no-ops on an existing index — + * the walker is never re-run, so the newly in-scope files are never picked up + * and analyze still reports success. + */ +export const INDEX_HIDDEN_SCOPE_FEATURE: AnalysisFeatureDescriptor = { + id: 'walker.index-hidden', + version: 1, + appliesTo: () => isIndexHiddenEnabled(), +}; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index b2fa92ab9..23c07a4a0 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -160,6 +160,7 @@ import { SPRING_CONDITIONALS_FEATURE, } from './ingestion/frameworks/spring/analysis-features.js'; import { SPRING_CONFIG_BINDINGS_FEATURE } from './ingestion/languages/java/analysis-features.js'; +import { INDEX_HIDDEN_SCOPE_FEATURE } from './ingestion/utils/index-hidden.js'; import { CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, findAnalysisFeatureMismatches, @@ -208,6 +209,7 @@ const stripControlCharacters = (msg: string): string => const ANALYSIS_FEATURES = [ CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, + INDEX_HIDDEN_SCOPE_FEATURE, SPRING_AOP_FEATURE, SPRING_BEAN_INVENTORY_FEATURE, SPRING_CONDITIONALS_FEATURE, diff --git a/gitnexus/test/unit/index-hidden.test.ts b/gitnexus/test/unit/index-hidden.test.ts new file mode 100644 index 000000000..a3af13be0 --- /dev/null +++ b/gitnexus/test/unit/index-hidden.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + DEFAULT_INDEX_HIDDEN, + isIndexHiddenEnabled, + getIndexHiddenBannerMessage, + _resetIndexHiddenWarnings, +} from '../../src/core/ingestion/utils/index-hidden.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +describe('isIndexHiddenEnabled', () => { + const ORIGINAL = process.env.GITNEXUS_INDEX_HIDDEN; + let cap: ReturnType; + + beforeEach(() => { + delete process.env.GITNEXUS_INDEX_HIDDEN; + _resetIndexHiddenWarnings(); + cap = _captureLogger(); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_INDEX_HIDDEN; + } else { + process.env.GITNEXUS_INDEX_HIDDEN = ORIGINAL; + } + cap.restore(); + }); + + it('defaults to excluding hidden paths when the env var is unset', () => { + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(DEFAULT_INDEX_HIDDEN).toBe(false); + expect(cap.records().length).toBe(0); + }); + + it('treats an empty value as unset rather than as a directive', () => { + process.env.GITNEXUS_INDEX_HIDDEN = ''; + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(cap.records().length).toBe(0); + }); + + for (const truthy of ['1', 'true', 'TRUE', ' true ']) { + it(`enables hidden-path indexing for ${JSON.stringify(truthy)}`, () => { + process.env.GITNEXUS_INDEX_HIDDEN = truthy; + expect(isIndexHiddenEnabled()).toBe(true); + expect(cap.records().length).toBe(0); + }); + } + + for (const falsy of ['0', 'false', 'FALSE', ' false ']) { + it(`keeps hidden paths excluded for ${JSON.stringify(falsy)}`, () => { + process.env.GITNEXUS_INDEX_HIDDEN = falsy; + expect(isIndexHiddenEnabled()).toBe(false); + expect(cap.records().length).toBe(0); + }); + } + + it('falls back to the default and warns once on an unparseable value', () => { + process.env.GITNEXUS_INDEX_HIDDEN = 'yes-please'; + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + + const records = cap.records(); + expect(records.length).toBe(1); + expect(records[0]?.msg).toContain('GITNEXUS_INDEX_HIDDEN'); + + // Repeated reads must not re-warn — the walker calls this per analyze run. + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(cap.records().length).toBe(1); + }); +}); + +describe('getIndexHiddenBannerMessage', () => { + const ORIGINAL = process.env.GITNEXUS_INDEX_HIDDEN; + + beforeEach(() => { + delete process.env.GITNEXUS_INDEX_HIDDEN; + _resetIndexHiddenWarnings(); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_INDEX_HIDDEN; + } else { + process.env.GITNEXUS_INDEX_HIDDEN = ORIGINAL; + } + }); + + it('returns null when the effective setting equals the default', () => { + expect(getIndexHiddenBannerMessage()).toBeNull(); + }); + + it('returns null when explicitly set to the default value', () => { + process.env.GITNEXUS_INDEX_HIDDEN = '0'; + expect(getIndexHiddenBannerMessage()).toBeNull(); + }); + + it('announces the override when hidden-path indexing is enabled', () => { + process.env.GITNEXUS_INDEX_HIDDEN = '1'; + const banner = getIndexHiddenBannerMessage(); + expect(banner).toContain('GITNEXUS_INDEX_HIDDEN'); + // Operators must be told the safety net still applies. + expect(banner).toContain('DEFAULT_IGNORE_LIST'); + }); +});