diff --git a/README.md b/README.md index d6534f658..8897fae7e 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PROFILE_DEFERRED_SLOW_MS` | `3000` (verbose) / `5000` | Per-file threshold in ms above which `processCallsFromExtracted` emits a `slow file …` log line. Parsed via `Number()`: accepts integers (`5000`), scientific notation (`2.5e3`), decimals (`.5`), and hex (`0x10`). Non-finite or non-positive values fall back to the default. | Hunting a few outlier files dominating the deferred call-resolution stage; lower to surface more, raise to focus only on the worst. | | `PROF_LBUG_LOAD` | unset | When `1`, emits one `[lbug-load prof]` summary line per `loadGraphToLbug` call breaking the graph-DB persistence wall into stages (`csv-emit` / `copy-nodes` / `copy-rels` / `fallback` / `total`) plus node & edge counts. Zero-cost when unset. | Attributing large-repo analyze wall time across CSV generation vs. LadybugDB `COPY` (issue #2203) — the analyze "emit" timing is the scope-resolution bucket, not this DB-write path. | | `GITNEXUS_MAX_FILE_SIZE` | `512` (KB) | Walker skip threshold in KB. Hard cap is `32768` (tree-sitter buffer ceiling). Equivalent to `--max-file-size `. | Indexing repos with intentionally-large source files (generated parsers, vendored bundles) that should still be parsed. | +| `GITNEXUS_INDEX_HIDDEN` | `0` (excluded) | When `1`/`true`, the walker enumerates hidden (dot-prefixed) paths. `DEFAULT_IGNORE_LIST` still applies, so `.git/` and `node_modules/` stay excluded; narrow the rest with `.gitnexusignore`. | Repos that keep first-party source under a dot-directory. Without it those paths are never enumerated, so `.gitnexusignore` negation cannot re-include them and analyze reports success while skipping them. | | `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS` | `30000` | Worker idle timeout in milliseconds before retry/fallback. Equivalent to `--worker-timeout ` × 1000. | Slow-parsing files (large minified JS, deeply-nested TS types) that legitimately need more than 30s. | | `GITNEXUS_WORKER_READY_TIMEOUT_MS` | `5000` | Startup budget in milliseconds for a parse worker to load its grammar bindings and report `{type:'ready'}`. Slots that miss it are treated as startup crashes. | Slow or heavily loaded hosts where a full pool cold-starting concurrently needs more than 5s, and analyze aborts with "did not report ready within 5000ms". | | `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 6851c2478..33f138959 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -53,6 +53,7 @@ import { import { runFullAnalysis } from '../core/run-analyze.js'; import { getRuntimeFingerprint } from '../core/platform/capabilities.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; +import { getIndexHiddenBannerMessage } from '../core/ingestion/utils/index-hidden.js'; import { warnMissingOptionalGrammars, getOptionalGrammarExtensions } from './optional-grammars.js'; import { glob } from 'glob'; import fs from 'fs/promises'; @@ -1297,6 +1298,11 @@ const analyzeCommandImpl = async ( console.log(`${maxFileSizeBanner}\n`); } + const indexHiddenBanner = getIndexHiddenBannerMessage(); + if (indexHiddenBanner) { + console.log(`${indexHiddenBanner}\n`); + } + // ── CLI progress bar setup ───────────────────────────────────────── const barOptions: cliProgress.Options & { terminal?: CliProgressTerminal } = { format: ' {bar} {percentage}% | {phase}', diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 823b3670f..3d02cc268 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -1,5 +1,6 @@ import { isVerboseIngestionEnabled } from './utils/verbose.js'; import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js'; +import { isIndexHiddenEnabled } from './utils/index-hidden.js'; import fs from 'fs/promises'; import path from 'path'; import { glob } from 'glob'; @@ -49,7 +50,13 @@ export const walkRepositoryPaths = async ( const filtered = await glob('**/*', { cwd: repoPath, nodir: true, - dot: false, + // Hidden paths are excluded by default — this is the primary mechanism that + // keeps `.git/`, `.vscode/` and `.venv/` out of the index, with the + // `DEFAULT_IGNORE_LIST` check in `ignore-service.ts` as defense-in-depth + // behind it. Opt in with `GITNEXUS_INDEX_HIDDEN=1` for repositories that + // keep first-party source under a dot-directory; `DEFAULT_IGNORE_LIST` + // still applies, so `.git/` and `node_modules/` remain excluded. + dot: isIndexHiddenEnabled(), ignore: ignoreFilter, }); const entries: ScannedFile[] = []; diff --git a/gitnexus/src/core/ingestion/utils/index-hidden.ts b/gitnexus/src/core/ingestion/utils/index-hidden.ts new file mode 100644 index 000000000..52ff0454f --- /dev/null +++ b/gitnexus/src/core/ingestion/utils/index-hidden.ts @@ -0,0 +1,89 @@ +import { logger } from '../../logger.js'; +import type { AnalysisFeatureDescriptor } from '../../analysis-features.js'; + +/** + * Default: hidden (dot-prefixed) paths are NOT indexed. + * + * Kept `false` deliberately. `glob`'s `dot` option is the primary mechanism that + * keeps `.git/`, `.vscode/`, `.venv/` and friends out of the index; the + * `DEFAULT_IGNORE_LIST` check in `config/ignore-service.ts` is defense-in-depth + * behind it, not a replacement. Flipping this default would change the indexed + * file set for every existing repository. + */ +export const DEFAULT_INDEX_HIDDEN = false; + +const warned = new Set(); + +const warnOnce = (key: string, message: string): void => { + if (warned.has(key)) return; + warned.add(key); + logger.warn(message); +}; + +/** + * Resolve whether the walker should enumerate hidden (dot-prefixed) paths. + * Reads `GITNEXUS_INDEX_HIDDEN`. Invalid values fall back to the default and + * emit a one-time warning. + * + * Why this exists: a repository that keeps first-party source under a + * dot-directory (for example `.codeops/src/`, or tooling that namespaces its + * package under a leading dot) is silently ~unindexed — the walker never + * enumerates those paths, so `.gitnexusignore` negation cannot re-include them + * either (negation only un-filters `DEFAULT_IGNORE_LIST` entries, which are + * checked *after* enumeration). Analysis still reports success, so the gap is + * invisible. This opt-in makes such repositories indexable without changing + * behavior for anyone else. + * + * `DEFAULT_IGNORE_LIST` still applies when enabled, so `.git/` and + * `node_modules/` stay excluded; narrow the rest with `.gitnexusignore`. + */ +export const isIndexHiddenEnabled = (): boolean => { + const raw = process.env.GITNEXUS_INDEX_HIDDEN; + if (raw === undefined || raw === '') return DEFAULT_INDEX_HIDDEN; + + const normalized = raw.trim().toLowerCase(); + if (normalized === '1' || normalized === 'true') return true; + if (normalized === '0' || normalized === 'false') return false; + + warnOnce( + `invalid:${raw}`, + ` GITNEXUS_INDEX_HIDDEN must be one of 1/0/true/false, got "${raw}" — using default ${DEFAULT_INDEX_HIDDEN}`, + ); + return DEFAULT_INDEX_HIDDEN; +}; + +/** + * Build the CLI banner message announcing that hidden-path indexing is active. + * Returns `null` when the effective setting equals the default — the caller + * should print nothing in that case. + */ +export const getIndexHiddenBannerMessage = (): string | null => { + if (isIndexHiddenEnabled() === DEFAULT_INDEX_HIDDEN) return null; + return ' GITNEXUS_INDEX_HIDDEN: indexing hidden (dot-prefixed) paths — DEFAULT_IGNORE_LIST still applies'; +}; + +/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */ +export const _resetIndexHiddenWarnings = (): void => { + warned.clear(); +}; + +/** + * Hidden-path indexing changes *which files the walker enumerates*, so an index + * built with it enabled is not interchangeable with one built without it. + * + * Declaring it as an analysis feature makes the toggle part of the index + * identity, so flipping it forces the full rebuild the new scope requires. + * Both directions are covered because `findAnalysisFeatureMismatches` treats a + * missing key and an extra key as mismatches: enabling it reports + * `missing:walker.index-hidden` against an index built without it, and + * disabling it leaves a recorded key this build no longer promises. + * + * Without this, `GITNEXUS_INDEX_HIDDEN=1` silently no-ops on an existing index — + * the walker is never re-run, so the newly in-scope files are never picked up + * and analyze still reports success. + */ +export const INDEX_HIDDEN_SCOPE_FEATURE: AnalysisFeatureDescriptor = { + id: 'walker.index-hidden', + version: 1, + appliesTo: () => isIndexHiddenEnabled(), +}; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index b2fa92ab9..23c07a4a0 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -160,6 +160,7 @@ import { SPRING_CONDITIONALS_FEATURE, } from './ingestion/frameworks/spring/analysis-features.js'; import { SPRING_CONFIG_BINDINGS_FEATURE } from './ingestion/languages/java/analysis-features.js'; +import { INDEX_HIDDEN_SCOPE_FEATURE } from './ingestion/utils/index-hidden.js'; import { CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, findAnalysisFeatureMismatches, @@ -208,6 +209,7 @@ const stripControlCharacters = (msg: string): string => const ANALYSIS_FEATURES = [ CLASS_FRAMEWORK_ANNOTATIONS_FEATURE, + INDEX_HIDDEN_SCOPE_FEATURE, SPRING_AOP_FEATURE, SPRING_BEAN_INVENTORY_FEATURE, SPRING_CONDITIONALS_FEATURE, diff --git a/gitnexus/test/unit/index-hidden.test.ts b/gitnexus/test/unit/index-hidden.test.ts new file mode 100644 index 000000000..a3af13be0 --- /dev/null +++ b/gitnexus/test/unit/index-hidden.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + DEFAULT_INDEX_HIDDEN, + isIndexHiddenEnabled, + getIndexHiddenBannerMessage, + _resetIndexHiddenWarnings, +} from '../../src/core/ingestion/utils/index-hidden.js'; +import { _captureLogger } from '../../src/core/logger.js'; + +describe('isIndexHiddenEnabled', () => { + const ORIGINAL = process.env.GITNEXUS_INDEX_HIDDEN; + let cap: ReturnType; + + beforeEach(() => { + delete process.env.GITNEXUS_INDEX_HIDDEN; + _resetIndexHiddenWarnings(); + cap = _captureLogger(); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_INDEX_HIDDEN; + } else { + process.env.GITNEXUS_INDEX_HIDDEN = ORIGINAL; + } + cap.restore(); + }); + + it('defaults to excluding hidden paths when the env var is unset', () => { + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(DEFAULT_INDEX_HIDDEN).toBe(false); + expect(cap.records().length).toBe(0); + }); + + it('treats an empty value as unset rather than as a directive', () => { + process.env.GITNEXUS_INDEX_HIDDEN = ''; + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(cap.records().length).toBe(0); + }); + + for (const truthy of ['1', 'true', 'TRUE', ' true ']) { + it(`enables hidden-path indexing for ${JSON.stringify(truthy)}`, () => { + process.env.GITNEXUS_INDEX_HIDDEN = truthy; + expect(isIndexHiddenEnabled()).toBe(true); + expect(cap.records().length).toBe(0); + }); + } + + for (const falsy of ['0', 'false', 'FALSE', ' false ']) { + it(`keeps hidden paths excluded for ${JSON.stringify(falsy)}`, () => { + process.env.GITNEXUS_INDEX_HIDDEN = falsy; + expect(isIndexHiddenEnabled()).toBe(false); + expect(cap.records().length).toBe(0); + }); + } + + it('falls back to the default and warns once on an unparseable value', () => { + process.env.GITNEXUS_INDEX_HIDDEN = 'yes-please'; + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + + const records = cap.records(); + expect(records.length).toBe(1); + expect(records[0]?.msg).toContain('GITNEXUS_INDEX_HIDDEN'); + + // Repeated reads must not re-warn — the walker calls this per analyze run. + expect(isIndexHiddenEnabled()).toBe(DEFAULT_INDEX_HIDDEN); + expect(cap.records().length).toBe(1); + }); +}); + +describe('getIndexHiddenBannerMessage', () => { + const ORIGINAL = process.env.GITNEXUS_INDEX_HIDDEN; + + beforeEach(() => { + delete process.env.GITNEXUS_INDEX_HIDDEN; + _resetIndexHiddenWarnings(); + }); + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.GITNEXUS_INDEX_HIDDEN; + } else { + process.env.GITNEXUS_INDEX_HIDDEN = ORIGINAL; + } + }); + + it('returns null when the effective setting equals the default', () => { + expect(getIndexHiddenBannerMessage()).toBeNull(); + }); + + it('returns null when explicitly set to the default value', () => { + process.env.GITNEXUS_INDEX_HIDDEN = '0'; + expect(getIndexHiddenBannerMessage()).toBeNull(); + }); + + it('announces the override when hidden-path indexing is enabled', () => { + process.env.GITNEXUS_INDEX_HIDDEN = '1'; + const banner = getIndexHiddenBannerMessage(); + expect(banner).toContain('GITNEXUS_INDEX_HIDDEN'); + // Operators must be told the safety net still applies. + expect(banner).toContain('DEFAULT_IGNORE_LIST'); + }); +});